From 31ce04c655d1fc6da7a0eac095f09f78ffa9854f Mon Sep 17 00:00:00 2001 From: YAO Date: Wed, 17 Jun 2026 18:43:02 +0800 Subject: [PATCH] Split meta skill CLI and review gates Merge the beta-ready Yao Meta Skill architecture, report, evidence gate, and release-boundary updates.\n\nRelease boundary: beta/public testing is allowed; formal world-class, fully reviewed, or superiority claims remain blocked until the pending evidence gates are accepted. --- .gitignore | 4 + .../geo-content-brief-skill/README.md | 72 + .../geo-content-brief-skill/SKILL.md | 57 + .../agents/interface.yaml | 29 + .../evals/output_cases.jsonl | 2 + .../evals/trigger_cases.jsonl | 4 + .../geo-content-brief-skill/manifest.json | 24 + .../references/brief-structure.md | 23 + .../references/review-checklist.md | 11 + .../reports/adoption_drift_report.json | 62 + .../reports/adoption_drift_report.md | 36 + .../reports/artifact-design-profile.json | 115 + .../reports/artifact-design-profile.md | 93 + .../reports/compiled_targets.json | 2229 ++ .../reports/compiled_targets.md | 73 + .../reports/intent-confidence.json | 83 + .../reports/intent-confidence.md | 32 + .../reports/intent-context.json | 11 + .../reports/intent-dialogue.json | 79 + .../reports/intent-dialogue.md | 75 + .../reports/iteration-directions.json | 52 + .../reports/iteration-directions.md | 62 + .../reports/output-risk-profile.json | 147 + .../reports/output-risk-profile.md | 60 + .../reports/prompt-quality-profile.json | 146 + .../reports/prompt-quality-profile.md | 94 + .../reports/reference-scan.json | 36 + .../reports/reference-scan.md | 54 + .../reports/reference-synthesis.json | 171 + .../reports/reference-synthesis.md | 81 + .../reports/review-studio.html | 720 + .../reports/review-studio.json | 4450 ++++ .../reports/review-viewer.html | 376 + .../reports/review-viewer.json | 1921 ++ .../reports/review_annotations.json | 24 + .../reports/review_annotations.md | 19 + .../reports/review_waivers.json | 44 + .../reports/review_waivers.md | 34 + .../reports/skill-ir.json | 93 + .../reports/skill-overview.html | 986 + .../reports/skill-overview.json | 1060 + .../reports/system-model.json | 166 + .../reports/system-model.md | 156 + AGENTS.md | 65 +- Makefile | 91 +- README.md | 98 +- SKILL.md | 48 +- agents/interface.yaml | 2 + assets/review-studio.css | 749 + assets/review-viewer.css | 176 + assets/skill-overview.css | 735 + assets/skill-overview.js | 37 + docs/migration-v2.md | 2 + evals/adaptation/user_signals.example.jsonl | 8 + evals/packaging_expectations.json | 5 +- evidence/world_class/README.md | 45 + evidence/world_class/intake.schema.json | 440 + .../templates/human-adjudication.intake.json | 64 + .../native-client-telemetry.intake.json | 53 + .../native-permission-enforcement.intake.json | 61 + .../templates/provider-holdout.intake.json | 50 + .../reports/context_budget.json | 43 +- .../reports/context_budget.json | 43 +- .../reports/description_optimization.json | 1367 +- .../reports/description_optimization.md | 24 +- .../optimization/semantic_config.json | 18 +- manifest.json | 3 +- references/autonomous-adaptation.md | 65 + references/distribution-registry-method.md | 9 +- references/operating-modes.md | 28 +- references/output-eval-method.md | 17 +- references/platform-capability-matrix.md | 2 + references/review-studio-method.md | 6 +- references/review-waiver-method.md | 19 + references/skill-atlas-method.md | 9 +- references/skillops-decision-policy.md | 53 + references/telemetry-drift-method.md | 124 +- references/trust-security-method.md | 1 + references/user-memory-policy.md | 35 + registry/index.json | 5 +- registry/packages/yao-meta-skill.json | 11 +- reports/adaptation_approval_ledger.json | 63 + reports/adaptation_proposals.json | 248 + reports/adaptation_proposals.md | 119 + reports/adaptation_regression_report.json | 87 + reports/adaptation_regression_report.md | 11 + reports/adoption_drift_report.json | 34 +- reports/adoption_drift_report.md | 13 +- reports/architecture_maintainability.json | 181 + reports/architecture_maintainability.md | 66 + reports/artifact-design-profile.json | 85 +- reports/artifact-design-profile.md | 52 +- reports/baseline-compare.json | 26 +- reports/baseline-compare.md | 10 +- reports/benchmark_methodology.md | 1 + reports/benchmark_reproducibility.json | 488 + reports/benchmark_reproducibility.md | 181 + reports/candidate_registry.json | 171 +- reports/candidate_registry.md | 18 +- reports/compiled_targets.json | 2071 +- reports/compiled_targets.md | 14 +- reports/conformance_matrix.json | 26 +- reports/context_budget.json | 71 +- reports/context_budget.md | 10 +- reports/context_budget_summary.json | 58 +- reports/description_optimization.json | 1026 +- reports/description_optimization.md | 14 +- reports/description_optimization_suite.json | 114 +- reports/description_optimization_suite.md | 4 +- reports/evidence_consistency.json | 3611 +++ reports/evidence_consistency.md | 57 + reports/install_simulation.json | 179 +- reports/install_simulation.md | 39 +- .../governed-incident-command/bundle.json | 43 +- .../governed-incident-command/bundle.md | 13 +- .../yao-meta-skill/bundle.json | 14 +- .../yao-meta-skill/bundle.md | 6 +- .../yao-meta-skill/review.md | 4 +- reports/iteration_ledger.md | 6 +- reports/output_execution_runs.json | 323 +- reports/output_execution_runs.md | 34 +- reports/output_review_adjudication.json | 227 +- reports/output_review_adjudication.md | 188 +- reports/output_review_decisions.json | 67 + reports/output_review_kit.html | 220 + reports/output_review_kit.json | 271 + reports/output_review_kit.md | 141 + reports/package_verification.json | 162 +- reports/package_verification.md | 36 +- reports/portability_score.json | 6 +- reports/portability_score.md | 2 +- reports/promotion_decisions.json | 171 +- reports/promotion_decisions.md | 2 +- reports/prompt-quality-profile.json | 38 +- reports/prompt-quality-profile.md | 28 +- reports/python_compatibility.json | 1423 ++ reports/python_compatibility.md | 20 + reports/reference-synthesis.json | 2 +- reports/reference-synthesis.md | 2 +- reports/registry_audit.json | 16 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 652 +- reports/review-studio.json | 18697 +++++++++++++++- reports/review-viewer.html | 386 +- reports/review-viewer.json | 3577 ++- reports/review_annotations.json | 2 +- reports/review_waivers.json | 86 +- reports/review_waivers.md | 46 + reports/route_scorecard.json | 46 +- reports/route_scorecard.md | 4 +- reports/runtime_permission_probes.json | 148 +- reports/runtime_permission_probes.md | 31 +- reports/security_trust_report.json | 1820 +- reports/security_trust_report.md | 111 +- reports/skill-interpretation.html | 1005 + reports/skill-interpretation.json | 2295 ++ reports/skill-os-2-review.md | 126 +- reports/skill-overview.html | 273 +- reports/skill-overview.json | 1225 +- reports/skill_atlas.html | 10 +- reports/skill_atlas.json | 254 +- reports/skill_os2_audit.json | 438 + reports/skill_os2_audit.md | 101 + reports/skill_os2_coverage.json | 811 + reports/skill_os2_coverage.md | 246 + reports/skillops/daily/2026-06-16.json | 513 + reports/skillops/daily/2026-06-16.md | 174 + reports/skillops/daily/2026-06-17.json | 513 + reports/skillops/daily/2026-06-17.md | 174 + reports/skillops/weekly/2026-W25.json | 320 + reports/skillops/weekly/2026-W25.md | 110 + reports/system-model.json | 2 +- reports/telemetry_hook_recipes.json | 338 + reports/telemetry_hook_recipes.md | 35 + reports/upgrade_check.json | 20 +- reports/upgrade_check.md | 4 +- reports/user_patterns.json | 136 + reports/user_patterns.md | 72 + reports/world_class_claim_guard.json | 783 + reports/world_class_claim_guard.md | 33 + reports/world_class_evidence_intake.json | 398 + reports/world_class_evidence_intake.md | 265 + reports/world_class_evidence_ledger.json | 602 + reports/world_class_evidence_ledger.md | 226 + reports/world_class_evidence_plan.json | 372 + reports/world_class_evidence_plan.md | 190 + reports/world_class_evidence_preflight.html | 992 + reports/world_class_evidence_preflight.json | 2038 ++ reports/world_class_evidence_preflight.md | 201 + reports/world_class_operator_runbook.html | 168 + reports/world_class_operator_runbook.json | 1752 ++ reports/world_class_operator_runbook.md | 408 + reports/world_class_submission_review.json | 442 + reports/world_class_submission_review.md | 178 + schemas/adaptation-proposal.schema.json | 226 + schemas/skill-interpretation.schema.json | 115 + schemas/skillops-opportunity.schema.json | 112 + scripts/adaptation_patch_safety.py | 76 + scripts/adaptation_report_contracts.py | 111 + scripts/adjudicate_output_review.py | 239 +- scripts/apply_adaptation.py | 543 + scripts/benchmark_release_lock.py | 98 + scripts/build_skill_atlas.py | 222 +- scripts/build_skill_atlas_layout.py | 103 + scripts/build_skill_atlas_opportunities.py | 82 + scripts/ci_test.py | 27 + scripts/compile_skill.py | 289 +- scripts/compile_skill_targets.py | 282 + scripts/context_sizer.py | 25 +- scripts/cross_packager.py | 192 +- scripts/cross_packager_contracts.py | 107 + scripts/description_optimizer_reporting.py | 141 + scripts/emit_telemetry_event.py | 128 + .../evidence_consistency_artifact_roles.py | 201 + scripts/evidence_consistency_core.py | 328 + scripts/evidence_consistency_phase_queue.py | 173 + scripts/evidence_consistency_release.py | 132 + .../evidence_consistency_skill_os2_review.py | 83 + scripts/evidence_consistency_world_class.py | 222 + scripts/export_skill_ir.py | 6 +- scripts/html_rendering.py | 14 + scripts/import_output_review_decisions.py | 329 + scripts/import_telemetry_events.py | 139 + scripts/init_skill.py | 44 +- scripts/optimize_description.py | 140 +- scripts/output_review_privacy.py | 81 + scripts/prepare_output_review_kit.py | 485 + scripts/prepare_world_class_submission_kit.py | 560 + scripts/probe_runtime_permissions.py | 179 +- scripts/propose_adaptation.py | 305 + scripts/provider_output_eval_runner.py | 345 + scripts/python_compat_check.py | 351 + scripts/reference_synthesis_markdown.py | 99 + scripts/registry_audit.py | 13 +- scripts/render_adoption_drift_report.py | 61 +- .../render_architecture_maintainability.py | 280 + scripts/render_benchmark_reproducibility.py | 685 + scripts/render_context_reports.py | 15 +- scripts/render_daily_skillops_report.py | 484 + scripts/render_evidence_consistency.py | 676 + scripts/render_portability_report.py | 25 +- scripts/render_reference_synthesis.py | 96 +- scripts/render_review_annotations.py | 3 + scripts/render_review_studio.py | 1294 +- scripts/render_review_viewer.py | 492 +- scripts/render_review_waivers.py | 144 +- scripts/render_skill_interpretation.py | 108 + scripts/render_skill_os2_audit.py | 433 + scripts/render_skill_os2_coverage.py | 555 + scripts/render_skill_overview.py | 332 +- scripts/render_social_preview.py | 14 +- scripts/render_telemetry_hook_recipes.py | 258 + scripts/render_weekly_curator_report.py | 433 + scripts/render_world_class_claim_guard.py | 269 + scripts/render_world_class_evidence_intake.py | 354 + scripts/render_world_class_evidence_ledger.py | 389 + scripts/render_world_class_evidence_plan.py | 327 + .../render_world_class_operator_runbook.py | 651 + scripts/render_world_class_preflight.py | 471 + .../render_world_class_submission_review.py | 305 + scripts/resource_boundary_check.py | 190 + scripts/review_studio_action_evidence.py | 388 + scripts/review_studio_actions.py | 374 + scripts/review_studio_data.py | 387 + scripts/review_studio_formatting.py | 71 + scripts/review_studio_gate_contract.py | 82 + scripts/review_studio_gate_helpers.py | 140 + scripts/review_studio_gates.py | 543 + scripts/review_studio_layout.py | 232 +- scripts/review_studio_output_review.py | 84 + scripts/review_studio_panels.py | 88 + scripts/review_studio_skillops.py | 48 + scripts/review_studio_waivers.py | 47 + scripts/review_studio_world_class.py | 143 + scripts/review_viewer_data.py | 316 + scripts/run_conformance_suite.py | 13 +- scripts/run_output_execution.py | 6 +- scripts/simulate_install.py | 130 +- scripts/skill_ir_paths.py | 76 + scripts/skill_os2_coverage_markdown.py | 101 + scripts/skill_report_charts.py | 78 +- scripts/skill_report_i18n.py | 386 + scripts/skill_report_layout.py | 626 +- scripts/skill_report_model.py | 337 +- scripts/skill_report_sections.py | 114 + scripts/skill_report_sources.py | 154 + scripts/skill_report_world_class.py | 128 + scripts/skillops_opportunity.py | 214 + scripts/summarize_user_signals.py | 321 + scripts/sync_local_install.py | 56 +- scripts/telemetry_native_host.py | 229 + scripts/trust_check.py | 142 +- scripts/trust_check_scripts.py | 146 + scripts/verify_package.py | 30 +- scripts/world_class_evidence_contract.py | 553 + scripts/world_class_human_evidence.py | 317 + .../world_class_native_permission_evidence.py | 157 + .../world_class_native_telemetry_evidence.py | 144 + ...rld_class_operator_runbook_coordination.py | 205 + scripts/world_class_phase_queue.py | 76 + scripts/world_class_preflight_layout.py | 350 + scripts/world_class_preflight_markdown.py | 175 + scripts/world_class_provider_evidence.py | 145 + scripts/world_class_repair_checklist.py | 286 + scripts/world_class_source_checks.py | 99 + .../world_class_submission_kit_rendering.py | 576 + scripts/world_class_submission_matrix.py | 161 + scripts/yao.py | 1450 +- scripts/yao_cli_adaptation_commands.py | 121 + scripts/yao_cli_config.py | 35 + scripts/yao_cli_create_commands.py | 338 + scripts/yao_cli_distribution_commands.py | 164 + scripts/yao_cli_output_commands.py | 134 + scripts/yao_cli_parser.py | 506 + scripts/yao_cli_parser_evidence.py | 135 + scripts/yao_cli_parser_operations.py | 268 + scripts/yao_cli_report_commands.py | 339 + scripts/yao_cli_runtime.py | 75 + scripts/yao_cli_telemetry.py | 95 + security/network_policy.json | 16 + security/permission_policy.json | 11 +- security/permission_policy.md | 6 +- skill-ir/examples/yao-meta-skill.json | 123 +- skill_atlas/catalog.json | 248 +- skill_atlas/drift_signals.json | 1 + skill_atlas/route_overlap_matrix.csv | 14 +- tests/review_studio_helpers.py | 194 + tests/review_studio_world_class_assertions.py | 148 + tests/snapshots/vscode_adapter.json | 26 + tests/verify_adaptation_safety.py | 460 + tests/verify_adapter_snapshots.py | 8 +- tests/verify_adoption_drift.py | 10 +- tests/verify_architecture_maintainability.py | 121 + tests/verify_benchmark_reproducibility.py | 314 + tests/verify_compile_skill.py | 8 +- tests/verify_conformance_suite.py | 30 + tests/verify_daily_skillops.py | 249 + tests/verify_evidence_consistency.py | 719 + tests/verify_html_rendering.py | 23 + tests/verify_install_simulation.py | 70 +- tests/verify_local_install_sync.py | 89 +- tests/verify_output_execution_runs.py | 351 + tests/verify_output_review_adjudication.py | 414 + tests/verify_output_review_kit.py | 223 + tests/verify_package_verification.py | 32 +- tests/verify_portability_report.py | 47 +- tests/verify_python_compat_check.py | 96 + tests/verify_quality_checks.py | 150 + tests/verify_registry_audit.py | 1 + tests/verify_review_studio.py | 637 +- tests/verify_review_viewer.py | 22 + tests/verify_review_waivers.py | 122 + tests/verify_runtime_permission_probes.py | 40 +- tests/verify_skill_atlas.py | 71 +- tests/verify_skill_interpretation.py | 86 + tests/verify_skill_ir.py | 12 + tests/verify_skill_ir_paths.py | 66 + tests/verify_skill_os2_audit.py | 72 + tests/verify_skill_os2_coverage.py | 189 + tests/verify_skill_overview.py | 165 +- tests/verify_skill_report_charts.py | 4 + tests/verify_skillops_opportunity.py | 96 + tests/verify_telemetry_emit.py | 172 + tests/verify_telemetry_hooks.py | 153 + tests/verify_telemetry_import.py | 184 + tests/verify_telemetry_native_host.py | 218 + tests/verify_trust_check.py | 76 +- tests/verify_upgrade_check.py | 1 + tests/verify_weekly_curator.py | 201 + tests/verify_world_class_claim_guard.py | 161 + tests/verify_world_class_evidence_intake.py | 703 + tests/verify_world_class_evidence_ledger.py | 587 + tests/verify_world_class_evidence_plan.py | 102 + tests/verify_world_class_operator_runbook.py | 313 + tests/verify_world_class_preflight.py | 355 + tests/verify_world_class_submission_kit.py | 351 + tests/verify_world_class_submission_review.py | 215 + tests/verify_yao_cli.py | 323 +- tests/verify_yao_cli_world_class.py | 183 + tests/world_class_human_fixtures.py | 364 + .../world_class_native_telemetry_fixtures.py | 221 + tests/yao_cli_helpers.py | 135 + tests/yao_cli_report_refresh.py | 16 + 383 files changed, 105679 insertions(+), 10107 deletions(-) create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/README.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/SKILL.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/agents/interface.yaml create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/output_cases.jsonl create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/trigger_cases.jsonl create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/manifest.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/brief-structure.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/review-checklist.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-context.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.html create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.html create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.md create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-ir.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.html create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.json create mode 100644 .previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.md create mode 100644 assets/review-studio.css create mode 100644 assets/review-viewer.css create mode 100644 assets/skill-overview.css create mode 100644 assets/skill-overview.js create mode 100644 evals/adaptation/user_signals.example.jsonl create mode 100644 evidence/world_class/README.md create mode 100644 evidence/world_class/intake.schema.json create mode 100644 evidence/world_class/templates/human-adjudication.intake.json create mode 100644 evidence/world_class/templates/native-client-telemetry.intake.json create mode 100644 evidence/world_class/templates/native-permission-enforcement.intake.json create mode 100644 evidence/world_class/templates/provider-holdout.intake.json create mode 100644 references/autonomous-adaptation.md create mode 100644 references/skillops-decision-policy.md create mode 100644 references/user-memory-policy.md create mode 100644 reports/adaptation_approval_ledger.json create mode 100644 reports/adaptation_proposals.json create mode 100644 reports/adaptation_proposals.md create mode 100644 reports/adaptation_regression_report.json create mode 100644 reports/adaptation_regression_report.md create mode 100644 reports/architecture_maintainability.json create mode 100644 reports/architecture_maintainability.md create mode 100644 reports/benchmark_reproducibility.json create mode 100644 reports/benchmark_reproducibility.md create mode 100644 reports/evidence_consistency.json create mode 100644 reports/evidence_consistency.md create mode 100644 reports/output_review_decisions.json create mode 100644 reports/output_review_kit.html create mode 100644 reports/output_review_kit.json create mode 100644 reports/output_review_kit.md create mode 100644 reports/python_compatibility.json create mode 100644 reports/python_compatibility.md create mode 100644 reports/skill-interpretation.html create mode 100644 reports/skill-interpretation.json create mode 100644 reports/skill_os2_audit.json create mode 100644 reports/skill_os2_audit.md create mode 100644 reports/skill_os2_coverage.json create mode 100644 reports/skill_os2_coverage.md create mode 100644 reports/skillops/daily/2026-06-16.json create mode 100644 reports/skillops/daily/2026-06-16.md create mode 100644 reports/skillops/daily/2026-06-17.json create mode 100644 reports/skillops/daily/2026-06-17.md create mode 100644 reports/skillops/weekly/2026-W25.json create mode 100644 reports/skillops/weekly/2026-W25.md create mode 100644 reports/telemetry_hook_recipes.json create mode 100644 reports/telemetry_hook_recipes.md create mode 100644 reports/user_patterns.json create mode 100644 reports/user_patterns.md create mode 100644 reports/world_class_claim_guard.json create mode 100644 reports/world_class_claim_guard.md create mode 100644 reports/world_class_evidence_intake.json create mode 100644 reports/world_class_evidence_intake.md create mode 100644 reports/world_class_evidence_ledger.json create mode 100644 reports/world_class_evidence_ledger.md create mode 100644 reports/world_class_evidence_plan.json create mode 100644 reports/world_class_evidence_plan.md create mode 100644 reports/world_class_evidence_preflight.html create mode 100644 reports/world_class_evidence_preflight.json create mode 100644 reports/world_class_evidence_preflight.md create mode 100644 reports/world_class_operator_runbook.html create mode 100644 reports/world_class_operator_runbook.json create mode 100644 reports/world_class_operator_runbook.md create mode 100644 reports/world_class_submission_review.json create mode 100644 reports/world_class_submission_review.md create mode 100644 schemas/adaptation-proposal.schema.json create mode 100644 schemas/skill-interpretation.schema.json create mode 100644 schemas/skillops-opportunity.schema.json create mode 100644 scripts/adaptation_patch_safety.py create mode 100644 scripts/adaptation_report_contracts.py create mode 100644 scripts/apply_adaptation.py create mode 100644 scripts/benchmark_release_lock.py create mode 100644 scripts/build_skill_atlas_layout.py create mode 100644 scripts/build_skill_atlas_opportunities.py create mode 100644 scripts/compile_skill_targets.py create mode 100644 scripts/cross_packager_contracts.py create mode 100644 scripts/description_optimizer_reporting.py create mode 100644 scripts/emit_telemetry_event.py create mode 100644 scripts/evidence_consistency_artifact_roles.py create mode 100644 scripts/evidence_consistency_core.py create mode 100644 scripts/evidence_consistency_phase_queue.py create mode 100644 scripts/evidence_consistency_release.py create mode 100644 scripts/evidence_consistency_skill_os2_review.py create mode 100644 scripts/evidence_consistency_world_class.py create mode 100644 scripts/html_rendering.py create mode 100644 scripts/import_output_review_decisions.py create mode 100644 scripts/import_telemetry_events.py create mode 100644 scripts/output_review_privacy.py create mode 100644 scripts/prepare_output_review_kit.py create mode 100644 scripts/prepare_world_class_submission_kit.py create mode 100644 scripts/propose_adaptation.py create mode 100644 scripts/provider_output_eval_runner.py create mode 100644 scripts/python_compat_check.py create mode 100644 scripts/reference_synthesis_markdown.py create mode 100644 scripts/render_architecture_maintainability.py create mode 100644 scripts/render_benchmark_reproducibility.py create mode 100644 scripts/render_daily_skillops_report.py create mode 100644 scripts/render_evidence_consistency.py create mode 100644 scripts/render_skill_interpretation.py create mode 100644 scripts/render_skill_os2_audit.py create mode 100644 scripts/render_skill_os2_coverage.py create mode 100644 scripts/render_telemetry_hook_recipes.py create mode 100644 scripts/render_weekly_curator_report.py create mode 100644 scripts/render_world_class_claim_guard.py create mode 100644 scripts/render_world_class_evidence_intake.py create mode 100644 scripts/render_world_class_evidence_ledger.py create mode 100644 scripts/render_world_class_evidence_plan.py create mode 100644 scripts/render_world_class_operator_runbook.py create mode 100644 scripts/render_world_class_preflight.py create mode 100644 scripts/render_world_class_submission_review.py create mode 100644 scripts/review_studio_action_evidence.py create mode 100644 scripts/review_studio_actions.py create mode 100644 scripts/review_studio_data.py create mode 100644 scripts/review_studio_gate_contract.py create mode 100644 scripts/review_studio_gate_helpers.py create mode 100644 scripts/review_studio_gates.py create mode 100644 scripts/review_studio_output_review.py create mode 100644 scripts/review_studio_panels.py create mode 100644 scripts/review_studio_skillops.py create mode 100644 scripts/review_studio_waivers.py create mode 100644 scripts/review_studio_world_class.py create mode 100644 scripts/review_viewer_data.py create mode 100644 scripts/skill_ir_paths.py create mode 100644 scripts/skill_os2_coverage_markdown.py create mode 100644 scripts/skill_report_i18n.py create mode 100644 scripts/skill_report_sections.py create mode 100644 scripts/skill_report_sources.py create mode 100644 scripts/skill_report_world_class.py create mode 100644 scripts/skillops_opportunity.py create mode 100644 scripts/summarize_user_signals.py create mode 100644 scripts/telemetry_native_host.py create mode 100644 scripts/trust_check_scripts.py create mode 100644 scripts/world_class_evidence_contract.py create mode 100644 scripts/world_class_human_evidence.py create mode 100644 scripts/world_class_native_permission_evidence.py create mode 100644 scripts/world_class_native_telemetry_evidence.py create mode 100644 scripts/world_class_operator_runbook_coordination.py create mode 100644 scripts/world_class_phase_queue.py create mode 100644 scripts/world_class_preflight_layout.py create mode 100644 scripts/world_class_preflight_markdown.py create mode 100644 scripts/world_class_provider_evidence.py create mode 100644 scripts/world_class_repair_checklist.py create mode 100644 scripts/world_class_source_checks.py create mode 100644 scripts/world_class_submission_kit_rendering.py create mode 100644 scripts/world_class_submission_matrix.py create mode 100644 scripts/yao_cli_adaptation_commands.py create mode 100644 scripts/yao_cli_create_commands.py create mode 100644 scripts/yao_cli_distribution_commands.py create mode 100644 scripts/yao_cli_output_commands.py create mode 100644 scripts/yao_cli_parser.py create mode 100644 scripts/yao_cli_parser_evidence.py create mode 100644 scripts/yao_cli_parser_operations.py create mode 100644 scripts/yao_cli_report_commands.py create mode 100644 scripts/yao_cli_runtime.py create mode 100644 scripts/yao_cli_telemetry.py create mode 100644 skill_atlas/drift_signals.json create mode 100644 tests/review_studio_helpers.py create mode 100644 tests/review_studio_world_class_assertions.py create mode 100644 tests/snapshots/vscode_adapter.json create mode 100644 tests/verify_adaptation_safety.py create mode 100644 tests/verify_architecture_maintainability.py create mode 100644 tests/verify_benchmark_reproducibility.py create mode 100644 tests/verify_daily_skillops.py create mode 100644 tests/verify_evidence_consistency.py create mode 100644 tests/verify_html_rendering.py create mode 100644 tests/verify_output_review_kit.py create mode 100644 tests/verify_python_compat_check.py create mode 100644 tests/verify_skill_interpretation.py create mode 100644 tests/verify_skill_ir_paths.py create mode 100644 tests/verify_skill_os2_audit.py create mode 100644 tests/verify_skill_os2_coverage.py create mode 100644 tests/verify_skillops_opportunity.py create mode 100644 tests/verify_telemetry_emit.py create mode 100644 tests/verify_telemetry_hooks.py create mode 100644 tests/verify_telemetry_import.py create mode 100644 tests/verify_telemetry_native_host.py create mode 100644 tests/verify_weekly_curator.py create mode 100644 tests/verify_world_class_claim_guard.py create mode 100644 tests/verify_world_class_evidence_intake.py create mode 100644 tests/verify_world_class_evidence_ledger.py create mode 100644 tests/verify_world_class_evidence_plan.py create mode 100644 tests/verify_world_class_operator_runbook.py create mode 100644 tests/verify_world_class_preflight.py create mode 100644 tests/verify_world_class_submission_kit.py create mode 100644 tests/verify_world_class_submission_review.py create mode 100644 tests/verify_yao_cli_world_class.py create mode 100644 tests/world_class_human_fixtures.py create mode 100644 tests/world_class_native_telemetry_fixtures.py create mode 100644 tests/yao_cli_helpers.py create mode 100644 tests/yao_cli_report_refresh.py diff --git a/.gitignore b/.gitignore index b4744fbb..bc323ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ dist/ *.zip __pycache__/ tests/tmp/ +tests/tmp_*/ tests/tmp_snapshot/ tests/tmp_cli/ tests/tmp_skill_overview/ @@ -23,9 +24,12 @@ tests/tmp_baseline_compare.* reports/release_snapshots/ reports/telemetry_events.jsonl .yao/ +evidence/world_class/submission-kit/ +evidence/world_class/submissions/ # Local business-skill experiments belong outside this meta-skill repo unless promoted intentionally. geo-ranking-article-generator/ # Private or one-off pattern analysis reports should not be committed as factory evidence. reports/*pattern-analysis*.md +reports/*research-plan*.md diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/README.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/README.md new file mode 100644 index 00000000..64551ea4 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/README.md @@ -0,0 +1,72 @@ +# Geo Content Brief Skill + +## What It Does + +`geo-content-brief-skill` is a reusable skill package for this job: + +> 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。 + +## 中文使用方式 + +1. 输入访谈纪要、关键词、渠道限制、竞品摘要、品牌禁区或已有内容方向。 +2. 先判断请求是否属于“生成内容简报”，而不是完整长文、广告口号或品牌策略。 +3. 按 `references/brief-structure.md` 输出一页中文简报。 +4. 用 `references/review-checklist.md` 检查事实、结构、证据、风险和下一步动作。 +5. 用 `evals/trigger_cases.jsonl` 和 `evals/output_cases.jsonl` 做轻量回归。 + +## How To Use + +1. Load the skill through `SKILL.md`. +2. Start with `reports/intent-dialogue.md` to tighten the real job, outputs, exclusions, and the standards you care about. +3. Open `reports/reference-scan.md` to capture external benchmarks and any user-supplied references worth learning from. +4. Review `reports/intent-confidence.md` to see whether the real job, inputs, outputs, and exclusions are clear enough yet. +5. Open `reports/reference-synthesis.md` to see the GitHub benchmarks plus curated official, research, and principle tracks in one place. +6. Follow the workflow steps in `SKILL.md`. +7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right. +8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view. +9. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up. +10. Open `reports/review-viewer.html` for a compact visual review of the package. +11. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. +12. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. +13. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. +14. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. +15. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts. +16. Review `reports/iteration-directions.md` for the three most valuable next moves. +17. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. +18. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals. +19. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates. + +## Honest Boundaries + +- This package starts from the current intent frame and should not pretend to cover unclear adjacent jobs. +- The first version should ask for clarification when the real input, output, or exclusion boundary is still fuzzy. +- New structure should be added only when it earns its keep through evidence, validation, or reviewer need. +- It should not fabricate search volume, competitor rankings, customer quotes, or third-party evidence. + +## Package Map + +- `SKILL.md`: trigger and workflow entrypoint +- `agents/interface.yaml`: portable interface metadata +- `manifest.json`: lifecycle and packaging metadata +- `references/brief-structure.md`: 中文 GEO 内容简报结构、字段说明和质量标准 +- `references/review-checklist.md`: 交付前编辑复核清单 +- `evals/trigger_cases.jsonl`: 应触发和不应触发样例 +- `evals/output_cases.jsonl`: 输出结构验收样例 +- `reports/intent-dialogue.md`: front-loaded discovery questions for better boundary design and clearer human alignment +- `reports/intent-confidence.md`: current clarity score, open gaps, and the next follow-up questions worth asking +- `reports/github-benchmark-scan.md`: top public benchmark repositories, extracted patterns, and borrow or avoid notes +- `reports/reference-scan.md`: benchmark notes from public references, user references, and local constraints +- `reports/reference-synthesis.md`: a combined view of GitHub benchmarks plus curated world-class pattern tracks +- `reports/output-risk-profile.md`: predicted output failure modes and self-repair constraints for this skill +- `reports/artifact-design-profile.md`: artifact-specific design direction, layout patterns, visual quality gates, and anti-patterns +- `reports/prompt-quality-profile.md`: prompt-facing need model, RTF mapping, complexity, and quality matrix +- `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points +- `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance +- `reports/compiled_targets.md`: target compiler report showing generated contracts, adapter modes, preserved semantics, warnings, and unsupported features +- `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap +- `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness +- `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps +- `reports/iteration-directions.md`: the top three next iteration directions +- `reports/adoption_drift_report.md`: local-first metadata-only telemetry summary for adoption, missed triggers, bad outputs, script errors, and review drift +- `reports/review_waivers.md`: human reviewer risk approval ledger for warning acceptance and expiry +- `reports/review_annotations.md`: source-line reviewer comments linked to Review Studio gates diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/SKILL.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/SKILL.md new file mode 100644 index 00000000..475ff937 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/SKILL.md @@ -0,0 +1,57 @@ +--- +name: geo-content-brief-skill +description: "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" +--- + +# Geo Content Brief Skill + +## When To Use This Skill + +- 用户提供访谈纪要、关键词、竞品片段、产品卖点或渠道限制，希望沉淀为中文内容简报。 +- 团队需要把零散输入转换成可执行的 GEO 内容选题、角度、结构、证据和交付要求。 +- 需要先明确触发边界、输入缺口和输出验收标准，再进入内容生产。 + +## Workflow + +1. 读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。 +2. 对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。 +3. 如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。 +4. 用 `evals/trigger_cases.jsonl` 检查当前请求是否属于 GEO 内容简报场景。 +5. 用 `evals/output_cases.jsonl` 检查输出是否包含输入摘要、内容结构、证据要求、禁区和下一步动作。 +6. 交付前回看渠道限制和品牌禁区，删除空泛营销话术、无来源断言和不可执行建议。 + +## Inputs + +- 访谈纪要、销售反馈、客户问题或专家观点。 +- 关键词、搜索意图、竞品页面摘要或排名线索。 +- 目标渠道、目标读者、品牌口径、禁用词和交付格式。 + +## Outputs + +- 一页中文内容简报，包含目标、受众、核心洞察、推荐角度、文章结构、关键词布置、证据需求和风险提示。 +- 缺口清单，说明继续生产前必须补齐的信息。 +- 下一步动作，面向写作者、编辑或 SEO/GEO 负责人。 + +## Output Quality Guardrails + +- Before final output, apply the likely failure modes in `reports/output-risk-profile.md` when that report is present. +- Before rendering reports, tutorials, review pages, dashboards, or visual artifacts, apply the artifact direction and visual quality gates in `reports/artifact-design-profile.md` when that report is present. +- When prompt behavior, role design, dialogue quality, or output contracts matter, apply `reports/prompt-quality-profile.md` when that report is present. +- Before adding more structure, apply the boundary, feedback-loop, drift, and leverage-point checks in `reports/system-model.md` when that report is present. +- Repair generic headings, cluttered notes, fragile visual assumptions, weak tables, and missing verification cues before handing work back. +- Map role, task, and format into skill behavior rather than copying a large prompt template into `SKILL.md`. +- Let the artifact's content choose the visual system; do not copy a fixed palette or report style from another skill without a clear reason. +- If output-specific evidence is missing, state the gap instead of inventing screenshots, citations, data, or examples. + +## Honest Boundaries + +- 不直接代写完整长文，不替代品牌策略判断，不伪造竞品数据、搜索量、排名或引用来源。 +- 当用户只想要标题灵感、广告文案、舆情分析或完整文章撰写时，不应触发本 Skill。 +- 缺少目标读者、渠道或关键词时，先收紧输入，再输出简报。 + +## Reference Map + +- `references/brief-structure.md`: 中文 GEO 内容简报结构、字段说明和质量标准。 +- `references/review-checklist.md`: 交付前的编辑复核清单。 +- `evals/trigger_cases.jsonl`: 应触发和不应触发样例。 +- `evals/output_cases.jsonl`: 输出结构验收样例。 diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/agents/interface.yaml b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/agents/interface.yaml new file mode 100644 index 00000000..e74f5ef5 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/agents/interface.yaml @@ -0,0 +1,29 @@ +interface: + display_name: "Geo Content Brief Skill" + short_description: "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。" + default_prompt: "当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。" +compatibility: + canonical_format: "agent-skills" + adapter_targets: + - "openai" + - "claude" + - "generic" + - "vscode" + activation: + mode: "manual" + paths: + - "SKILL.md" + - "references/brief-structure.md" + - "evals/trigger_cases.jsonl" + execution: + context: "inline" + shell: "bash" + trust: + source_tier: "local" + remote_inline_execution: "forbid" + remote_metadata_policy: "allow-metadata-only" + degradation: + openai: "metadata-adapter" + claude: "neutral-source-plus-adapter" + generic: "neutral-source" + vscode: "agent-skills-source-with-vscode-notes" diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/output_cases.jsonl b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/output_cases.jsonl new file mode 100644 index 00000000..d0b9c730 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/output_cases.jsonl @@ -0,0 +1,2 @@ +{"id":"brief_minimum_contract","input":"访谈纪要、关键词、渠道、品牌禁区","must_include":["输入摘要","目标读者","搜索意图","内容角度","结构建议","证据需求","风险禁区","下一步动作"],"must_not_include":["虚构搜索量","无来源竞品排名","完整长文正文"]} +{"id":"missing_evidence_behavior","input":"只有关键词，没有访谈、渠道和品牌限制","must_include":["缺口清单","最多三个澄清问题","待补充证据"],"must_not_include":["假设用户品牌口径","编造客户原话"]} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/trigger_cases.jsonl b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/trigger_cases.jsonl new file mode 100644 index 00000000..4358b45b --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/evals/trigger_cases.jsonl @@ -0,0 +1,4 @@ +{"id":"should_geo_interview_notes","input":"这里有一段客户访谈和 8 个 GEO 关键词，帮我整理成一份中文内容简报，给写作者用。","expected":"trigger","reason":"用户明确要把访谈和关键词整理为内容简报。"} +{"id":"should_competitor_brief","input":"把这些竞品页面摘要、品牌禁区和知乎渠道限制整理成一页选题简报。","expected":"trigger","reason":"输入包含竞品、品牌边界和渠道约束，输出是一页简报。"} +{"id":"should_not_full_article","input":"根据这些关键词直接帮我写一篇 3000 字完整文章。","expected":"no_trigger","reason":"用户要完整长文写作，不是生成内容简报。"} +{"id":"should_not_ad_copy","input":"给这个活动想 20 条广告口号，越短越好。","expected":"no_trigger","reason":"广告口号生成不属于 GEO 内容简报。"} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/manifest.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/manifest.json new file mode 100644 index 00000000..ca9462df --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/manifest.json @@ -0,0 +1,24 @@ +{ + "name": "geo-content-brief-skill", + "version": "0.1.0", + "owner": "Yao Team", + "updated_at": "2026-06-15", + "status": "active", + "maturity_tier": "production", + "lifecycle_stage": "production-preview", + "context_budget_tier": "production", + "review_cadence": "per-release", + "skill_archetype": "production", + "target_platforms": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ], + "factory_components": [ + "references", + "scripts", + "reports" + ] +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/brief-structure.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/brief-structure.md new file mode 100644 index 00000000..e6cfd132 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/brief-structure.md @@ -0,0 +1,23 @@ +# GEO 内容简报结构 + +## 目标 + +把零散材料整理成写作者可以直接执行的一页内容简报。简报应帮助团队快速判断：写给谁、解决什么问题、采用什么角度、需要哪些证据、哪些话不能说。 + +## 推荐字段 + +- 项目背景：产品、业务目标、目标渠道和期望交付物。 +- 目标读者：读者身份、已知问题、决策阶段和内容理解门槛。 +- 搜索意图：核心关键词、近义词、长尾问题和需要避开的无关意图。 +- 内容角度：建议主张、差异化角度、可引用观点和与竞品的区分点。 +- 结构建议：标题方向、章节顺序、每节回答的问题和必要证据。 +- 证据需求：内部数据、案例、截图、第三方来源、专家观点或客户原话。 +- 风险禁区：禁止承诺、敏感表述、未证实数据、品牌不采用的语气。 +- 验收标准：完成后需要检查的结构、事实、语气、关键词和行动建议。 + +## 质量标准 + +- 每个推荐角度都应能追溯到输入材料或明确标记为待验证假设。 +- 缺少事实证据时写“待补充证据”，不要编造来源。 +- 输出要服务内容生产交接，避免泛泛的市场建议。 +- 中文表达要具体、可执行、便于编辑继续拆稿。 diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/review-checklist.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/review-checklist.md new file mode 100644 index 00000000..43d7bdba --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/references/review-checklist.md @@ -0,0 +1,11 @@ +# 内容简报复核清单 + +交付前按以下顺序检查： + +1. 输入摘要是否覆盖访谈、关键词、渠道限制和品牌禁区。 +2. 目标读者是否具体到角色、问题和决策阶段。 +3. 推荐角度是否能从输入材料中找到依据。 +4. 文章结构是否能直接分配给写作者执行。 +5. 证据需求是否列出已拥有证据和待补证据。 +6. 风险提示是否包含禁用表达、未证实数据和不应承诺的结果。 +7. 下一步动作是否明确谁继续补材料、谁写稿、谁审核。 diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.json new file mode 100644 index 00000000..f1d576ef --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.json @@ -0,0 +1,62 @@ +{ + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-15T09:03:27Z", + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "summary": { + "event_count": 0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "no-data", + "event_types": {}, + "failure_types": {}, + "source_types": {}, + "command_counts": {} + }, + "adoption_by_skill": [], + "next_iteration_candidates": [ + { + "signal": "no telemetry", + "recommendation": "Start with a small metadata-only sample before using telemetry for release decisions." + } + ], + "recent_events": [], + "failures": [], + "artifacts": { + "events_jsonl": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/telemetry_events.jsonl", + "json": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.json", + "markdown": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.md" + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.md new file mode 100644 index 00000000..f4e68f51 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.md @@ -0,0 +1,36 @@ +# Adoption And Drift Report + +Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, transcripts, and notes are not allowed in the event stream. + +## Summary + +- Events: `0` +- Adoption samples: `0` +- Activation events: `0` +- Adoption rate: `0` +- Missed trigger signals: `0` +- Bad output signals: `0` +- Script error signals: `0` +- Review overdue signals: `0` +- Risk band: `no-data` + +## Privacy Contract + +- Storage is local-first. +- Events are metadata-only. +- Raw user prompts, model outputs, transcripts, notes, and messages are blocked. +- Distributed packages should include this aggregate report, not raw `reports/telemetry_events.jsonl`. + +## Adoption By Skill + +| Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `none` | 0 | 0 | 0 | 0 | 0 | 0 | 0 | + +## Next Iteration Candidates + +- `no telemetry`: Start with a small metadata-only sample before using telemetry for release decisions. + +## Recent Metadata Events + +- No metadata events captured yet. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.json new file mode 100644 index 00000000..e9885482 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.json @@ -0,0 +1,115 @@ +{ + "skill_name": "geo-content-brief-skill", + "design_system": "metric editorial", + "primary_artifact": { + "key": "report", + "label": "Report or brief", + "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "matched_keywords": [ + "report", + "brief", + "简报" + ] + }, + "artifact_families": [ + { + "key": "report", + "label": "Report or brief", + "score": 3, + "matched_keywords": [ + "report", + "brief", + "简报" + ], + "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." + }, + { + "key": "review_viewer", + "label": "Review viewer", + "score": 2, + "matched_keywords": [ + "review", + "审查" + ], + "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." + }, + { + "key": "dashboard", + "label": "Dashboard or metrics page", + "score": 2, + "matched_keywords": [ + "dashboard", + "table" + ], + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." + }, + { + "key": "tutorial", + "label": "Tutorial or guide", + "score": 1, + "matched_keywords": [ + "tutorial" + ], + "direction": "Progressive instructional layout with domain-specific section names, short success checks, and examples close to the user's real input." + }, + { + "key": "visual_capture", + "label": "Screenshot or visual evidence", + "score": 1, + "matched_keywords": [ + "screenshot" + ], + "direction": "Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect." + } + ], + "layout_patterns": [ + "thesis", + "evidence blocks", + "decision table", + "risks", + "next actions", + "summary", + "variant comparison", + "evidence" + ], + "design_tokens": { + "type": [ + "Use a distinctive display face or serif for major claims when the artifact is editorial.", + "Use a restrained sans for dense body text and technical details.", + "Use mono only for metadata, paths, commands, labels, and evidence tags." + ], + "color": [ + "Choose colors from the artifact's domain, brand, or evidence mood.", + "Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.", + "Keep accent color limited to decisions, active states, risk, or section anchors." + ], + "spacing": [ + "Prefer clear grid rhythm over floating decorative cards.", + "Increase whitespace around decisions and shrink whitespace around supporting metadata.", + "Split dense content instead of shrinking type or adding scroll traps." + ], + "components": [ + "Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.", + "Avoid cards inside cards.", + "Keep reviewer-only detail visible but visually quieter than user-facing guidance." + ] + }, + "quality_gates": [ + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers.", + "Make differences visible instead of hiding them in prose.", + "Separate author-facing recommendations from reviewer-only evidence.", + "Surface conflicts clearly and keep routine benchmark synthesis quiet.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each." + ], + "anti_patterns": [ + "Do not copy Kami's fixed parchment background as a default.", + "Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.", + "Do not let Markdown tables become the default shape for every comparison or explanation.", + "Do not turn reviewer evidence into user-facing clutter.", + "Do not invent screenshots, citations, charts, or UI states." + ], + "reviewer_note": "Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered." +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.md new file mode 100644 index 00000000..6e09e083 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/artifact-design-profile.md @@ -0,0 +1,93 @@ +# Artifact Design Profile + +Skill: `geo-content-brief-skill` +Design system: `metric editorial` + +## Primary Artifact Direction + +**Report or brief** + +High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. + +## Matched Artifact Families + +### Report or brief +- Matched keywords: report, brief, 简报 +- Score: `3` +- Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. + +### Review viewer +- Matched keywords: review, 审查 +- Score: `2` +- Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix. + +### Dashboard or metrics page +- Matched keywords: dashboard, table +- Score: `2` +- Direction: Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation. + +### Tutorial or guide +- Matched keywords: tutorial +- Score: `1` +- Direction: Progressive instructional layout with domain-specific section names, short success checks, and examples close to the user's real input. + +### Screenshot or visual evidence +- Matched keywords: screenshot +- Score: `1` +- Direction: Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect. + +## Layout Patterns To Prefer + +- thesis +- evidence blocks +- decision table +- risks +- next actions +- summary +- variant comparison +- evidence + +## Design Tokens + +### Type +- Use a distinctive display face or serif for major claims when the artifact is editorial. +- Use a restrained sans for dense body text and technical details. +- Use mono only for metadata, paths, commands, labels, and evidence tags. + +### Color +- Choose colors from the artifact's domain, brand, or evidence mood. +- Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it. +- Keep accent color limited to decisions, active states, risk, or section anchors. + +### Spacing +- Prefer clear grid rhythm over floating decorative cards. +- Increase whitespace around decisions and shrink whitespace around supporting metadata. +- Split dense content instead of shrinking type or adding scroll traps. + +### Components +- Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence. +- Avoid cards inside cards. +- Keep reviewer-only detail visible but visually quieter than user-facing guidance. + +## Quality Gates + +- Keep the first screen useful without requiring the reader to parse every detail. +- Use tables only for comparisons; move explanations below the table. +- Keep source notes readable without flooding the body with markers. +- Make differences visible instead of hiding them in prose. +- Separate author-facing recommendations from reviewer-only evidence. +- Surface conflicts clearly and keep routine benchmark synthesis quiet. +- Avoid paragraph-heavy table cells. +- Keep charts tied to one analytical question each. + +## Anti-Patterns + +- Do not copy Kami's fixed parchment background as a default. +- Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them. +- Do not let Markdown tables become the default shape for every comparison or explanation. +- Do not turn reviewer evidence into user-facing clutter. +- Do not invent screenshots, citations, charts, or UI states. + +## Reviewer Note + +Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.json new file mode 100644 index 00000000..64a319d4 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.json @@ -0,0 +1,2229 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15", + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "schema_version": "1.0", + "target": "openai", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "openai", + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "degradation_strategy": "metadata-adapter", + "target_runtime": { + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "target_transform": { + "target": "openai", + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "metadata-adapter", + "permission_representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "permission_enforcement": "metadata-only" + }, + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "claude", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "claude", + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "degradation_strategy": "neutral-source-plus-adapter", + "target_runtime": { + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Claude-compatible neutral source folder with adapter notes" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "target_transform": { + "target": "claude", + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source-plus-adapter", + "permission_representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "permission_enforcement": "metadata-fallback" + }, + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "generic", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "generic", + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills compatible neutral package" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "target_transform": { + "target": "generic", + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "targets/generic/adapter.json::target_permission_contract", + "native_surface": "Agent Skills compatible neutral package", + "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "agent-skills-compatible", + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills standard source tree" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "target_transform": { + "target": "agent-skills-compatible", + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "adapter metadata", + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "vscode", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "vscode", + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "target_runtime": { + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "VS Code/Copilot Agent Skills project or user scope" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "target_transform": { + "target": "vscode", + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "permission_representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "permission_enforcement": "client-or-workspace-trust" + }, + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [], + "failures": [] + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "json": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.json", + "markdown": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.md" + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.md new file mode 100644 index 00000000..fbcb81f9 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.md @@ -0,0 +1,73 @@ +# Compiled Targets + +- OK: `True` +- Targets: `5` +- Pass: `5` +- Warn: `0` +- Block: `0` + +## Target Transforms + +| Target | Status | Native Surface | Adapter Mode | Permissions | Degradation | Generated Files | +| --- | --- | --- | --- | --- | --- | --- | +| `openai` | `pass` | OpenAI-style interface metadata plus neutral Agent Skills source | `metadata-adapter` | `none` | `metadata-adapter` | targets/openai/adapter.json, targets/openai/agents/openai.yaml | +| `claude` | `pass` | Claude-compatible neutral source folder with adapter notes | `neutral-source-plus-adapter` | `none` | `neutral-source-plus-adapter` | targets/claude/adapter.json, targets/claude/README.md | +| `generic` | `pass` | Agent Skills compatible neutral package | `agent-skills-compatible` | `none` | `neutral-source` | targets/generic/adapter.json | +| `agent-skills-compatible` | `pass` | Agent Skills standard source tree | `neutral-agent-skills-source` | `none` | `neutral-source` | SKILL.md, agents/interface.yaml | +| `vscode` | `pass` | VS Code/Copilot Agent Skills project or user scope | `vscode-agent-skills-adapter` | `none` | `agent-skills-source-with-vscode-notes` | targets/vscode/adapter.json, targets/vscode/README.md | + +## Native Behavior Contracts + +### openai + +- Native surface: OpenAI-style interface metadata plus neutral Agent Skills source +- Activation: Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata. +- Resources: Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file. +- Scripts: Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution. +- Permission enforcement: `metadata-only`; native enforcement `False` +- Review artifacts: targets/openai/agents/openai.yaml, targets/openai/adapter.json, reports/review-studio.html + +### claude + +- Native surface: Claude-compatible neutral source folder with adapter notes +- Activation: Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata. +- Resources: Preserve the source tree directly; write target notes in targets/claude/README.md. +- Scripts: Scripts remain local package resources and must be reviewed through trust and permission reports before use. +- Permission enforcement: `metadata-fallback`; native enforcement `False` +- Review artifacts: targets/claude/README.md, targets/claude/adapter.json, reports/review-studio.html + +### generic + +- Native surface: Agent Skills compatible neutral package +- Activation: Use SKILL.md name and description; consumers decide automatic or manual activation. +- Resources: Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources. +- Scripts: Expose script and permission metadata for downstream clients or installers to enforce. +- Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False` +- Review artifacts: targets/generic/adapter.json, reports/review-studio.html + +### agent-skills-compatible + +- Native surface: Agent Skills standard source tree +- Activation: Use SKILL.md frontmatter name and description for progressive disclosure. +- Resources: Keep optional directories as relative resources next to SKILL.md. +- Scripts: Scripts remain local optional resources and should advertise --help when executable. +- Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False` +- Review artifacts: SKILL.md, agents/interface.yaml, reports/review-studio.html + +### vscode + +- Native surface: VS Code/Copilot Agent Skills project or user scope +- Activation: Use folder name plus SKILL.md name/description; keep description under platform limits. +- Resources: Install as project or user scoped skill source, preserving relative references and scripts. +- Scripts: Scripts require workspace trust and operator/client approval outside this compiler. +- Permission enforcement: `client-or-workspace-trust`; native enforcement `False` +- Review artifacts: SKILL.md, agents/interface.yaml, reports/review-studio.html + + +## Failures + +- None + +## Warnings + +- None diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.json new file mode 100644 index 00000000..973a08ef --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.json @@ -0,0 +1,83 @@ +{ + "score": 10, + "band": "low", + "gate_passed": false, + "strengths": [], + "gaps": [ + { + "key": "job_specificity", + "label": "Recurring job is still generic", + "reason": "The current job statement sounds more like a packaging goal than a concrete repeated task.", + "severity": "high" + }, + { + "key": "real_inputs", + "label": "Real inputs are missing", + "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", + "severity": "high" + }, + { + "key": "primary_output", + "label": "Primary output is missing", + "reason": "The package does not yet know what it must hand back.", + "severity": "high" + }, + { + "key": "exclusions", + "label": "Near-neighbor exclusions are missing", + "reason": "The route may blur into nearby requests without an exclusion list.", + "severity": "high" + }, + { + "key": "constraints", + "label": "Constraints are missing", + "reason": "The package does not yet know which tradeoffs matter most.", + "severity": "high" + }, + { + "key": "standards", + "label": "Quality bar is implied, not explicit", + "reason": "The first evaluation target is still underspecified.", + "severity": "medium" + } + ], + "follow_up_questions": [ + { + "slot": "job", + "question": "If you say it plainly, what concrete repeated task should this skill own every time?", + "why": "A skill needs a real recurring job, not only a generic packaging goal.", + "list": false, + "label": "Recurring job is still generic", + "severity": "high" + }, + { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": true, + "label": "Real inputs are missing", + "severity": "high" + }, + { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": false, + "label": "Primary output is missing", + "severity": "high" + } + ], + "anchor_sentence": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "context": { + "job": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "real_inputs": [], + "primary_output": "", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.md new file mode 100644 index 00000000..f06d664b --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-confidence.md @@ -0,0 +1,32 @@ +# Intent Confidence + +- Confidence score: `10/100` +- Confidence band: `low` +- Gate passed: `False` +- Recommended action: Pause before deep authoring and close the highest-leverage gaps first. + +## Current Reading + +将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 + +## Strong Signals + +- No strong signals yet. + +## Gaps To Close + +- **Recurring job is still generic** (`high`): The current job statement sounds more like a packaging goal than a concrete repeated task. +- **Real inputs are missing** (`high`): Without real inputs, it is hard to choose assets, scripts, or examples. +- **Primary output is missing** (`high`): The package does not yet know what it must hand back. +- **Near-neighbor exclusions are missing** (`high`): The route may blur into nearby requests without an exclusion list. +- **Constraints are missing** (`high`): The package does not yet know which tradeoffs matter most. +- **Quality bar is implied, not explicit** (`medium`): The first evaluation target is still underspecified. + +## Follow-Up Questions + +- **If you say it plainly, what concrete repeated task should this skill own every time?** + - Why: A skill needs a real recurring job, not only a generic packaging goal. +- **What material will people actually hand to this skill in practice?** + - Why: Real input shape decides whether references, scripts, or examples are needed. +- **What finished hand-back should this skill return so the next person can keep moving?** + - Why: The output is the anchor for package design and review. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-context.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-context.json new file mode 100644 index 00000000..6a5c4447 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-context.json @@ -0,0 +1,11 @@ +{ + "job": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "real_inputs": [], + "primary_output": "", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.json new file mode 100644 index 00000000..ab53569a --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.json @@ -0,0 +1,79 @@ +{ + "skill_name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "focus": "trigger-and-output", + "opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.", + "reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.", + "conversation_path": [ + "Start with the user's own words, not package vocabulary.", + "Reflect the job, output, and non-goals back in one clean sentence.", + "Only then offer a tiny scaffold if it would help the user move faster." + ], + "opening_styles": [ + { + "label": "温柔陪伴型", + "best_when": "用户想法还散、还在试探，或者需要先被接住。", + "message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。" + }, + { + "label": "专业教练型", + "best_when": "用户目标比较明确，希望被高效带着走。", + "message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。" + }, + { + "label": "共创伙伴型", + "best_when": "用户已经有一些想法，希望一起打磨，不想被填表。", + "message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。" + } + ], + "optional_scaffold": [ + "The repeated job it should reliably handle", + "The real inputs people will hand to it", + "The useful output it should hand back", + "What it should clearly refuse" + ], + "questions": [ + { + "question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?", + "why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape." + }, + { + "question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?", + "why": "Input shape decides whether references, scripts, or templates are needed." + }, + { + "question": "What finished output should it hand back so the user can immediately keep moving?", + "why": "Outputs should drive the package structure before extra guidance is added." + }, + { + "question": "Which nearby requests should this skill politely refuse so the boundary stays clean?", + "why": "The exclusion list is the fastest route to better trigger quality." + }, + { + "question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?", + "why": "Constraints decide how much structure, packaging, and review the skill actually needs." + }, + { + "question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?", + "why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material." + }, + { + "question": "What repeated manual step should become a deterministic asset first?", + "why": "This usually reveals whether a script or reference should be created next." + } + ], + "output": { + "capability_sentence": "Geo Content Brief Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.", + "required_capture": [ + "recurring job", + "real inputs", + "required outputs", + "exclusions", + "constraints", + "reference preferences", + "first evaluation target" + ], + "recommended_first_gate": "trigger and boundary" + } +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.md new file mode 100644 index 00000000..98d3efdc --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/intent-dialogue.md @@ -0,0 +1,75 @@ +# Intent Dialogue + +Skill: `geo-content-brief-skill` + +## Opening Frame + +Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that. + +## Opening Tone Options + +### 温柔陪伴型 + +- Best when: 用户想法还散、还在试探，或者需要先被接住。 +- Example: 我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。 + +### 专业教练型 + +- Best when: 用户目标比较明确，希望被高效带着走。 +- Example: 我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。 + +### 共创伙伴型 + +- Best when: 用户已经有一些想法，希望一起打磨，不想被填表。 +- Example: 我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。 + +## Conversation Path + +1. Start with the user's own words, not package vocabulary. +2. Reflect the job, output, and non-goals back in one clean sentence. +3. Only then offer a tiny scaffold if it would help the user move faster. + +## Why Start Here + +Use this short dialogue before deep authoring. The goal is to learn the real job, output, exclusions, and constraints so the first package is small but accurate. + +## Current Anchor + +- Title: `Geo Content Brief Skill` +- Description: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 +- Focus: `trigger-and-output` +- Reference note: If you already have examples you admire, bring them in. We will learn the pattern, not copy the source. + +## Questions To Ask + +1. If this skill worked beautifully, what recurring job would it reliably handle for the user every time? + Why: This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape. +2. When someone reaches for this skill in the real world, what materials will they actually hand to it? + Why: Input shape decides whether references, scripts, or templates are needed. +3. What finished output should it hand back so the user can immediately keep moving? + Why: Outputs should drive the package structure before extra guidance is added. +4. Which nearby requests should this skill politely refuse so the boundary stays clean? + Why: The exclusion list is the fastest route to better trigger quality. +5. What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit? + Why: Constraints decide how much structure, packaging, and review the skill actually needs. +6. Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example? + Why: A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material. +7. What repeated manual step should become a deterministic asset first? + Why: This usually reveals whether a script or reference should be created next. + +## Capture Before Drafting + +- Capability sentence: Geo Content Brief Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily. +- Recommended first gate: `trigger and boundary` +- Tiny optional scaffold: + - The repeated job it should reliably handle + - The real inputs people will hand to it + - The useful output it should hand back + - What it should clearly refuse +- Capture: `recurring job` +- Capture: `real inputs` +- Capture: `required outputs` +- Capture: `exclusions` +- Capture: `constraints` +- Capture: `reference preferences` +- Capture: `first evaluation target` diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.json new file mode 100644 index 00000000..59ef33e3 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.json @@ -0,0 +1,52 @@ +{ + "summary": { + "skill_name": "geo-content-brief-skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "maturity_tier": "scaffold", + "selection_rule": "Pick the three smallest next steps that increase reliability more than they increase context cost.", + "recommended_now": "Tighten trigger and exclusions", + "recommended_now_why": "The package needs clearer near-neighbor exclusions before it grows.", + "defer_for_now": "Promote from scaffold to production-ready" + }, + "directions": [ + { + "priority": 1, + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations.", + "do_now": "Do this first.", + "wait_on": "Wait to add broader structure until this move clearly improves reliability." + }, + { + "priority": 2, + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint.", + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." + }, + { + "priority": 3, + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset.", + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." + } + ] +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.md new file mode 100644 index 00000000..62453b19 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/iteration-directions.md @@ -0,0 +1,62 @@ +# Iteration Directions + +Skill: `geo-content-brief-skill` + +- Maturity tier: `scaffold` +- Selection rule: Pick the three smallest next steps that increase reliability more than they increase context cost. +- Start here: `Tighten trigger and exclusions` +- Why first: The package needs clearer near-neighbor exclusions before it grows. +- Defer for now: `Promote from scaffold to production-ready` + +## Top 3 Next Moves + +### 1. Tighten trigger and exclusions + +- Why now: The package needs clearer near-neighbor exclusions before it grows. +- Timing: Do this first. +- Recommended actions: + - Add 3 to 5 should-trigger and should-not-trigger examples. +- Unlocks: Cleaner routing and fewer accidental activations. +- Wait on: Wait to add broader structure until this move clearly improves reliability. + + - Refine the frontmatter description to name the recurring job and non-goals. +- Unlocks: Cleaner routing and fewer accidental activations. +- Wait on: Wait to add broader structure until this move clearly improves reliability. + + - Run a first trigger evaluation pass before expanding the package. +- Unlocks: Cleaner routing and fewer accidental activations. +- Wait on: Wait to add broader structure until this move clearly improves reliability. + +### 2. Add the first execution asset + +- Why now: The package is still mostly prose. Add one asset that removes repeated manual work. +- Timing: Do this after the first move lands cleanly. +- Recommended actions: + - Move stable procedural guidance into references if users will need it repeatedly. +- Unlocks: Stronger execution quality without bloating the entrypoint. +- Wait on: Wait until the package has evidence that this extra structure is justified. + + - Create one deterministic helper script if a repeated step can be executed instead of described. +- Unlocks: Stronger execution quality without bloating the entrypoint. +- Wait on: Wait until the package has evidence that this extra structure is justified. + + - Keep the main SKILL.md compact and route-oriented. +- Unlocks: Stronger execution quality without bloating the entrypoint. +- Wait on: Wait until the package has evidence that this extra structure is justified. + +### 3. Promote from scaffold to production-ready + +- Why now: The first version exists; the next gain usually comes from adding the smallest useful gates. +- Timing: Do this after the first move lands cleanly. +- Recommended actions: + - Decide whether this skill is personal, team-reused, or library-grade. +- Unlocks: A clearer path from exploratory package to maintained asset. +- Wait on: Wait until the package has evidence that this extra structure is justified. + + - Add only the gates that match that risk level. +- Unlocks: A clearer path from exploratory package to maintained asset. +- Wait on: Wait until the package has evidence that this extra structure is justified. + + - Record lifecycle metadata and review cadence once reuse becomes real. +- Unlocks: A clearer path from exploratory package to maintained asset. +- Wait on: Wait until the package has evidence that this extra structure is justified. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.json new file mode 100644 index 00000000..35b20fbc --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.json @@ -0,0 +1,147 @@ +{ + "skill_name": "geo-content-brief-skill", + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "table", + "report", + "brief" + ], + "score": 4, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "screenshot", + "visual", + "screen" + ], + "score": 3, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "citation", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "copy", + "content" + ], + "score": 2, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + }, + { + "key": "tutorial_quality", + "label": "Tutorial quality", + "matched_keywords": [ + "tutorial" + ], + "score": 1, + "risks": [ + "Generic section headings make the tutorial feel templated instead of fitted to the learner's task.", + "Steps may explain what to do without naming the exact check that proves the step worked.", + "Examples can become too abstract when the input material is concrete." + ], + "constraints": [ + "Write headings from the user's domain nouns and desired outcome, not from generic labels like Overview or Key Points.", + "Pair each major step with a visible success check or expected intermediate output.", + "Use one concrete worked example before adding broad principles." + ], + "self_repair": [ + "Replace generic H2/H3 headings with task-specific headings before final output.", + "Scan every numbered step for a missing verification cue." + ] + } + ], + "top_risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims." + ], + "output_constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability." + ], + "self_repair_checks": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards.", + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state.", + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ], + "reviewer_note": "Use this report before deepening the package and again before approving example outputs." +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.md new file mode 100644 index 00000000..547566b5 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/output-risk-profile.md @@ -0,0 +1,60 @@ +# Output Risk Profile + +Skill: `geo-content-brief-skill` + +## Why This Exists + +Generated skills often fail in small output details: generic headings, cluttered citations, fragile screenshots, weak Markdown rendering, or missing execution assumptions. This profile predicts the most likely output mistakes before the skill is used heavily. + +## Matched Risk Families + +### Markdown readability +- Matched keywords: md, table, report, brief +- Score: `4` + +### Screenshot and visual capture +- Matched keywords: screenshot, visual, screen +- Score: `3` + +### Citation and footnote clutter +- Matched keywords: citation, reference +- Score: `2` + +### Tone and specificity +- Matched keywords: copy, content +- Score: `2` + +### Tutorial quality +- Matched keywords: tutorial +- Score: `1` + +## Likely Output Mistakes + +- Tables can render as dense grids with weak hierarchy or poor mobile readability. +- Long bullets can make the output look complete while hiding the actual decision logic. +- Screenshots can be captured from the wrong state, wrong viewport, or wrong crop. +- Missing screenshots can cause the skill to invent visual references instead of declaring the gap. +- Footnote markers or dense citation notes can interrupt the reading flow. +- Evidence can be over-attached to obvious statements and under-attached to risky claims. + +## Output Constraints To Apply + +- Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets. +- Keep table cells short and move explanations below the table. +- Never invent a screenshot; state when visual evidence is missing. +- Record the source, viewport, and crop intent for any screenshot-dependent output. +- Attach citations only to claims that need evidence, not to every sentence. +- Group source notes at the end of a section when inline markers would hurt readability. + +## Self-Repair Checks + +- Preview whether each table still reads well when columns are narrow. +- Convert any table with paragraph-length cells into bullets or cards. +- Check that every screenshot reference points to a real provided or generated asset. +- Reword any visual instruction that depends on an unseen screen state. +- Remove decorative citations that do not support a material claim. +- Move repeated source explanations into one compact source note. + +## Reviewer Note + +Use this report before deepening the package and again before approving example outputs. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.json new file mode 100644 index 00000000..05acd531 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.json @@ -0,0 +1,146 @@ +{ + "skill_name": "geo-content-brief-skill", + "relevance": "prompt-heavy", + "primary_task_family": { + "key": "creative_generation", + "label": "Creative generation", + "matched_keywords": [ + "copy", + "content", + "内容" + ] + }, + "task_families": [ + { + "key": "creative_generation", + "label": "Creative generation", + "score": 3, + "matched_keywords": [ + "copy", + "content", + "内容" + ], + "role_guidance": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.", + "task_guidance": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.", + "format_guidance": "Return options with rationale, selection criteria, and refinement paths." + }, + { + "key": "execution_operation", + "label": "Execution operation", + "score": 3, + "matched_keywords": [ + "workflow", + "execute", + "执行" + ], + "role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", + "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", + "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." + }, + { + "key": "prompt_engineering", + "label": "Prompt engineering", + "score": 3, + "matched_keywords": [ + "prompt", + "role", + "format" + ], + "role_guidance": "Use a prompt engineer role only when role design materially improves execution.", + "task_guidance": "Map Role, Task, and Format into skill behavior rather than copying a large prompt template.", + "format_guidance": "Return a compact prompt contract plus tests, quality matrix, and usage notes." + }, + { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "score": 2, + "matched_keywords": [ + "dialogue", + "访谈" + ], + "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." + } + ], + "complexity": { + "band": "complex", + "score": 6, + "reason": "multiple inputs, constraints, or task families require tradeoff handling" + }, + "need_model": { + "explicit_need": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.", + "scenario": "not yet explicit", + "user_level": "infer from examples and standards; ask only if it changes output depth", + "success_standard": "usable output with clear validation cues" + }, + "rtf_to_skill": { + "role": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.", + "task": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.", + "format": "Return options with rationale, selection criteria, and refinement paths." + }, + "quality_matrix": [ + { + "key": "completeness", + "label": "Completeness", + "score": 80, + "matched_signals": [ + "input", + "output", + "constraint", + "example", + "约束" + ], + "repair": "Name missing inputs, outputs, constraints, or success standards before deepening the package." + }, + { + "key": "clarity", + "label": "Clarity", + "score": 90, + "matched_signals": [ + "clear", + "specific" + ], + "repair": "Replace broad verbs with observable actions and define what done means." + }, + { + "key": "consistency", + "label": "Consistency", + "score": 90, + "matched_signals": [ + "boundary", + "边界" + ], + "repair": "Check that role, task, format, exclusions, and examples do not contradict each other." + }, + { + "key": "practicality", + "label": "Practicality", + "score": 95, + "matched_signals": [ + "execute", + "use", + "workflow", + "执行" + ], + "repair": "Add runnable steps, examples, or verification cues instead of abstract advice." + }, + { + "key": "specificity", + "label": "Specificity", + "score": 70, + "matched_signals": [], + "repair": "Anchor wording in the user's audience, domain nouns, and target outcome." + } + ], + "overall_quality_score": 85.0, + "self_repair_checks": [ + "Check explicit need, implicit need, scenario, user level, and success standard before deepening.", + "Map Role, Task, and Format into skill behavior, not decorative prompt labels.", + "Ask one focused clarification only when missing information changes the package boundary.", + "Add tests or examples for prompt-heavy behavior before treating it as reusable.", + "Keep prompt methodology in references and reports instead of bloating SKILL.md." + ], + "reviewer_note": "Use this profile when the package depends on prompt behavior, role design, output contracts, or conversation quality." +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.md new file mode 100644 index 00000000..5c2995a6 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/prompt-quality-profile.md @@ -0,0 +1,94 @@ +# Prompt Quality Profile + +Skill: `geo-content-brief-skill` +Relevance: `prompt-heavy` +Overall quality score: `85.0/100` + +## Primary Task Family + +**Creative generation** +- Matched keywords: copy, content, 内容 + +## Complexity + +- Band: `complex` +- Score: `6` +- Reason: multiple inputs, constraints, or task families require tradeoff handling + +## Need Model + +- Explicit Need: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 +- Implicit Need: The reusable skill needs a stable role, task, and output contract rather than a one-off prompt. +- Scenario: not yet explicit +- User Level: infer from examples and standards; ask only if it changes output depth +- Success Standard: usable output with clear validation cues + +## RTF To Skill Mapping + +- Role: Use a taste-aware creator role with clear audience, tone, and originality boundaries. +- Task: Generate variants, explain selection logic, and preserve the user's distinctive constraints. +- Format: Return options with rationale, selection criteria, and refinement paths. + +## Quality Matrix + +### Completeness — 80/100 +- Matched signals: input, output, constraint, example, 约束 +- Repair: Name missing inputs, outputs, constraints, or success standards before deepening the package. + +### Clarity — 90/100 +- Matched signals: clear, specific +- Repair: Replace broad verbs with observable actions and define what done means. + +### Consistency — 90/100 +- Matched signals: boundary, 边界 +- Repair: Check that role, task, format, exclusions, and examples do not contradict each other. + +### Practicality — 95/100 +- Matched signals: execute, use, workflow, 执行 +- Repair: Add runnable steps, examples, or verification cues instead of abstract advice. + +### Specificity — 70/100 +- Matched signals: none +- Repair: Anchor wording in the user's audience, domain nouns, and target outcome. + +## Matched Task Families + +### Creative generation +- Score: `3` +- Keywords: copy, content, 内容 +- Role: Use a taste-aware creator role with clear audience, tone, and originality boundaries. +- Task: Generate variants, explain selection logic, and preserve the user's distinctive constraints. +- Format: Return options with rationale, selection criteria, and refinement paths. + +### Execution operation +- Score: `3` +- Keywords: workflow, execute, 执行 +- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling. +- Task: Convert the job into ordered steps with validation checks and stop conditions. +- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant. + +### Prompt engineering +- Score: `3` +- Keywords: prompt, role, format +- Role: Use a prompt engineer role only when role design materially improves execution. +- Task: Map Role, Task, and Format into skill behavior rather than copying a large prompt template. +- Format: Return a compact prompt contract plus tests, quality matrix, and usage notes. + +### Dialogue interaction +- Score: `2` +- Keywords: dialogue, 访谈 +- Role: Use a conversational role that asks only high-leverage questions and remembers the user's goal. +- Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list. +- Format: Return concise prompts, decision points, and reviewer-visible assumptions. + +## Self-Repair Checks + +- Check explicit need, implicit need, scenario, user level, and success standard before deepening. +- Map Role, Task, and Format into skill behavior, not decorative prompt labels. +- Ask one focused clarification only when missing information changes the package boundary. +- Add tests or examples for prompt-heavy behavior before treating it as reusable. +- Keep prompt methodology in references and reports instead of bloating SKILL.md. + +## Reviewer Note + +Use this profile when the package depends on prompt behavior, role design, output contracts, or conversation quality. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.json new file mode 100644 index 00000000..57534651 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.json @@ -0,0 +1,36 @@ +{ + "skill_name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "scan_focus": [ + { + "label": "Execution pattern", + "reason": "There is deterministic logic in scripts, so compare how strong references separate prose from executable steps." + }, + { + "label": "Portability pattern", + "reason": "The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source." + }, + { + "label": "Method pattern", + "reason": "Use the core job description as the anchor for comparison: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + } + ], + "external_references": [], + "user_references": [], + "local_constraints": [], + "borrow_plan": [ + "External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability.", + "User references second: use them to understand taste, standards, and directional preferences without copying source phrasing.", + "Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints.", + "Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language.", + "Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability." + ], + "priority_rule": "External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility.", + "non_goals": [ + "Do not copy source prose or branding into the new skill.", + "Do not import gates that cost more context than they save.", + "Do not use benchmark scanning to justify scope creep.", + "Do not let local historical habits outrank stronger public benchmarks." + ] +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.md new file mode 100644 index 00000000..06b83aa8 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-scan.md @@ -0,0 +1,54 @@ +# Reference Scan + +Skill: `geo-content-brief-skill` + +## Why This Step Exists + +Use a short benchmark pass before authoring the package in depth. External benchmark objects should define the pattern ceiling. Local files are used afterward only to calibrate fit, privacy, naming, and compatibility. + +## Current Skill Anchor + +- Title: `Geo Content Brief Skill` +- Description: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 + +## Scan Focus + +- **Execution pattern**: There is deterministic logic in scripts, so compare how strong references separate prose from executable steps. +- **Portability pattern**: The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source. +- **Method pattern**: Use the core job description as the anchor for comparison: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 + +## Priority Rule + +- External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility. + +## External Benchmark Objects + +- No explicit external benchmark objects recorded yet. +- Recommended: capture 2 to 5 external references at most. +- Suggested mix: one method reference, one structure reference, one execution or portability reference. + +## User-Supplied References + +- No user-supplied references recorded yet. +- Ask whether the user has a repo, product, page, workflow, or prompt example worth learning from. +- Treat these as pattern references only, not as text to be copied. + +## Local Fit Check + +- No local constraints recorded yet. +- Use this section for naming collisions, private dependencies, compatibility limits, or existing library conventions. + +## Borrow Plan + +- External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability. +- User references second: use them to understand taste, standards, and directional preferences without copying source phrasing. +- Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints. +- Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language. +- Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability. + +## Non-Goals + +- Do not copy source prose or branding into the new skill. +- Do not import gates that cost more context than they save. +- Do not use benchmark scanning to justify scope creep. +- Do not let local historical habits outrank stronger public benchmarks. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.json new file mode 100644 index 00000000..c998eab2 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.json @@ -0,0 +1,171 @@ +{ + "skill_name": "geo-content-brief-skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "intent_confidence": { + "score": 10, + "band": "low", + "gate_passed": false + }, + "github_benchmarks": [], + "source_tracks": [ + { + "source_type": "official", + "name": "Official skill anatomy and context discipline", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "why_relevant": "This track matches: general fit." + }, + { + "source_type": "research", + "name": "Hypothesis-test-learn loop", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "why_relevant": "This track matches: general fit." + }, + { + "source_type": "principles", + "name": "Boundary-first design", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "why_relevant": "This track matches: general fit." + } + ], + "synthesis": { + "borrow_now": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "avoid_now": [ + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not create experimental overhead that exceeds the skill's real risk tier.", + "Do not expand execution assets until route boundaries stay clean." + ], + "quality_risers": [ + "Use GitHub repositories for concrete package and workflow patterns.", + "Use curated official or commercial tracks for entrypoint and operator ergonomics.", + "Use research tracks to justify the smallest evaluation loop that still catches regressions.", + "Use principle tracks to keep the package small, boundary-aware, and outcome-driven." + ], + "pattern_gate": { + "threshold": 2, + "source_count": 3, + "accepted": [ + { + "name": "Official skill anatomy and context discipline", + "source_type": "official", + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + }, + { + "name": "Hypothesis-test-learn loop", + "source_type": "research", + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + }, + { + "name": "Boundary-first design", + "source_type": "principles", + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + } + ], + "deferred": [], + "summary": "3 accepted, 0 deferred using threshold 2/4." + }, + "conflicts": [ + { + "key": "lightweight_vs_governance", + "summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_preference": "lightweight or speed-first", + "benchmark_pressure": "governance or evaluation-heavy patterns" + } + ], + "recommendation": { + "summary": "Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.", + "borrow_now": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed." + ], + "avoid_for_now": [ + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not create experimental overhead that exceeds the skill's real risk tier." + ], + "why": "There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_decision_required": true + }, + "visibility": { + "mode": "explicit", + "user_decision_required": true, + "reasons": [ + "intent_uncertain", + "design_conflict" + ], + "user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.", + "reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers." + }, + "decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.", + "source_mix": { + "github_benchmarks": 0, + "curated_tracks": 3, + "user_references": 0 + } + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.md new file mode 100644 index 00000000..8824a75a --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/reference-synthesis.md @@ -0,0 +1,81 @@ +# Reference Synthesis + +Skill: `geo-content-brief-skill` +- Description: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 +- Intent confidence: `10/100` (`low`) + +## Live GitHub Benchmarks + +- No live GitHub benchmarks are attached yet. + +## Curated World-Class Pattern Tracks + +### Official skill anatomy and context discipline +- Type: `official` +- Evidence mode: `curated-pattern-track` +- Why relevant: This track matches: general fit. +- Borrow: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. +- Avoid: Do not let packaging or platform concerns swallow the core job boundary. + +### Hypothesis-test-learn loop +- Type: `research` +- Evidence mode: `curated-pattern-track` +- Why relevant: This track matches: general fit. +- Borrow: Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. +- Avoid: Do not create experimental overhead that exceeds the skill's real risk tier. + +### Boundary-first design +- Type: `principles` +- Evidence mode: `curated-pattern-track` +- Why relevant: This track matches: general fit. +- Borrow: Borrow the discipline of defining what the skill should not own before growing the package. +- Avoid: Do not expand execution assets until route boundaries stay clean. + +## Borrow Now + +- Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. +- Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. +- Borrow the discipline of defining what the skill should not own before growing the package. + +## Avoid Now + +- Do not let packaging or platform concerns swallow the core job boundary. +- Do not create experimental overhead that exceeds the skill's real risk tier. +- Do not expand execution assets until route boundaries stay clean. + +## Pattern Gate + +- Summary: 3 accepted, 0 deferred using threshold 2/4. +- Acceptance threshold: `2/4` +- Accepted patterns: + - **Official skill anatomy and context discipline**: 3/4 (recurrence, generativity, boundary) + - **Hypothesis-test-learn loop**: 3/4 (recurrence, generativity, boundary) + - **Boundary-first design**: 3/4 (recurrence, generativity, boundary) + +## Default Recommendation + +- Summary: Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary. +- Why: There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure. +- User decision required: `True` + +## Visibility Mode + +- Mode: `explicit` +- Reasons: intent_uncertain, design_conflict +- User note: Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call. +- Reviewer note: Keep the full benchmark and synthesis evidence visible for authors and reviewers. + +## Conflict Check + +- **lightweight_vs_governance**: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure. + +## Quality Lift Thesis + +- Use GitHub repositories for concrete package and workflow patterns. +- Use curated official or commercial tracks for entrypoint and operator ergonomics. +- Use research tracks to justify the smallest evaluation loop that still catches regressions. +- Use principle tracks to keep the package small, boundary-aware, and outcome-driven. + +## Decision Prompt + +Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.html b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.html new file mode 100644 index 00000000..a1d72d2b --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.html @@ -0,0 +1,720 @@ + + + + + + Geo Content Brief Skill Review Studio 2.0 + + + + +
+
+
Review Studio 2.0
+

Geo Content Brief Skill

+

将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。

+
+ 审查结论 + review + Score 67/100 +
+
+ +
+

核心指标

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Deltan/a

0 cases; 0 file-backed

Exec Runs0

command 0; model 0; recorded 0

Blind A/B0

review pairs hide baseline vs with-skill labels

Review Kit0/0

pending 0; answer key hidden

Review A/B0/0

adjudication decisions; pending 0

Public Claimblocked

0 blockers; local reproducible false

Blueprint0/0

2.0 coverage; evidence pending 0

Runtime0/0

target conformance pass rate

Perm Probe0/0

0 native; 0 installer-enforced

Trust0

0 scripts scanned; secrets found

Py Compat0

0 files scanned for Python 3.11

Arch Debt0

0 largest lines; 0 CLI handlers

Atlas0

0 scanned skills; route collisions

Driftno-data

0 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake0/0

0 valid submissions; 0 invalid

Claim Guard0

0 public surfaces scanned

Notes0/0

0 open blocker annotations

Registryn/a

0 targets; no license license

Archiven/a

0 zip entries; package verification

Installn/a

0 adapters; 0 permissions enforced; 0 permission failures

Upgraden/a

declared n/a; 0 breaking changes

+
+ +
+

审查闸门

+
关注

意图画布

intent confidence 10/100; Pause before deep authoring and close the highest-leverage gaps first.

reports/intent-confidence.json 证据
关注

触发实验

route scorecard is missing; run route-scorecard before release review

reports/route_scorecard.json 证据
关注

输出实验

output eval scorecard is missing; generate it before production review

reports/output_quality_scorecard.json 证据
关注

上下文

initial load n/a/n/a; deferred n/a/n/a; top deferred none; resource governance unknown; quality density n/a

reports/context_budget.json 证据
关注

运行矩阵

runtime conformance matrix is missing

reports/conformance_matrix.json 证据
关注

信任报告

security trust report is missing

reports/security_trust_report.json 证据
关注

Python 兼容

Python compatibility report is missing

reports/python_compatibility.json 证据
通过

架构维护

architecture maintainability report is optional until code surface grows

reports/architecture_maintainability.json 证据
关注

权限批准

permission governance evidence is missing because trust report is missing

reports/security_trust_report.json + security/permission_policy.json 证据
关注

权限探针

runtime permission probe report is missing

reports/runtime_permission_probes.json 证据
关注

组合治理

skill atlas is missing; portfolio-level conflicts are unknown

reports/skill_atlas.json 证据
关注

运营回路

0 metadata events; adoption 0; missed 0; bad-output 0; risk no-data

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 11 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

世界证据

world-class evidence ledger is optional until governed or public world-class readiness is claimed

reports/world_class_evidence_ledger.json 证据
通过

注册审计

registry audit is optional until team distribution is required

reports/registry_audit.json + reports/install_simulation.json 证据
关注

发布路线

promotion decisions are missing; release notes need reviewer confirmation

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
+ +
+

阻断事项

无。

+

关注事项

  • 意图画布intent confidence 10/100; Pause before deep authoring and close the highest-leverage gaps first.
  • 触发实验route scorecard is missing; run route-scorecard before release review
  • 输出实验output eval scorecard is missing; generate it before production review
  • 上下文initial load n/a/n/a; deferred n/a/n/a; top deferred none; resource governance unknown; quality density n/a
  • 运行矩阵runtime conformance matrix is missing
  • 信任报告security trust report is missing
  • Python 兼容Python compatibility report is missing
  • 权限批准permission governance evidence is missing because trust report is missing
  • 权限探针runtime permission probe report is missing
  • 组合治理skill atlas is missing; portfolio-level conflicts are unknown
  • 运营回路0 metadata events; adoption 0; missed 0; bad-output 0; risk no-data
  • 人工批准0 active waivers; 11 warning gates still need reviewer decision
  • 发布路线promotion decisions are missing; release notes need reviewer confirmation
+
+ +
+

修复动作

+
关注

意图画布

收紧真实任务、输入、输出、排除项和成功标准。

低 intent confidence 会让后续 Skill IR、输出评测和 Review Studio 结论建立在模糊意图上。
修复位置
reports/intent-dialogue.md + reports/intent-confidence.md
验证命令
python3 scripts/yao.py intent-confidence .
关注

触发实验

修正 route scorecard 中的误触发、漏触发或 ambiguous case。

触发错误会让正确 Skill 失活，或让相邻 Skill 被错误调用。
修复位置
SKILL.md frontmatter description + evals/*/trigger_cases.json
验证命令
python3 scripts/build_confusion_matrix.py
  • SKILL.md:3frontmatter description · source
  • evals/trigger_cases.json · missingtrigger eval cases · eval
  • reports/route_scorecard.md · missingroute scorecard · report
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
  • evals/output/cases.jsonl · missingoutput eval cases · eval
  • reports/output_quality_scorecard.md · missingoutput scorecard · report
  • reports/output_execution_runs.md · missingoutput execution runs · report
  • reports/output_blind_review_pack.md · missingblind A/B review pack · report
  • reports/output_review_kit.html · missingreviewer cockpit · report
  • reports/output_review_adjudication.md · missingreview adjudication · report
关注

上下文

压缩或拆分高成本 deferred resources，保留最小可路由上下文。

初始加载可以安全，但后续 references、scripts、evals 体量过大时，reviewer 仍需要看到维护和读取成本。
修复位置
SKILL.md + references/ + scripts/ + evals/
验证命令
python3 scripts/render_context_reports.py
  • SKILL.md:1entrypoint · source
  • reports/context_budget.md · missingcontext budget · report
  • scripts/resource_boundary_check.py · missingresource boundary checker · source
  • references/skill-engineering-method.md · missingskill engineering method · method
关注

运行矩阵

修复目标端结构、metadata、相对路径、fallback 或 adapter target 声明。

runtime conformance 失败意味着包可能被目标客户端错误加载或静默降级。
修复位置
agents/interface.yaml + reports/conformance_matrix.md
验证命令
python3 scripts/run_conformance_suite.py .
  • agents/interface.yaml:7portable interface · source
  • reports/conformance_matrix.md · missingconformance matrix · report
关注

信任报告

处理脚本 help surface、依赖 pin、network policy、secret 和权限声明。

团队分发时，脚本和依赖是主要供应链风险面，warning 必须有明确处置。
修复位置
reports/security_trust_report.md + security/*.md + scripts/
验证命令
python3 scripts/trust_check.py .
  • reports/security_trust_report.md · missingtrust report · report
  • security/script_policy.md · missingscript policy · policy
  • security/network_policy.md · missingnetwork policy · policy
关注

Python 兼容

修复 Python 3.11 语法兼容问题，尤其是 f-string 表达式内的反斜杠转义。

目标运行环境可能仍停留在 Python 3.11，语法漂移会让 CLI、报告生成和 CI 在发布后直接失败。
修复位置
reports/python_compatibility.md + scripts/*.py + tests/*.py
验证命令
python3 scripts/yao.py python-compat .
  • reports/python_compatibility.md · missingPython compatibility · report
  • scripts/python_compat_check.py · missingcompatibility checker · source
  • .github/workflows/test.yml · missingCI test workflow · ci
关注

权限批准

补齐高权限能力的 reviewer、scope、reason、expires_at 和目标端 enforcement 说明。

权限契约只有在批准人、有效期和目标端处置方式明确时，才能支撑 governed release。
修复位置
security/permission_policy.json + security/permission_policy.md
验证命令
python3 scripts/trust_check.py .
  • security/permission_policy.json · missingpermission approvals · policy
  • security/permission_policy.md · missingpermission method · policy
reports/security_trust_report.json + security/permission_policy.json 打开证据
关注

权限探针

生成并修复目标包的 runtime permission probe 报告。

目标端即使只能提供 metadata fallback，也必须明确 native enforcement 缺口、表示位置和 operator note。
修复位置
dist/targets/*/adapter.json + reports/runtime_permission_probes.md
验证命令
python3 scripts/probe_runtime_permissions.py . --package-dir dist
  • reports/runtime_permission_probes.md · missingruntime permission probes · report
  • dist/targets/openai/adapter.json · missingOpenAI adapter · package
  • dist/targets/claude/adapter.json · missingClaude adapter · package
  • dist/targets/generic/adapter.json · missinggeneric adapter · package
关注

组合治理

处理 portfolio 里的路由冲突、owner 缺口、stale skill 和重复能力。

单个 Skill 质量很高仍可能在团队 skill library 中被相邻 Skill 冲突削弱。
修复位置
reports/skill_atlas.html + skill_atlas/catalog.json
验证命令
python3 scripts/build_skill_atlas.py --workspace-root .
  • skill_atlas/catalog.json · missingskill atlas catalog · atlas
  • skill_atlas/policy.json · missingatlas scope policy · policy
  • reports/skill_atlas.html · missingskill atlas report · report
关注

运营回路

记录 metadata-only 使用事件，或明确当前 release 缺少真实使用信号。

没有运营回路时，reviewer 无法判断采用率、误触发、坏输出和 review overdue 的真实影响。
修复位置
reports/adoption_drift_report.md
验证命令
python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

发布路线

确认 promotion、upgrade diff、breaking changes、migration guide 和 known limitations。

发布说明不完整会让使用者无法判断升级风险和迁移动作。
修复位置
reports/upgrade_check.md + docs/migration-v2.md
验证命令
python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json
  • reports/upgrade_check.md · missingupgrade check · report
  • docs/migration-v2.md · missingmigration guide · docs
  • reports/promotion_decisions.md · missingpromotion decisions · report
reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 打开证据
+
+ +
+

审查批注

+

当前批注：0 条批注；开放 0；阻断 0

+

当前没有 reviewer 批注。

+
+ +
+
+

意图画布

+

将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。

+
+
+

证据路径

+
  • skill_overviewreports/skill-overview.html
  • review_viewerreports/review-viewer.html
  • output_evalreports/output_quality_scorecard.md
  • output_executionreports/output_execution_runs.md
  • output_blind_reviewreports/output_blind_review_pack.md
  • output_review_kitreports/output_review_kit.md
  • output_review_kit_htmlreports/output_review_kit.html
  • output_review_decisionsreports/output_review_decisions.json
  • output_review_adjudicationreports/output_review_adjudication.md
  • benchmark_reproducibilityreports/benchmark_reproducibility.md
  • skill_os2_coveragereports/skill_os2_coverage.md
  • runtime_conformancereports/conformance_matrix.md
  • trust_reportreports/security_trust_report.md
  • python_compatibilityreports/python_compatibility.md
  • architecture_maintainabilityreports/architecture_maintainability.md
  • permission_policysecurity/permission_policy.md
  • runtime_permissionsreports/runtime_permission_probes.md
  • skill_atlasreports/skill_atlas.html
  • compiled_targetsreports/compiled_targets.md
  • adoption_driftreports/adoption_drift_report.md
  • review_waiversreports/review_waivers.md
  • review_annotationsreports/review_annotations.md
  • world_class_evidence_planreports/world_class_evidence_plan.md
  • world_class_evidence_ledgerreports/world_class_evidence_ledger.md
  • world_class_evidence_intakereports/world_class_evidence_intake.md
  • world_class_submission_reviewreports/world_class_submission_review.md
  • world_class_operator_runbookreports/world_class_operator_runbook.md
  • world_class_operator_runbook_htmlreports/world_class_operator_runbook.html
  • world_class_claim_guardreports/world_class_claim_guard.md
  • registry_auditreports/registry_audit.md
  • package_verificationreports/package_verification.md
  • install_simulationreports/install_simulation.md
  • upgrade_checkreports/upgrade_check.md
  • migrationdocs/migration-v2.md
  • skill_irreports/skill-ir.json
+
+
+ +
+

触发实验

route scorecard is missing; run route-scorecard before release review

+

组合治理

skill atlas summary missing

+
+ +
+

输出实验

output eval scorecard missing

+

执行证据

output execution report missing

+
+ +
+

盲评包

blind A/B review pack missing

+

审定报告

review adjudication report missing

+
+ +

评审清单

先打开 reports/output_review_kit.md；每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

当前没有输出评审操作清单。

+
+

发布标准

Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。

+

人工结论

没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。

+
+ +
+

评审方式

先打开 reports/output_review_kit.html 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

+

运行方式

reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

+
+ +
+

运行矩阵

runtime conformance matrix missing

+

目标编译

目标数
5
通过数
5
Warn Count
0
Block Count
0
失败数
0
+
+ +
+

上下文

initial load n/a/n/a; deferred n/a/n/a; top deferred none; resource governance unknown; quality density n/a

+

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

+
+ +
+

信任报告

security trust report missing

+

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

+
+ +
+

Python 兼容

python compatibility report missing

+

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。

+
+ +
+

权限批准

permission governance evidence is missing because trust report is missing

+

批准策略

高权限能力需要 reviewer、scope、reason、expires_at 和 openai/claude/generic 目标端 enforcement 说明。

+
+ +
+

权限探针

runtime permission probe report is missing

+

运行边界

runtime permission probe summary missing

+
+ +
+

组合治理

skill atlas is missing; portfolio-level conflicts are unknown

+

下一动作

优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再用运营回路判断真实影响。

+
+ +
+

运营回路

0 metadata events; adoption 0; missed 0; bad-output 0; risk no-data

+

漂移信号

事件数
0
采用率
0
漏触发
0
Bad Output Count
0
风险带
no-data
+
+ +
+

人工批准

+

warning 可以被有边界地接受，但必须写入 reviewer、理由、范围和到期时间；blocker 与 world-class 完成证据不能通过 waiver 变成通过。

+
+

批准概况

0 active waivers; 11 warning gates still need reviewer decision

+

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0
+
+

批准候选

+

当前没有需要 reviewer 处理的批准候选。

+
+ +
+

世界证据

+

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

+

当前没有 world-class 证据条目。

+
+ +
+

提交清单

+

每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界；只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后，才进入 ledger review。

+

当前没有 world-class 证据操作清单。

+
+ +
+

蓝图覆盖

Skill OS 2.0 blueprint coverage missing

+

覆盖边界

蓝图覆盖只证明 2.0 模块、建议 PR、脚本、报告和测试在本地闭环；public world-class 仍以 world-class evidence ledger 的真人和外部证据为准。

+
+ +
+

公开声明

benchmark reproducibility report missing

+

声明阻断

无公开声明阻断。

+
+ +
+

世界证据

world-class evidence ledger is optional until governed or public world-class readiness is claimed

+

证据台账

world-class evidence ledger missing

+
+ +
+

证据入口

world-class evidence intake missing

+

入口边界

intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。

+
+ +
+

声明守卫

world-class claim guard missing

+

声明边界

claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

+
+ +
+

注册审计

registry audit is optional until team distribution is required

+

包体元数据

registry package metadata missing

+
+ +
+

发布路线

promotion decisions are missing; release notes need reviewer confirmation

+

包体验证

package verification missing

+
+ +
+

安装模拟

install simulation missing

+

权限覆盖

安装模拟会读取解压后的 target adapter，并按 declared_capabilities 核对 security/permission_policy.json 中的有效批准、证据、过期时间和目标端 enforcement 说明。

+
+
+ + diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.json new file mode 100644 index 00000000..a66a20d4 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-studio.json @@ -0,0 +1,4450 @@ +{ + "schema_version": "2.0", + "ok": true, + "skill_dir": ".", + "summary": { + "decision": "review", + "world_class_score": 67, + "gate_count": 16, + "blocker_count": 0, + "warning_count": 13, + "action_count": 13, + "annotation_count": 0, + "open_annotation_count": 0, + "open_annotation_blocker_count": 0, + "open_annotation_warning_count": 0 + }, + "gates": [ + { + "key": "intent-canvas", + "label": "意图画布", + "status": "warn", + "detail": "intent confidence 10/100; Pause before deep authoring and close the highest-leverage gaps first.", + "evidence": "reports/intent-confidence.json", + "link": "intent-confidence.md" + }, + { + "key": "trigger-lab", + "label": "触发实验", + "status": "warn", + "detail": "route scorecard is missing; run route-scorecard before release review", + "evidence": "reports/route_scorecard.json", + "link": "route_scorecard.md" + }, + { + "key": "output-lab", + "label": "输出实验", + "status": "warn", + "detail": "output eval scorecard is missing; generate it before production review", + "evidence": "reports/output_quality_scorecard.json", + "link": "output_quality_scorecard.md" + }, + { + "key": "context-budget", + "label": "上下文", + "status": "warn", + "detail": "initial load n/a/n/a; deferred n/a/n/a; top deferred none; resource governance unknown; quality density n/a", + "evidence": "reports/context_budget.json", + "link": "context_budget.md" + }, + { + "key": "runtime-matrix", + "label": "运行矩阵", + "status": "warn", + "detail": "runtime conformance matrix is missing", + "evidence": "reports/conformance_matrix.json", + "link": "conformance_matrix.md" + }, + { + "key": "trust-report", + "label": "信任报告", + "status": "warn", + "detail": "security trust report is missing", + "evidence": "reports/security_trust_report.json", + "link": "security_trust_report.md" + }, + { + "key": "python-compat", + "label": "Python 兼容", + "status": "warn", + "detail": "Python compatibility report is missing", + "evidence": "reports/python_compatibility.json", + "link": "python_compatibility.md" + }, + { + "key": "architecture-maintainability", + "label": "架构维护", + "status": "pass", + "detail": "architecture maintainability report is optional until code surface grows", + "evidence": "reports/architecture_maintainability.json", + "link": "architecture_maintainability.md" + }, + { + "key": "permission-gates", + "label": "权限批准", + "status": "warn", + "detail": "permission governance evidence is missing because trust report is missing", + "evidence": "reports/security_trust_report.json + security/permission_policy.json", + "link": "../security/permission_policy.md" + }, + { + "key": "permission-runtime", + "label": "权限探针", + "status": "warn", + "detail": "runtime permission probe report is missing", + "evidence": "reports/runtime_permission_probes.json", + "link": "runtime_permission_probes.md" + }, + { + "key": "skill-atlas", + "label": "组合治理", + "status": "warn", + "detail": "skill atlas is missing; portfolio-level conflicts are unknown", + "evidence": "reports/skill_atlas.json", + "link": "skill_atlas.html" + }, + { + "key": "operations-loop", + "label": "运营回路", + "status": "warn", + "detail": "0 metadata events; adoption 0; missed 0; bad-output 0; risk no-data", + "evidence": "reports/adoption_drift_report.json", + "link": "adoption_drift_report.md" + }, + { + "key": "review-waivers", + "label": "人工批准", + "status": "warn", + "detail": "0 active waivers; 11 warning gates still need reviewer decision", + "evidence": "reports/review_waivers.json", + "link": "review_waivers.md" + }, + { + "key": "world-class-evidence", + "label": "世界证据", + "status": "pass", + "detail": "world-class evidence ledger is optional until governed or public world-class readiness is claimed", + "evidence": "reports/world_class_evidence_ledger.json", + "link": "world_class_evidence_ledger.md" + }, + { + "key": "registry-audit", + "label": "注册审计", + "status": "pass", + "detail": "registry audit is optional until team distribution is required", + "evidence": "reports/registry_audit.json + reports/install_simulation.json", + "link": "registry_audit.md" + }, + { + "key": "release-notes", + "label": "发布路线", + "status": "warn", + "detail": "promotion decisions are missing; release notes need reviewer confirmation", + "evidence": "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", + "link": "/Users/laoyao/AI Coding/03-Development/Skills/yao-meta-skill/docs/migration-v2.md" + } + ], + "blockers": [], + "warnings": [ + { + "key": "intent-canvas", + "label": "意图画布", + "status": "warn", + "detail": "intent confidence 10/100; Pause before deep authoring and close the highest-leverage gaps first.", + "evidence": "reports/intent-confidence.json", + "link": "intent-confidence.md" + }, + { + "key": "trigger-lab", + "label": "触发实验", + "status": "warn", + "detail": "route scorecard is missing; run route-scorecard before release review", + "evidence": "reports/route_scorecard.json", + "link": "route_scorecard.md" + }, + { + "key": "output-lab", + "label": "输出实验", + "status": "warn", + "detail": "output eval scorecard is missing; generate it before production review", + "evidence": "reports/output_quality_scorecard.json", + "link": "output_quality_scorecard.md" + }, + { + "key": "context-budget", + "label": "上下文", + "status": "warn", + "detail": "initial load n/a/n/a; deferred n/a/n/a; top deferred none; resource governance unknown; quality density n/a", + "evidence": "reports/context_budget.json", + "link": "context_budget.md" + }, + { + "key": "runtime-matrix", + "label": "运行矩阵", + "status": "warn", + "detail": "runtime conformance matrix is missing", + "evidence": "reports/conformance_matrix.json", + "link": "conformance_matrix.md" + }, + { + "key": "trust-report", + "label": "信任报告", + "status": "warn", + "detail": "security trust report is missing", + "evidence": "reports/security_trust_report.json", + "link": "security_trust_report.md" + }, + { + "key": "python-compat", + "label": "Python 兼容", + "status": "warn", + "detail": "Python compatibility report is missing", + "evidence": "reports/python_compatibility.json", + "link": "python_compatibility.md" + }, + { + "key": "permission-gates", + "label": "权限批准", + "status": "warn", + "detail": "permission governance evidence is missing because trust report is missing", + "evidence": "reports/security_trust_report.json + security/permission_policy.json", + "link": "../security/permission_policy.md" + }, + { + "key": "permission-runtime", + "label": "权限探针", + "status": "warn", + "detail": "runtime permission probe report is missing", + "evidence": "reports/runtime_permission_probes.json", + "link": "runtime_permission_probes.md" + }, + { + "key": "skill-atlas", + "label": "组合治理", + "status": "warn", + "detail": "skill atlas is missing; portfolio-level conflicts are unknown", + "evidence": "reports/skill_atlas.json", + "link": "skill_atlas.html" + }, + { + "key": "operations-loop", + "label": "运营回路", + "status": "warn", + "detail": "0 metadata events; adoption 0; missed 0; bad-output 0; risk no-data", + "evidence": "reports/adoption_drift_report.json", + "link": "adoption_drift_report.md" + }, + { + "key": "review-waivers", + "label": "人工批准", + "status": "warn", + "detail": "0 active waivers; 11 warning gates still need reviewer decision", + "evidence": "reports/review_waivers.json", + "link": "review_waivers.md" + }, + { + "key": "release-notes", + "label": "发布路线", + "status": "warn", + "detail": "promotion decisions are missing; release notes need reviewer confirmation", + "evidence": "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", + "link": "/Users/laoyao/AI Coding/03-Development/Skills/yao-meta-skill/docs/migration-v2.md" + } + ], + "review_actions": [ + { + "gate_key": "intent-canvas", + "label": "意图画布", + "status": "warn", + "priority": "warning", + "summary": "收紧真实任务、输入、输出、排除项和成功标准。", + "why": "低 intent confidence 会让后续 Skill IR、输出评测和 Review Studio 结论建立在模糊意图上。", + "source_fix": "reports/intent-dialogue.md + reports/intent-confidence.md", + "source_refs": [ + { + "path": "reports/intent-dialogue.md", + "label": "intent dialogue", + "kind": "report", + "line": 1, + "exists": true, + "link": "intent-dialogue.md" + }, + { + "path": "reports/intent-confidence.md", + "label": "intent confidence", + "kind": "report", + "line": 1, + "exists": true, + "link": "intent-confidence.md" + } + ], + "evidence": "reports/intent-confidence.json", + "evidence_link": "intent-confidence.md", + "verification_command": "python3 scripts/yao.py intent-confidence ." + }, + { + "gate_key": "trigger-lab", + "label": "触发实验", + "status": "warn", + "priority": "warning", + "summary": "修正 route scorecard 中的误触发、漏触发或 ambiguous case。", + "why": "触发错误会让正确 Skill 失活，或让相邻 Skill 被错误调用。", + "source_fix": "SKILL.md frontmatter description + evals/*/trigger_cases.json", + "source_refs": [ + { + "path": "SKILL.md", + "label": "frontmatter description", + "kind": "source", + "line": 3, + "exists": true, + "link": "../SKILL.md" + }, + { + "path": "evals/trigger_cases.json", + "label": "trigger eval cases", + "kind": "eval", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/route_scorecard.md", + "label": "route scorecard", + "kind": "report", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/route_scorecard.json", + "evidence_link": "route_scorecard.md", + "verification_command": "python3 scripts/build_confusion_matrix.py" + }, + { + "gate_key": "output-lab", + "label": "输出实验", + "status": "warn", + "priority": "warning", + "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。", + "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md", + "source_refs": [ + { + "path": "evals/output/cases.jsonl", + "label": "output eval cases", + "kind": "eval", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/output_quality_scorecard.md", + "label": "output scorecard", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/output_execution_runs.md", + "label": "output execution runs", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/output_blind_review_pack.md", + "label": "blind A/B review pack", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/output_review_kit.html", + "label": "reviewer cockpit", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/output_review_adjudication.md", + "label": "review adjudication", + "kind": "report", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/output_quality_scorecard.json", + "evidence_link": "output_quality_scorecard.md", + "verification_command": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review" + }, + { + "gate_key": "context-budget", + "label": "上下文", + "status": "warn", + "priority": "warning", + "summary": "压缩或拆分高成本 deferred resources，保留最小可路由上下文。", + "why": "初始加载可以安全，但后续 references、scripts、evals 体量过大时，reviewer 仍需要看到维护和读取成本。", + "source_fix": "SKILL.md + references/ + scripts/ + evals/", + "source_refs": [ + { + "path": "SKILL.md", + "label": "entrypoint", + "kind": "source", + "line": 1, + "exists": true, + "link": "../SKILL.md" + }, + { + "path": "reports/context_budget.md", + "label": "context budget", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "scripts/resource_boundary_check.py", + "label": "resource boundary checker", + "kind": "source", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "references/skill-engineering-method.md", + "label": "skill engineering method", + "kind": "method", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/context_budget.json", + "evidence_link": "context_budget.md", + "verification_command": "python3 scripts/render_context_reports.py" + }, + { + "gate_key": "runtime-matrix", + "label": "运行矩阵", + "status": "warn", + "priority": "warning", + "summary": "修复目标端结构、metadata、相对路径、fallback 或 adapter target 声明。", + "why": "runtime conformance 失败意味着包可能被目标客户端错误加载或静默降级。", + "source_fix": "agents/interface.yaml + reports/conformance_matrix.md", + "source_refs": [ + { + "path": "agents/interface.yaml", + "label": "portable interface", + "kind": "source", + "line": 7, + "exists": true, + "link": "../agents/interface.yaml" + }, + { + "path": "reports/conformance_matrix.md", + "label": "conformance matrix", + "kind": "report", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/conformance_matrix.json", + "evidence_link": "conformance_matrix.md", + "verification_command": "python3 scripts/run_conformance_suite.py ." + }, + { + "gate_key": "trust-report", + "label": "信任报告", + "status": "warn", + "priority": "warning", + "summary": "处理脚本 help surface、依赖 pin、network policy、secret 和权限声明。", + "why": "团队分发时，脚本和依赖是主要供应链风险面，warning 必须有明确处置。", + "source_fix": "reports/security_trust_report.md + security/*.md + scripts/", + "source_refs": [ + { + "path": "reports/security_trust_report.md", + "label": "trust report", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "security/script_policy.md", + "label": "script policy", + "kind": "policy", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "security/network_policy.md", + "label": "network policy", + "kind": "policy", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/security_trust_report.json", + "evidence_link": "security_trust_report.md", + "verification_command": "python3 scripts/trust_check.py ." + }, + { + "gate_key": "python-compat", + "label": "Python 兼容", + "status": "warn", + "priority": "warning", + "summary": "修复 Python 3.11 语法兼容问题，尤其是 f-string 表达式内的反斜杠转义。", + "why": "目标运行环境可能仍停留在 Python 3.11，语法漂移会让 CLI、报告生成和 CI 在发布后直接失败。", + "source_fix": "reports/python_compatibility.md + scripts/*.py + tests/*.py", + "source_refs": [ + { + "path": "reports/python_compatibility.md", + "label": "Python compatibility", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "scripts/python_compat_check.py", + "label": "compatibility checker", + "kind": "source", + "line": null, + "exists": false, + "link": "" + }, + { + "path": ".github/workflows/test.yml", + "label": "CI test workflow", + "kind": "ci", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/python_compatibility.json", + "evidence_link": "python_compatibility.md", + "verification_command": "python3 scripts/yao.py python-compat ." + }, + { + "gate_key": "permission-gates", + "label": "权限批准", + "status": "warn", + "priority": "warning", + "summary": "补齐高权限能力的 reviewer、scope、reason、expires_at 和目标端 enforcement 说明。", + "why": "权限契约只有在批准人、有效期和目标端处置方式明确时，才能支撑 governed release。", + "source_fix": "security/permission_policy.json + security/permission_policy.md", + "source_refs": [ + { + "path": "security/permission_policy.json", + "label": "permission approvals", + "kind": "policy", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "security/permission_policy.md", + "label": "permission method", + "kind": "policy", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/security_trust_report.json + security/permission_policy.json", + "evidence_link": "../security/permission_policy.md", + "verification_command": "python3 scripts/trust_check.py ." + }, + { + "gate_key": "permission-runtime", + "label": "权限探针", + "status": "warn", + "priority": "warning", + "summary": "生成并修复目标包的 runtime permission probe 报告。", + "why": "目标端即使只能提供 metadata fallback，也必须明确 native enforcement 缺口、表示位置和 operator note。", + "source_fix": "dist/targets/*/adapter.json + reports/runtime_permission_probes.md", + "source_refs": [ + { + "path": "reports/runtime_permission_probes.md", + "label": "runtime permission probes", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "dist/targets/openai/adapter.json", + "label": "OpenAI adapter", + "kind": "package", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "dist/targets/claude/adapter.json", + "label": "Claude adapter", + "kind": "package", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "dist/targets/generic/adapter.json", + "label": "generic adapter", + "kind": "package", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/runtime_permission_probes.json", + "evidence_link": "runtime_permission_probes.md", + "verification_command": "python3 scripts/probe_runtime_permissions.py . --package-dir dist" + }, + { + "gate_key": "skill-atlas", + "label": "组合治理", + "status": "warn", + "priority": "warning", + "summary": "处理 portfolio 里的路由冲突、owner 缺口、stale skill 和重复能力。", + "why": "单个 Skill 质量很高仍可能在团队 skill library 中被相邻 Skill 冲突削弱。", + "source_fix": "reports/skill_atlas.html + skill_atlas/catalog.json", + "source_refs": [ + { + "path": "skill_atlas/catalog.json", + "label": "skill atlas catalog", + "kind": "atlas", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "skill_atlas/policy.json", + "label": "atlas scope policy", + "kind": "policy", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/skill_atlas.html", + "label": "skill atlas report", + "kind": "report", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/skill_atlas.json", + "evidence_link": "skill_atlas.html", + "verification_command": "python3 scripts/build_skill_atlas.py --workspace-root ." + }, + { + "gate_key": "operations-loop", + "label": "运营回路", + "status": "warn", + "priority": "warning", + "summary": "记录 metadata-only 使用事件，或明确当前 release 缺少真实使用信号。", + "why": "没有运营回路时，reviewer 无法判断采用率、误触发、坏输出和 review overdue 的真实影响。", + "source_fix": "reports/adoption_drift_report.md", + "source_refs": [ + { + "path": "reports/adoption_drift_report.md", + "label": "adoption drift report", + "kind": "report", + "line": 1, + "exists": true, + "link": "adoption_drift_report.md" + }, + { + "path": "references/telemetry-drift-method.md", + "label": "telemetry method", + "kind": "method", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/adoption_drift_report.json", + "evidence_link": "adoption_drift_report.md", + "verification_command": "python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted" + }, + { + "gate_key": "review-waivers", + "label": "人工批准", + "status": "warn", + "priority": "warning", + "summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "why": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "source_fix": "reports/review_waivers.md", + "source_refs": [ + { + "path": "reports/review_waivers.md", + "label": "waiver ledger", + "kind": "report", + "line": 1, + "exists": true, + "link": "review_waivers.md" + }, + { + "path": "references/review-waiver-method.md", + "label": "waiver method", + "kind": "method", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/review_waivers.json", + "evidence_link": "review_waivers.md", + "verification_command": "python3 scripts/render_review_waivers.py ." + }, + { + "gate_key": "release-notes", + "label": "发布路线", + "status": "warn", + "priority": "warning", + "summary": "确认 promotion、upgrade diff、breaking changes、migration guide 和 known limitations。", + "why": "发布说明不完整会让使用者无法判断升级风险和迁移动作。", + "source_fix": "reports/upgrade_check.md + docs/migration-v2.md", + "source_refs": [ + { + "path": "reports/upgrade_check.md", + "label": "upgrade check", + "kind": "report", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "docs/migration-v2.md", + "label": "migration guide", + "kind": "docs", + "line": null, + "exists": false, + "link": "" + }, + { + "path": "reports/promotion_decisions.md", + "label": "promotion decisions", + "kind": "report", + "line": null, + "exists": false, + "link": "" + } + ], + "evidence": "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", + "evidence_link": "/Users/laoyao/AI Coding/03-Development/Skills/yao-meta-skill/docs/migration-v2.md", + "verification_command": "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json" + } + ], + "evidence_paths": { + "skill_overview": "reports/skill-overview.html", + "review_viewer": "reports/review-viewer.html", + "output_eval": "reports/output_quality_scorecard.md", + "output_execution": "reports/output_execution_runs.md", + "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_kit": "reports/output_review_kit.md", + "output_review_kit_html": "reports/output_review_kit.html", + "output_review_decisions": "reports/output_review_decisions.json", + "output_review_adjudication": "reports/output_review_adjudication.md", + "benchmark_reproducibility": "reports/benchmark_reproducibility.md", + "skill_os2_coverage": "reports/skill_os2_coverage.md", + "runtime_conformance": "reports/conformance_matrix.md", + "trust_report": "reports/security_trust_report.md", + "python_compatibility": "reports/python_compatibility.md", + "architecture_maintainability": "reports/architecture_maintainability.md", + "permission_policy": "security/permission_policy.md", + "runtime_permissions": "reports/runtime_permission_probes.md", + "skill_atlas": "reports/skill_atlas.html", + "compiled_targets": "reports/compiled_targets.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "world_class_evidence_plan": "reports/world_class_evidence_plan.md", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", + "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_submission_review": "reports/world_class_submission_review.md", + "world_class_operator_runbook": "reports/world_class_operator_runbook.md", + "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", + "world_class_claim_guard": "reports/world_class_claim_guard.md", + "registry_audit": "reports/registry_audit.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "upgrade_check": "reports/upgrade_check.md", + "migration": "docs/migration-v2.md", + "skill_ir": "reports/skill-ir.json" + }, + "data": { + "overview": { + "skill_summary": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "display_name": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "maturity": "production", + "updated_at": "2026-06-15", + "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", + "audience": "Skill 作者、复用团队和后续 reviewer。", + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-ir.json", + "reports/compiled_targets.md", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", + "reports/review-studio.html", + "reports/skill-overview.html" + ], + "flow": [ + "输入材料", + "Skill 包体", + "可复用能力" + ] + }, + "scorecard": { + "completeness_score": { + "label": "完整度", + "score": 92, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 60, + "reasons": [ + "已生成 12 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。", + "证据不足：缺少 output_quality_scorecard.json, conformance_matrix.json, security_trust_report.json。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 85, + "reasons": [ + "SKILL.md 约 864 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 证据不足，重复执行逻辑可能仍靠人工。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 78, + "reasons": [ + "入口约 864 个词/字，references 约 565 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本处于可控区间。" + ] + } + }, + "capability_profile": { + "archetype": "production", + "task_family": "Creative generation", + "maturity": "production", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "principle_model": { + "nodes": [ + { + "title": "意图澄清", + "body": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "contract_boundary": { + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "已有原始资料，但还没有清晰输入、输出和后续迭代路径。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "quality_review": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。" + ], + "gaps": [ + "证据充分需要补强：已生成 12 / 20 类报告证据。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 85.0, + "highlights": [ + "Primary prompt task family: Creative generation.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 90/100." + ] + }, + "system_model": { + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (77/100).", + "Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + } + }, + "risk_governance": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 2, + "signal": "已生成 12 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 2, + "signal": "已生成 12 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 2, + "signal": "SKILL.md 约 864 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "table", + "report", + "brief" + ], + "score": 4, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "screenshot", + "visual", + "screen" + ], + "score": 3, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "citation", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "copy", + "content" + ], + "score": 2, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + }, + { + "key": "tutorial_quality", + "label": "Tutorial quality", + "matched_keywords": [ + "tutorial" + ], + "score": 1, + "risks": [ + "Generic section headings make the tutorial feel templated instead of fitted to the learner's task.", + "Steps may explain what to do without naming the exact check that proves the step worked.", + "Examples can become too abstract when the input material is concrete." + ], + "constraints": [ + "Write headings from the user's domain nouns and desired outcome, not from generic labels like Overview or Key Points.", + "Pair each major step with a visible success check or expected intermediate output.", + "Use one concrete worked example before adding broad principles." + ], + "self_repair": [ + "Replace generic H2/H3 headings with task-specific headings before final output.", + "Scan every numbered step for a missing verification cue." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "package_assets": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 2 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 0 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 2 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 34 + } + ], + "file_count": 42, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 2 + }, + { + "label": "scripts", + "value": 1 + }, + { + "label": "evals", + "value": 2 + }, + { + "label": "reports", + "value": 34 + } + ] + }, + "iteration_roadmap": { + "items": [ + { + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations." + }, + { + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint." + }, + { + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset." + } + ] + }, + "report_contract": { + "schema_version": "2.0", + "html_report": "reports/skill-overview.html", + "language": "zh-CN", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "layout": "kami-white-audit-v2", + "nav_labels": [ + "技能概述", + "总览指标", + "能力画像", + "原理结构", + "契约边界", + "质量评估", + "风险治理", + "包体资产", + "迭代路线" + ], + "nav_labels_en": [ + "Overview", + "Metrics", + "Profile", + "Principle", + "Contract", + "Quality", + "Risk", + "Assets", + "Roadmap" + ] + }, + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "display_name": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "logic_steps": [ + "读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。", + "对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。", + "如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。", + "用 `evals/trigger_cases.jsonl` 检查当前请求是否属于 GEO 内容简报场景。", + "用 `evals/output_cases.jsonl` 检查输出是否包含输入摘要、内容结构、证据要求、禁区和下一步动作。" + ], + "usage_steps": [ + "当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "Use this skill when the request matches: 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + ], + "package_map": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 2 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 0 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 2 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 34 + } + ], + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。" + ], + "scenarios": [ + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "已有原始资料，但还没有清晰输入、输出和后续迭代路径。" + ], + "trigger_contract": { + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "io_contract": { + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ] + }, + "principles": [ + { + "title": "意图澄清", + "body": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "roadmap": [ + { + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations." + }, + { + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint." + }, + { + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset." + } + ], + "cards": [], + "introduction": [ + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" + ], + "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "target_count": 5, + "trigger_samples": 1, + "output_eval_cases": 0 + }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, + "output_quality": {}, + "output_execution": { + "ok": false, + "summary": {}, + "runner": {}, + "failures": [] + }, + "output_blind_review": { + "summary": {}, + "seed": "", + "pair_count": 0, + "answer_key_separate": false + }, + "output_review_kit": { + "ok": false, + "summary": {}, + "artifacts": {}, + "failures": [] + }, + "output_review_adjudication": { + "ok": false, + "summary": {}, + "reviewer": "", + "reviewed_at": "", + "failures": [] + }, + "benchmark_reproducibility": { + "ok": false, + "summary": {}, + "commit": "", + "missing_artifacts": [], + "limitations": [] + }, + "runtime_conformance": {}, + "runtime_permissions": { + "ok": false, + "summary": {}, + "expected_capabilities": [], + "targets": [], + "failures": [] + }, + "trust_security": {}, + "skill_atlas": {}, + "registry_distribution": { + "ok": false, + "package": {}, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": false, + "summary": {}, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": false, + "summary": {}, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": false, + "summary": {}, + "upgrade_diff": {}, + "release_notes": [], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "no-data", + "event_types": {}, + "failure_types": {}, + "source_types": {}, + "command_counts": {} + }, + "next_iteration_candidates": [ + { + "signal": "no telemetry", + "recommendation": "Start with a small metadata-only sample before using telemetry for release decisions." + } + ], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 0, + "waiverable_open_count": 0, + "non_waivable_count": 0 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] + }, + "world_class_evidence_plan": { + "ok": false, + "summary": {}, + "tasks": [], + "source_audit": {} + }, + "world_class_evidence_ledger": { + "ok": false, + "summary": {}, + "entries": [], + "source_plan": {} + }, + "synthesis_highlights": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 85.0, + "highlights": [ + "Primary prompt task family: Creative generation.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 90/100." + ] + }, + "system_model": { + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (77/100).", + "Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + }, + "metadata": { + "canonical_format": "agent-skills", + "targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "maturity_tier": "production", + "skill_archetype": "production", + "updated_at": "2026-06-15" + } + }, + "intent_confidence": { + "score": 10, + "band": "low", + "gate_passed": false, + "strengths": [], + "gaps": [ + { + "key": "job_specificity", + "label": "Recurring job is still generic", + "reason": "The current job statement sounds more like a packaging goal than a concrete repeated task.", + "severity": "high" + }, + { + "key": "real_inputs", + "label": "Real inputs are missing", + "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", + "severity": "high" + }, + { + "key": "primary_output", + "label": "Primary output is missing", + "reason": "The package does not yet know what it must hand back.", + "severity": "high" + }, + { + "key": "exclusions", + "label": "Near-neighbor exclusions are missing", + "reason": "The route may blur into nearby requests without an exclusion list.", + "severity": "high" + }, + { + "key": "constraints", + "label": "Constraints are missing", + "reason": "The package does not yet know which tradeoffs matter most.", + "severity": "high" + }, + { + "key": "standards", + "label": "Quality bar is implied, not explicit", + "reason": "The first evaluation target is still underspecified.", + "severity": "medium" + } + ], + "follow_up_questions": [ + { + "slot": "job", + "question": "If you say it plainly, what concrete repeated task should this skill own every time?", + "why": "A skill needs a real recurring job, not only a generic packaging goal.", + "list": false, + "label": "Recurring job is still generic", + "severity": "high" + }, + { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": true, + "label": "Real inputs are missing", + "severity": "high" + }, + { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": false, + "label": "Primary output is missing", + "severity": "high" + } + ], + "anchor_sentence": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "context": { + "job": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "real_inputs": [], + "primary_output": "", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] + } + }, + "intent_dialogue": { + "skill_name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "focus": "trigger-and-output", + "opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.", + "reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.", + "conversation_path": [ + "Start with the user's own words, not package vocabulary.", + "Reflect the job, output, and non-goals back in one clean sentence.", + "Only then offer a tiny scaffold if it would help the user move faster." + ], + "opening_styles": [ + { + "label": "温柔陪伴型", + "best_when": "用户想法还散、还在试探，或者需要先被接住。", + "message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。" + }, + { + "label": "专业教练型", + "best_when": "用户目标比较明确，希望被高效带着走。", + "message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。" + }, + { + "label": "共创伙伴型", + "best_when": "用户已经有一些想法，希望一起打磨，不想被填表。", + "message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。" + } + ], + "optional_scaffold": [ + "The repeated job it should reliably handle", + "The real inputs people will hand to it", + "The useful output it should hand back", + "What it should clearly refuse" + ], + "questions": [ + { + "question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?", + "why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape." + }, + { + "question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?", + "why": "Input shape decides whether references, scripts, or templates are needed." + }, + { + "question": "What finished output should it hand back so the user can immediately keep moving?", + "why": "Outputs should drive the package structure before extra guidance is added." + }, + { + "question": "Which nearby requests should this skill politely refuse so the boundary stays clean?", + "why": "The exclusion list is the fastest route to better trigger quality." + }, + { + "question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?", + "why": "Constraints decide how much structure, packaging, and review the skill actually needs." + }, + { + "question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?", + "why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material." + }, + { + "question": "What repeated manual step should become a deterministic asset first?", + "why": "This usually reveals whether a script or reference should be created next." + } + ], + "output": { + "capability_sentence": "Geo Content Brief Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.", + "required_capture": [ + "recurring job", + "real inputs", + "required outputs", + "exclusions", + "constraints", + "reference preferences", + "first evaluation target" + ], + "recommended_first_gate": "trigger and boundary" + } + }, + "route_scorecard": {}, + "output_quality": {}, + "output_execution": {}, + "output_blind_review": {}, + "output_review_kit": {}, + "output_review_adjudication": {}, + "benchmark_reproducibility": {}, + "skill_os2_coverage": {}, + "compiled_targets": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15", + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "schema_version": "1.0", + "target": "openai", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "openai", + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "degradation_strategy": "metadata-adapter", + "target_runtime": { + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "target_transform": { + "target": "openai", + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "metadata-adapter", + "permission_representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "permission_enforcement": "metadata-only" + }, + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "claude", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "claude", + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "degradation_strategy": "neutral-source-plus-adapter", + "target_runtime": { + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Claude-compatible neutral source folder with adapter notes" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "target_transform": { + "target": "claude", + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source-plus-adapter", + "permission_representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "permission_enforcement": "metadata-fallback" + }, + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "generic", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "generic", + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills compatible neutral package" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "target_transform": { + "target": "generic", + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "targets/generic/adapter.json::target_permission_contract", + "native_surface": "Agent Skills compatible neutral package", + "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "agent-skills-compatible", + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills standard source tree" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "target_transform": { + "target": "agent-skills-compatible", + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "adapter metadata", + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "vscode", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "reports/skill-ir.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "version": "0.1.0", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ], + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + } + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill", + "counts": { + "trigger": 0, + "output": 0, + "adversarial": 0, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "vscode", + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "target_runtime": { + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "VS Code/Copilot Agent Skills project or user scope" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "missing-security-trust-report", + "source_available": false, + "declared_capabilities": [], + "review_required": false, + "capabilities": { + "network": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "file_write": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "subprocess": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": false, + "checked_count": 0, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 0, + "file_write_script_count": 0, + "subprocess_script_count": 0, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": false, + "declared_capabilities": [], + "capability_counts": { + "network": 0, + "file_write": 0, + "subprocess": 0, + "interactive": 0 + }, + "evidence": "missing-security-trust-report", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 0, + "scripts": 0, + "assets": 0, + "reports": 13 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 0, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [], + "review_required": false + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "target_transform": { + "target": "vscode", + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "permission_representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "permission_enforcement": "client-or-workspace-trust" + }, + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [], + "failures": [] + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "json": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.json", + "markdown": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/compiled_targets.md" + } + }, + "conformance": {}, + "runtime_permissions": {}, + "trust": {}, + "python_compatibility": {}, + "architecture_maintainability": {}, + "context_budget": {}, + "promotion": {}, + "atlas": {}, + "adoption_drift": { + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-15T09:03:27Z", + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "summary": { + "event_count": 0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "no-data", + "event_types": {}, + "failure_types": {}, + "source_types": {}, + "command_counts": {} + }, + "adoption_by_skill": [], + "next_iteration_candidates": [ + { + "signal": "no telemetry", + "recommendation": "Start with a small metadata-only sample before using telemetry for release decisions." + } + ], + "recent_events": [], + "failures": [], + "artifacts": { + "events_jsonl": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/telemetry_events.jsonl", + "json": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.json", + "markdown": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/adoption_drift_report.md" + } + }, + "review_waivers": { + "schema_version": "1.0", + "ok": true, + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "generated_at": "2026-06-15", + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 0, + "waiverable_open_count": 0, + "non_waivable_count": 0 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "waivers": [], + "waiver_candidates": [], + "failures": [], + "warnings": [], + "artifacts": { + "json": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.json", + "markdown": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.md" + } + }, + "review_annotations": { + "schema_version": "1.0", + "ok": true, + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "source": "reports/review_annotations_input.json", + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [], + "template_written": false, + "artifacts": { + "json": "reports/review_annotations.json", + "markdown": "reports/review_annotations.md" + } + }, + "world_class_evidence_ledger": {}, + "world_class_evidence_intake": {}, + "world_class_submission_review": {}, + "world_class_operator_runbook": {}, + "world_class_claim_guard": {}, + "registry": {}, + "package_verification": {}, + "install_simulation": {}, + "upgrade_check": {}, + "manifest": { + "name": "geo-content-brief-skill", + "version": "0.1.0", + "owner": "Yao Team", + "updated_at": "2026-06-15", + "status": "active", + "maturity_tier": "production", + "lifecycle_stage": "production-preview", + "context_budget_tier": "production", + "review_cadence": "per-release", + "skill_archetype": "production", + "target_platforms": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ], + "factory_components": [ + "references", + "scripts", + "reports" + ] + }, + "frontmatter": { + "name": "geo-content-brief-skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + }, + "interface": { + "interface": { + "display_name": "Geo Content Brief Skill", + "short_description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。", + "default_prompt": "当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。" + }, + "compatibility": { + "canonical_format": "agent-skills", + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "activation": { + "mode": "manual", + "paths": [ + "SKILL.md", + "references/brief-structure.md", + "evals/trigger_cases.jsonl" + ] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "degradation": { + "openai": "metadata-adapter", + "claude": "neutral-source-plus-adapter", + "generic": "neutral-source", + "vscode": "agent-skills-source-with-vscode-notes" + } + } + } + }, + "artifacts": { + "html": "reports/review-studio.html", + "json": "reports/review-studio.json" + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.html b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.html new file mode 100644 index 00000000..1b701dd9 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.html @@ -0,0 +1,376 @@ + + + + + + Geo Content Brief Skill Review Viewer + + + +
+
+

Review Viewer

+

Geo Content Brief Skill

+

将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。

+
+ maturity: production + archetype: production + format: agent-skills + updated: 2026-06-15 + intent confidence: 10 / 100 +
+
+ +
+

Architecture at a glance

+
Inputs
workflow, prompt, transcript, docs, or notes
Boundary
将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。
Logic
读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。; 对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。; 如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。
Usage
当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。; Use this skill when the request matches: 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。
Next
Review the top iteration directions before growing the package.
+
+ +
+
+

Core logic

+
  • 读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。
  • 对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。
  • 如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。
  • 用 `evals/trigger_cases.jsonl` 检查当前请求是否属于 GEO 内容简报场景。
  • 用 `evals/output_cases.jsonl` 检查输出是否包含输入摘要、内容结构、证据要求、禁区和下一步动作。
+
+
+

How to use it

+
  • 当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。
  • Use this skill when the request matches: 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。
+
+
+ +
+
+

Intent questions

+
  • If this skill worked beautifully, what recurring job would it reliably handle for the user every time?
    This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape.
  • When someone reaches for this skill in the real world, what materials will they actually hand to it?
    Input shape decides whether references, scripts, or templates are needed.
  • What finished output should it hand back so the user can immediately keep moving?
    Outputs should drive the package structure before extra guidance is added.
  • Which nearby requests should this skill politely refuse so the boundary stays clean?
    The exclusion list is the fastest route to better trigger quality.
  • What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?
    Constraints decide how much structure, packaging, and review the skill actually needs.
+
+
+

Why this package is strong

+
  • 触发面保持精简，并锚定在 frontmatter description。
  • 已生成 Skill IR，核心语义可先于平台打包被审查和迁移。
  • 已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。
  • 已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。
  • 已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。
  • 已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。
+
+
+ +
+
+

Borrow plan

+
  • No external benchmark objects recorded yet. Add 2 to 5 references before deepening the package.
+
+
+

Compare view

+

No baseline comparison has been recorded for this package yet.

+
+
+ +
+

Variant diff studio

+

No description optimization compare payload is attached yet.

+
+ +
+
+

Evidence readiness

+

Readiness score: 57/100

+
  • Intent clarity · needs review
    10/100 intent confidence.
  • Benchmark coverage · needs evidence
    0 GitHub benchmark repositories attached.
  • Pattern gate · ready
    3 accepted, 0 deferred using threshold 2/4.
  • Conflict handling · decision needed
    The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.
  • Output risk profile · ready
    5 output risk families attached.
  • Artifact design profile · ready
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Prompt quality profile · ready
    85.0/100 prompt-facing quality score.
+
+
+

Honest boundary check

+
    +
  • Are the known limits visible before the package deepens?
    • +
  • Does the evidence support the borrowed patterns?
    • +
  • Should uncertainty become a clarification question instead of more structure?
    • +
+
+
+ +
+
+

Output risk profile

+
  • Markdown readability
    Tables can render as dense grids with weak hierarchy or poor mobile readability.; Long bullets can make the output look complete while hiding the actual decision logic.
  • Screenshot and visual capture
    Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.; Missing screenshots can cause the skill to invent visual references instead of declaring the gap.
  • Citation and footnote clutter
    Footnote markers or dense citation notes can interrupt the reading flow.; Evidence can be over-attached to obvious statements and under-attached to risky claims.
+
+
+

Self-repair checks

+
  • Preview whether each table still reads well when columns are narrow.
  • Convert any table with paragraph-length cells into bullets or cards.
  • Check that every screenshot reference points to a real provided or generated asset.
  • Reword any visual instruction that depends on an unseen screen state.
  • Remove decorative citations that do not support a material claim.
+
+
+ +
+
+

Artifact design profile

+

Design system: metric editorial

+
  • Report or brief
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Review viewer
    Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.
  • Dashboard or metrics page
    Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.
+
+
+

Visual quality gates

+
  • Keep the first screen useful without requiring the reader to parse every detail.
  • Use tables only for comparisons; move explanations below the table.
  • Keep source notes readable without flooding the body with markers.
  • Make differences visible instead of hiding them in prose.
  • Separate author-facing recommendations from reviewer-only evidence.
+
+
+ +
+
+

Prompt quality profile

+

Relevance: prompt-heavy · score 85.0 / 100 · complexity complex

+
  • Completeness · 80 / 100
    Name missing inputs, outputs, constraints, or success standards before deepening the package.
  • Clarity · 90 / 100
    Replace broad verbs with observable actions and define what done means.
  • Consistency · 90 / 100
    Check that role, task, format, exclusions, and examples do not contradict each other.
  • Practicality · 95 / 100
    Add runnable steps, examples, or verification cues instead of abstract advice.
  • Specificity · 70 / 100
    Anchor wording in the user's audience, domain nouns, and target outcome.
+
+
+

RTF to skill mapping

+
  • Role
    Use a taste-aware creator role with clear audience, tone, and originality boundaries.
  • Task
    Generate variants, explain selection logic, and preserve the user's distinctive constraints.
  • Format
    Return options with rationale, selection criteria, and refinement paths.
+
+
+ +
+
+

Reference coach

+

No GitHub benchmark scan has been attached to this package yet.

+
+
+

Decide before you deepen

+
    +
  • Choose one pattern to borrow on purpose, not three at once.
    • +
  • State one thing this skill will not inherit from the benchmark objects.
    • +
  • Only deepen the package after that choice is visible in the boundary or execution flow.
    • +
+
+
+ +
+
+

Reference synthesis

+

Official skill anatomy and context discipline

Borrow now

  • Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.

Avoid

  • Do not let packaging or platform concerns swallow the core job boundary.

Hypothesis-test-learn loop

Borrow now

  • Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.

Avoid

  • Do not create experimental overhead that exceeds the skill's real risk tier.

Boundary-first design

Borrow now

  • Borrow the discipline of defining what the skill should not own before growing the package.

Avoid

  • Do not expand execution assets until route boundaries stay clean.
+
+
+

Borrow now

+
  • Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.
  • Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.
  • Borrow the discipline of defining what the skill should not own before growing the package.
+

Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.

+
+
+ +
+

Top three next moves

+

Tighten trigger and exclusions

The package needs clearer near-neighbor exclusions before it grows.

  • Add 3 to 5 should-trigger and should-not-trigger examples.
  • Refine the frontmatter description to name the recurring job and non-goals.
  • Run a first trigger evaluation pass before expanding the package.
Unlocks: Cleaner routing and fewer accidental activations.

Add the first execution asset

The package is still mostly prose. Add one asset that removes repeated manual work.

  • Move stable procedural guidance into references if users will need it repeatedly.
  • Create one deterministic helper script if a repeated step can be executed instead of described.
  • Keep the main SKILL.md compact and route-oriented.
Unlocks: Stronger execution quality without bloating the entrypoint.

Promote from scaffold to production-ready

The first version exists; the next gain usually comes from adding the smallest useful gates.

  • Decide whether this skill is personal, team-reused, or library-grade.
  • Add only the gates that match that risk level.
  • Record lifecycle metadata and review cadence once reuse becomes real.
Unlocks: A clearer path from exploratory package to maintained asset.
+
+ +
+
+

Recent feedback

+
  • No lightweight feedback captured yet. Use `yao.py feedback` to record quick review notes.
+
+
+

Promotion status

+

No promotion summary is attached to this package yet.

+
+
+ +
+
+

Package map

+
  • SKILL.md — Skill entrypoint
  • README.md — Human-readable usage guide
  • agents/interface.yaml — Neutral interface metadata
  • manifest.json — Lifecycle and portability metadata
  • references — Extended guidance and reusable notes
  • scripts — Deterministic helpers or local tooling
  • evals — Trigger and quality checks
  • reports — Generated evidence and overview artifacts
+
+
+

First-pass review frame

+
    +
  • Does the trigger stay narrow enough for the intended job?
    • +
  • Does the archetype match the real reuse level?
    • +
  • Are we adding structure faster than we are adding reliability?
    • +
  • Should the next step be trigger tightening, execution assets, or portability hardening?
    • +
+
+
+ +
+
+

Authoring discipline

+
    +
  • Name unresolved assumptions before deepening the package.
    • +
  • Keep the package no larger than the recurring job requires.
    • +
  • Touch only files that directly support the requested change.
    • +
  • Tie every meaningful new artifact to a check or reviewer note.
    • +
+
+
+

Reviewer guardrails

+
    +
  • Block speculative features that are not backed by real workflow variation.
    • +
  • Move unverifiable ideas into next-step candidates instead of baseline structure.
    • +
  • Reject decorative folders, reports, or governance that do not reduce risk.
    • +
  • Ask for one high-leverage clarification when job, output, or exclusion is still fuzzy.
    • +
+
+
+
+ + diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.json new file mode 100644 index 00000000..d956d9a5 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review-viewer.json @@ -0,0 +1,1921 @@ +{ + "overview": { + "skill_summary": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "display_name": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "maturity": "production", + "updated_at": "2026-06-15", + "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", + "audience": "Skill 作者、复用团队和后续 reviewer。", + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-ir.json", + "reports/compiled_targets.md", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", + "reports/review-studio.html", + "reports/skill-overview.html" + ], + "flow": [ + "输入材料", + "Skill 包体", + "可复用能力" + ] + }, + "scorecard": { + "completeness_score": { + "label": "完整度", + "score": 92, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 60, + "reasons": [ + "已生成 12 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。", + "证据不足：缺少 output_quality_scorecard.json, conformance_matrix.json, security_trust_report.json。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 85, + "reasons": [ + "SKILL.md 约 864 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 证据不足，重复执行逻辑可能仍靠人工。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 78, + "reasons": [ + "入口约 864 个词/字，references 约 565 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本处于可控区间。" + ] + } + }, + "capability_profile": { + "archetype": "production", + "task_family": "Creative generation", + "maturity": "production", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "principle_model": { + "nodes": [ + { + "title": "意图澄清", + "body": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "contract_boundary": { + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "已有原始资料，但还没有清晰输入、输出和后续迭代路径。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "quality_review": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。" + ], + "gaps": [ + "证据充分需要补强：已生成 12 / 20 类报告证据。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 85.0, + "highlights": [ + "Primary prompt task family: Creative generation.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 90/100." + ] + }, + "system_model": { + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (77/100).", + "Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + } + }, + "risk_governance": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 2, + "signal": "已生成 12 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 2, + "signal": "已生成 12 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 2, + "signal": "SKILL.md 约 864 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "table", + "report", + "brief" + ], + "score": 4, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "screenshot", + "visual", + "screen" + ], + "score": 3, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "citation", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "copy", + "content" + ], + "score": 2, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + }, + { + "key": "tutorial_quality", + "label": "Tutorial quality", + "matched_keywords": [ + "tutorial" + ], + "score": 1, + "risks": [ + "Generic section headings make the tutorial feel templated instead of fitted to the learner's task.", + "Steps may explain what to do without naming the exact check that proves the step worked.", + "Examples can become too abstract when the input material is concrete." + ], + "constraints": [ + "Write headings from the user's domain nouns and desired outcome, not from generic labels like Overview or Key Points.", + "Pair each major step with a visible success check or expected intermediate output.", + "Use one concrete worked example before adding broad principles." + ], + "self_repair": [ + "Replace generic H2/H3 headings with task-specific headings before final output.", + "Scan every numbered step for a missing verification cue." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "package_assets": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 2 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 0 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 2 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 34 + } + ], + "file_count": 42, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 2 + }, + { + "label": "scripts", + "value": 1 + }, + { + "label": "evals", + "value": 2 + }, + { + "label": "reports", + "value": 34 + } + ] + }, + "iteration_roadmap": { + "items": [ + { + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations." + }, + { + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint." + }, + { + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset." + } + ] + }, + "report_contract": { + "schema_version": "2.0", + "html_report": "reports/skill-overview.html", + "language": "zh-CN", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "layout": "kami-white-audit-v2", + "nav_labels": [ + "技能概述", + "总览指标", + "能力画像", + "原理结构", + "契约边界", + "质量评估", + "风险治理", + "包体资产", + "迭代路线" + ], + "nav_labels_en": [ + "Overview", + "Metrics", + "Profile", + "Principle", + "Contract", + "Quality", + "Risk", + "Assets", + "Roadmap" + ] + }, + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "display_name": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "logic_steps": [ + "读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。", + "对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。", + "如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。", + "用 `evals/trigger_cases.jsonl` 检查当前请求是否属于 GEO 内容简报场景。", + "用 `evals/output_cases.jsonl` 检查输出是否包含输入摘要、内容结构、证据要求、禁区和下一步动作。" + ], + "usage_steps": [ + "当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "Use this skill when the request matches: 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + ], + "package_map": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 2 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 0 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 2 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 34 + } + ], + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。" + ], + "scenarios": [ + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "已有原始资料，但还没有清晰输入、输出和后续迭代路径。" + ], + "trigger_contract": { + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "io_contract": { + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ] + }, + "principles": [ + { + "title": "意图澄清", + "body": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "roadmap": [ + { + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations." + }, + { + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint." + }, + { + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset." + } + ], + "cards": [], + "introduction": [ + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" + ], + "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "target_count": 5, + "trigger_samples": 1, + "output_eval_cases": 0 + }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, + "output_quality": {}, + "output_execution": { + "ok": false, + "summary": {}, + "runner": {}, + "failures": [] + }, + "output_blind_review": { + "summary": {}, + "seed": "", + "pair_count": 0, + "answer_key_separate": false + }, + "output_review_kit": { + "ok": false, + "summary": {}, + "artifacts": {}, + "failures": [] + }, + "output_review_adjudication": { + "ok": false, + "summary": {}, + "reviewer": "", + "reviewed_at": "", + "failures": [] + }, + "benchmark_reproducibility": { + "ok": false, + "summary": {}, + "commit": "", + "missing_artifacts": [], + "limitations": [] + }, + "runtime_conformance": {}, + "runtime_permissions": { + "ok": false, + "summary": {}, + "expected_capabilities": [], + "targets": [], + "failures": [] + }, + "trust_security": {}, + "skill_atlas": {}, + "registry_distribution": { + "ok": false, + "package": {}, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": false, + "summary": {}, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": false, + "summary": {}, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": false, + "summary": {}, + "upgrade_diff": {}, + "release_notes": [], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "no-data", + "event_types": {}, + "failure_types": {}, + "source_types": {}, + "command_counts": {} + }, + "next_iteration_candidates": [ + { + "signal": "no telemetry", + "recommendation": "Start with a small metadata-only sample before using telemetry for release decisions." + } + ], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 0, + "waiverable_open_count": 0, + "non_waivable_count": 0 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] + }, + "world_class_evidence_plan": { + "ok": false, + "summary": {}, + "tasks": [], + "source_audit": {} + }, + "world_class_evidence_ledger": { + "ok": false, + "summary": {}, + "entries": [], + "source_plan": {} + }, + "synthesis_highlights": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 85.0, + "highlights": [ + "Primary prompt task family: Creative generation.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 90/100." + ] + }, + "system_model": { + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (77/100).", + "Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + }, + "metadata": { + "canonical_format": "agent-skills", + "targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "maturity_tier": "production", + "skill_archetype": "production", + "updated_at": "2026-06-15" + } + }, + "intent_confidence": { + "score": 10, + "band": "low", + "gate_passed": false, + "strengths": [], + "gaps": [ + { + "key": "job_specificity", + "label": "Recurring job is still generic", + "reason": "The current job statement sounds more like a packaging goal than a concrete repeated task.", + "severity": "high" + }, + { + "key": "real_inputs", + "label": "Real inputs are missing", + "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", + "severity": "high" + }, + { + "key": "primary_output", + "label": "Primary output is missing", + "reason": "The package does not yet know what it must hand back.", + "severity": "high" + }, + { + "key": "exclusions", + "label": "Near-neighbor exclusions are missing", + "reason": "The route may blur into nearby requests without an exclusion list.", + "severity": "high" + }, + { + "key": "constraints", + "label": "Constraints are missing", + "reason": "The package does not yet know which tradeoffs matter most.", + "severity": "high" + }, + { + "key": "standards", + "label": "Quality bar is implied, not explicit", + "reason": "The first evaluation target is still underspecified.", + "severity": "medium" + } + ], + "follow_up_questions": [ + { + "slot": "job", + "question": "If you say it plainly, what concrete repeated task should this skill own every time?", + "why": "A skill needs a real recurring job, not only a generic packaging goal.", + "list": false, + "label": "Recurring job is still generic", + "severity": "high" + }, + { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": true, + "label": "Real inputs are missing", + "severity": "high" + }, + { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": false, + "label": "Primary output is missing", + "severity": "high" + } + ], + "anchor_sentence": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "context": { + "job": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "real_inputs": [], + "primary_output": "", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] + } + }, + "intent": { + "skill_name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "focus": "trigger-and-output", + "opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.", + "reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.", + "conversation_path": [ + "Start with the user's own words, not package vocabulary.", + "Reflect the job, output, and non-goals back in one clean sentence.", + "Only then offer a tiny scaffold if it would help the user move faster." + ], + "opening_styles": [ + { + "label": "温柔陪伴型", + "best_when": "用户想法还散、还在试探，或者需要先被接住。", + "message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。" + }, + { + "label": "专业教练型", + "best_when": "用户目标比较明确，希望被高效带着走。", + "message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。" + }, + { + "label": "共创伙伴型", + "best_when": "用户已经有一些想法，希望一起打磨，不想被填表。", + "message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。" + } + ], + "optional_scaffold": [ + "The repeated job it should reliably handle", + "The real inputs people will hand to it", + "The useful output it should hand back", + "What it should clearly refuse" + ], + "questions": [ + { + "question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?", + "why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape." + }, + { + "question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?", + "why": "Input shape decides whether references, scripts, or templates are needed." + }, + { + "question": "What finished output should it hand back so the user can immediately keep moving?", + "why": "Outputs should drive the package structure before extra guidance is added." + }, + { + "question": "Which nearby requests should this skill politely refuse so the boundary stays clean?", + "why": "The exclusion list is the fastest route to better trigger quality." + }, + { + "question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?", + "why": "Constraints decide how much structure, packaging, and review the skill actually needs." + }, + { + "question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?", + "why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material." + }, + { + "question": "What repeated manual step should become a deterministic asset first?", + "why": "This usually reveals whether a script or reference should be created next." + } + ], + "output": { + "capability_sentence": "Geo Content Brief Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.", + "required_capture": [ + "recurring job", + "real inputs", + "required outputs", + "exclusions", + "constraints", + "reference preferences", + "first evaluation target" + ], + "recommended_first_gate": "trigger and boundary" + } + }, + "reference": { + "skill_name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "scan_focus": [ + { + "label": "Execution pattern", + "reason": "There is deterministic logic in scripts, so compare how strong references separate prose from executable steps." + }, + { + "label": "Portability pattern", + "reason": "The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source." + }, + { + "label": "Method pattern", + "reason": "Use the core job description as the anchor for comparison: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + } + ], + "external_references": [], + "user_references": [], + "local_constraints": [], + "borrow_plan": [ + "External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability.", + "User references second: use them to understand taste, standards, and directional preferences without copying source phrasing.", + "Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints.", + "Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language.", + "Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability." + ], + "priority_rule": "External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility.", + "non_goals": [ + "Do not copy source prose or branding into the new skill.", + "Do not import gates that cost more context than they save.", + "Do not use benchmark scanning to justify scope creep.", + "Do not let local historical habits outrank stronger public benchmarks." + ] + }, + "iteration": { + "summary": { + "skill_name": "geo-content-brief-skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "maturity_tier": "scaffold", + "selection_rule": "Pick the three smallest next steps that increase reliability more than they increase context cost.", + "recommended_now": "Tighten trigger and exclusions", + "recommended_now_why": "The package needs clearer near-neighbor exclusions before it grows.", + "defer_for_now": "Promote from scaffold to production-ready" + }, + "directions": [ + { + "priority": 1, + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations.", + "do_now": "Do this first.", + "wait_on": "Wait to add broader structure until this move clearly improves reliability." + }, + { + "priority": 2, + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint.", + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." + }, + { + "priority": 3, + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset.", + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." + } + ] + }, + "feedback": {}, + "baseline": {}, + "compare": {}, + "promotion": {}, + "benchmark": {}, + "reference_synthesis": { + "skill_name": "geo-content-brief-skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "intent_confidence": { + "score": 10, + "band": "low", + "gate_passed": false + }, + "github_benchmarks": [], + "source_tracks": [ + { + "source_type": "official", + "name": "Official skill anatomy and context discipline", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "why_relevant": "This track matches: general fit." + }, + { + "source_type": "research", + "name": "Hypothesis-test-learn loop", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "why_relevant": "This track matches: general fit." + }, + { + "source_type": "principles", + "name": "Boundary-first design", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "why_relevant": "This track matches: general fit." + } + ], + "synthesis": { + "borrow_now": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "avoid_now": [ + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not create experimental overhead that exceeds the skill's real risk tier.", + "Do not expand execution assets until route boundaries stay clean." + ], + "quality_risers": [ + "Use GitHub repositories for concrete package and workflow patterns.", + "Use curated official or commercial tracks for entrypoint and operator ergonomics.", + "Use research tracks to justify the smallest evaluation loop that still catches regressions.", + "Use principle tracks to keep the package small, boundary-aware, and outcome-driven." + ], + "pattern_gate": { + "threshold": 2, + "source_count": 3, + "accepted": [ + { + "name": "Official skill anatomy and context discipline", + "source_type": "official", + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + }, + { + "name": "Hypothesis-test-learn loop", + "source_type": "research", + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + }, + { + "name": "Boundary-first design", + "source_type": "principles", + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + } + ], + "deferred": [], + "summary": "3 accepted, 0 deferred using threshold 2/4." + }, + "conflicts": [ + { + "key": "lightweight_vs_governance", + "summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_preference": "lightweight or speed-first", + "benchmark_pressure": "governance or evaluation-heavy patterns" + } + ], + "recommendation": { + "summary": "Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.", + "borrow_now": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed." + ], + "avoid_for_now": [ + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not create experimental overhead that exceeds the skill's real risk tier." + ], + "why": "There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_decision_required": true + }, + "visibility": { + "mode": "explicit", + "user_decision_required": true, + "reasons": [ + "intent_uncertain", + "design_conflict" + ], + "user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.", + "reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers." + }, + "decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.", + "source_mix": { + "github_benchmarks": 0, + "curated_tracks": 3, + "user_references": 0 + } + } + }, + "output_risk": { + "skill_name": "geo-content-brief-skill", + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "table", + "report", + "brief" + ], + "score": 4, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "screenshot", + "visual", + "screen" + ], + "score": 3, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "citation", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "copy", + "content" + ], + "score": 2, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + }, + { + "key": "tutorial_quality", + "label": "Tutorial quality", + "matched_keywords": [ + "tutorial" + ], + "score": 1, + "risks": [ + "Generic section headings make the tutorial feel templated instead of fitted to the learner's task.", + "Steps may explain what to do without naming the exact check that proves the step worked.", + "Examples can become too abstract when the input material is concrete." + ], + "constraints": [ + "Write headings from the user's domain nouns and desired outcome, not from generic labels like Overview or Key Points.", + "Pair each major step with a visible success check or expected intermediate output.", + "Use one concrete worked example before adding broad principles." + ], + "self_repair": [ + "Replace generic H2/H3 headings with task-specific headings before final output.", + "Scan every numbered step for a missing verification cue." + ] + } + ], + "top_risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims." + ], + "output_constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability." + ], + "self_repair_checks": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards.", + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state.", + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ], + "reviewer_note": "Use this report before deepening the package and again before approving example outputs." + }, + "artifact_design": { + "skill_name": "geo-content-brief-skill", + "design_system": "metric editorial", + "primary_artifact": { + "key": "report", + "label": "Report or brief", + "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "matched_keywords": [ + "report", + "brief", + "简报" + ] + }, + "artifact_families": [ + { + "key": "report", + "label": "Report or brief", + "score": 3, + "matched_keywords": [ + "report", + "brief", + "简报" + ], + "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." + }, + { + "key": "review_viewer", + "label": "Review viewer", + "score": 2, + "matched_keywords": [ + "review", + "审查" + ], + "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." + }, + { + "key": "dashboard", + "label": "Dashboard or metrics page", + "score": 2, + "matched_keywords": [ + "dashboard", + "table" + ], + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." + }, + { + "key": "tutorial", + "label": "Tutorial or guide", + "score": 1, + "matched_keywords": [ + "tutorial" + ], + "direction": "Progressive instructional layout with domain-specific section names, short success checks, and examples close to the user's real input." + }, + { + "key": "visual_capture", + "label": "Screenshot or visual evidence", + "score": 1, + "matched_keywords": [ + "screenshot" + ], + "direction": "Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect." + } + ], + "layout_patterns": [ + "thesis", + "evidence blocks", + "decision table", + "risks", + "next actions", + "summary", + "variant comparison", + "evidence" + ], + "design_tokens": { + "type": [ + "Use a distinctive display face or serif for major claims when the artifact is editorial.", + "Use a restrained sans for dense body text and technical details.", + "Use mono only for metadata, paths, commands, labels, and evidence tags." + ], + "color": [ + "Choose colors from the artifact's domain, brand, or evidence mood.", + "Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.", + "Keep accent color limited to decisions, active states, risk, or section anchors." + ], + "spacing": [ + "Prefer clear grid rhythm over floating decorative cards.", + "Increase whitespace around decisions and shrink whitespace around supporting metadata.", + "Split dense content instead of shrinking type or adding scroll traps." + ], + "components": [ + "Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.", + "Avoid cards inside cards.", + "Keep reviewer-only detail visible but visually quieter than user-facing guidance." + ] + }, + "quality_gates": [ + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers.", + "Make differences visible instead of hiding them in prose.", + "Separate author-facing recommendations from reviewer-only evidence.", + "Surface conflicts clearly and keep routine benchmark synthesis quiet.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each." + ], + "anti_patterns": [ + "Do not copy Kami's fixed parchment background as a default.", + "Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.", + "Do not let Markdown tables become the default shape for every comparison or explanation.", + "Do not turn reviewer evidence into user-facing clutter.", + "Do not invent screenshots, citations, charts, or UI states." + ], + "reviewer_note": "Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered." + }, + "prompt_quality": { + "skill_name": "geo-content-brief-skill", + "relevance": "prompt-heavy", + "primary_task_family": { + "key": "creative_generation", + "label": "Creative generation", + "matched_keywords": [ + "copy", + "content", + "内容" + ] + }, + "task_families": [ + { + "key": "creative_generation", + "label": "Creative generation", + "score": 3, + "matched_keywords": [ + "copy", + "content", + "内容" + ], + "role_guidance": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.", + "task_guidance": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.", + "format_guidance": "Return options with rationale, selection criteria, and refinement paths." + }, + { + "key": "execution_operation", + "label": "Execution operation", + "score": 3, + "matched_keywords": [ + "workflow", + "execute", + "执行" + ], + "role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", + "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", + "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." + }, + { + "key": "prompt_engineering", + "label": "Prompt engineering", + "score": 3, + "matched_keywords": [ + "prompt", + "role", + "format" + ], + "role_guidance": "Use a prompt engineer role only when role design materially improves execution.", + "task_guidance": "Map Role, Task, and Format into skill behavior rather than copying a large prompt template.", + "format_guidance": "Return a compact prompt contract plus tests, quality matrix, and usage notes." + }, + { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "score": 2, + "matched_keywords": [ + "dialogue", + "访谈" + ], + "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." + } + ], + "complexity": { + "band": "complex", + "score": 6, + "reason": "multiple inputs, constraints, or task families require tradeoff handling" + }, + "need_model": { + "explicit_need": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.", + "scenario": "not yet explicit", + "user_level": "infer from examples and standards; ask only if it changes output depth", + "success_standard": "usable output with clear validation cues" + }, + "rtf_to_skill": { + "role": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.", + "task": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.", + "format": "Return options with rationale, selection criteria, and refinement paths." + }, + "quality_matrix": [ + { + "key": "completeness", + "label": "Completeness", + "score": 80, + "matched_signals": [ + "input", + "output", + "constraint", + "example", + "约束" + ], + "repair": "Name missing inputs, outputs, constraints, or success standards before deepening the package." + }, + { + "key": "clarity", + "label": "Clarity", + "score": 90, + "matched_signals": [ + "clear", + "specific" + ], + "repair": "Replace broad verbs with observable actions and define what done means." + }, + { + "key": "consistency", + "label": "Consistency", + "score": 90, + "matched_signals": [ + "boundary", + "边界" + ], + "repair": "Check that role, task, format, exclusions, and examples do not contradict each other." + }, + { + "key": "practicality", + "label": "Practicality", + "score": 95, + "matched_signals": [ + "execute", + "use", + "workflow", + "执行" + ], + "repair": "Add runnable steps, examples, or verification cues instead of abstract advice." + }, + { + "key": "specificity", + "label": "Specificity", + "score": 70, + "matched_signals": [], + "repair": "Anchor wording in the user's audience, domain nouns, and target outcome." + } + ], + "overall_quality_score": 85.0, + "self_repair_checks": [ + "Check explicit need, implicit need, scenario, user level, and success standard before deepening.", + "Map Role, Task, and Format into skill behavior, not decorative prompt labels.", + "Ask one focused clarification only when missing information changes the package boundary.", + "Add tests or examples for prompt-heavy behavior before treating it as reusable.", + "Keep prompt methodology in references and reports instead of bloating SKILL.md." + ], + "reviewer_note": "Use this profile when the package depends on prompt behavior, role design, output contracts, or conversation quality." + }, + "evidence_readiness": { + "score": 57, + "checks": [ + { + "label": "Intent clarity", + "status": "needs review", + "detail": "10/100 intent confidence." + }, + { + "label": "Benchmark coverage", + "status": "needs evidence", + "detail": "0 GitHub benchmark repositories attached." + }, + { + "label": "Pattern gate", + "status": "ready", + "detail": "3 accepted, 0 deferred using threshold 2/4." + }, + { + "label": "Conflict handling", + "status": "decision needed", + "detail": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure." + }, + { + "label": "Output risk profile", + "status": "ready", + "detail": "5 output risk families attached." + }, + { + "label": "Artifact design profile", + "status": "ready", + "detail": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." + }, + { + "label": "Prompt quality profile", + "status": "ready", + "detail": "85.0/100 prompt-facing quality score." + } + ], + "reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery." + } +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.json new file mode 100644 index 00000000..36bd1473 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.json @@ -0,0 +1,24 @@ +{ + "schema_version": "1.0", + "ok": true, + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "source": "reports/review_annotations_input.json", + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [], + "template_written": false, + "artifacts": { + "json": "reports/review_annotations.json", + "markdown": "reports/review_annotations.md" + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.md new file mode 100644 index 00000000..68230eb6 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_annotations.md @@ -0,0 +1,19 @@ +# Review Annotations + +This report renders reviewer annotations attached to Review Studio gates and source/report paths. + +- Annotations: `0` +- Open: `0` +- Resolved: `0` +- Deferred: `0` +- Open blockers: `0` +- Open warnings: `0` + +No reviewer annotations recorded yet. + + +## Review Rule + +- Use annotations for reviewer comments tied to a gate or source line. +- Use waivers only for explicit acceptance of warning-level release risk. +- Open blocker annotations should block a release decision until resolved or deferred with rationale. diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.json new file mode 100644 index 00000000..bce91bb1 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.json @@ -0,0 +1,44 @@ +{ + "schema_version": "1.0", + "ok": true, + "skill_dir": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill", + "generated_at": "2026-06-15", + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 0, + "waiverable_open_count": 0, + "non_waivable_count": 0 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "waivers": [], + "waiver_candidates": [], + "failures": [], + "warnings": [], + "artifacts": { + "json": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.json", + "markdown": ".previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.md" + } +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.md new file mode 100644 index 00000000..0e34c96f --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/review_waivers.md @@ -0,0 +1,34 @@ +# Review Waivers + +- OK: `True` +- Waivers: `0` +- Active: `0` +- Expired: `0` +- Invalid: `0` +- Covered gates: `none` +- Waiver candidates: `0` +- Open waiverable candidates: `0` +- Non-waivable boundaries: `0` + +## Policy + +- Blocker waivers allowed: `False` +- Minimum reason chars: `20` +- Expiry is required for every waiver. +- World-class evidence completion cannot be waived; it can only be proven by accepted ledger evidence. + +## Waivers + +- None + +## Candidate Actions + +- None + +## Failures + +- None + +## Warnings + +- None diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-ir.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-ir.json new file mode 100644 index 00000000..771b377f --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-ir.json @@ -0,0 +1,93 @@ +{ + "schema_version": "2.0.0", + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "job_to_be_done": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "trigger_surface": { + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "should_trigger": [ + "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + ], + "should_not_trigger": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [] + }, + "workflow": { + "steps": [ + "Understand the request.", + "Execute the main task.", + "Validate the result." + ], + "decision_points": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [], + "scripts": [], + "assets": [], + "reports": [ + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md" + ] + }, + "eval_plan": { + "trigger": [], + "output": [], + "adversarial": [], + "baseline": "without_skill" + }, + "risk": { + "output_risk": "high", + "execution_risk": "low", + "trust_boundary": "personal" + }, + "governance": { + "owner": "Yao Team", + "maturity": "scaffold", + "review_cadence": "per-release", + "review_due": "" + }, + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ], + "source_files": [ + "SKILL.md", + "manifest.json", + "agents/interface.yaml", + "reports/intent-context.json", + "reports/output-risk-profile.json", + "reports/system-model.json" + ] +} diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.html b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.html new file mode 100644 index 00000000..45f5c88a --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.html @@ -0,0 +1,986 @@ + + + + + + geo-content-brief-skill Skill 生成审计报告 + + + + +
+ +
+
+ Skill audit + +
+
+
+
+
+
+
+
+

YAO Skill 生成审计报告YAO Skill Generation Audit

+

geo-content-brief-skill

+

技能审计报告Geo Content Brief Skill Audit Report

+

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.

+
技能名称：geo-content-brief-skillSkill name: geo-content-brief-skill成熟度：生产Maturity: production格式：Agent SkillsFormat: agent-skills更新时间：2026-06-15Updated: 2026-06-15
+
openaiopenaiclaudeclaudegenericgenericvscodevscode
+
+ +
+
完整度Completeness92SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth60已生成 12 / 20 类报告证据。Generated 12 / 20 evidence report types.
上下文成本Context cost78入口约 864 个词/字，references 约 565 个词/字。Entrypoint is about 864 words/characters; references are about 565.
+
+ +
+
+
+

技能概述Overview

+

先用一屏说明这个 Skill 是什么、给谁用、交付什么。A first-screen explanation of what this skill is, who it serves, and what it delivers.

+
+
+
+

作用定位Role

+
  • 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.htmlDeliverables: SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html
+
+
交付流程Delivery Flow输入材料Input materialSkill 包体Skill package可复用能力Reusable capability
交付流程把用户输入、生成的包体和可复用能力放在一条线上。The delivery flow places user input, generated package, and reusable capability on one path.
+
+
+
+ +
+
+
+

总览指标Metrics

+

分数来自本地文件和 reports 证据，缺失时明确标为证据不足。Scores are derived from package files and reports; missing inputs are shown as evidence gaps.

+
+
+

指标判读Reading

+

先看雷达图判断能力短板，再看每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

+
+
+
+
+
+
评分雷达Rating Radar完整度Completeness触发清晰Trigger clarity证据充分Evidence depth可维护性Maintainability可迁移性Portability
评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。The radar chart compares completeness, trigger clarity, evidence, maintainability, and portability.
+
+

成熟度条Maturity Bar

+
  1. 稳定Stable完整度Completeness92SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 关注Watch证据充分Evidence depth60已生成 12 / 20 类报告证据。Generated 12 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability85SKILL.md 约 864 个词/字。SKILL.md is about 864 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 可用Usable上下文成本Context cost78入口约 864 个词/字，references 约 565 个词/字。Entrypoint is about 864 words/characters; references are about 565.
+
+
+
完整度Completeness92
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth60
  • 已生成 12 / 20 类报告证据。Generated 12 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability85
  • SKILL.md 约 864 个词/字。SKILL.md is about 864 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost78
  • 入口约 864 个词/字，references 约 565 个词/字。Entrypoint is about 864 words/characters; references are about 565.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本处于可控区间。Context cost is within a controlled range.
+
+
+
+ +
+
+
+

能力画像Capability

+

判断这个 Skill 在能力地图中的位置和复用范围。Places this skill on a capability map and clarifies reuse scope.

+
+
+
能力矩阵Capability Matrix执行确定性Execution certainty知识密度Knowledge densityCreative generationCreative generation
能力矩阵说明这个 Skill 更偏知识密集还是执行确定。The capability matrix shows whether the skill leans toward knowledge density or execution certainty.
+
+

画像摘要Profile

+
  • 能力类型：Creative generationCapability type: Creative generation
  • 成熟度：productionMaturity: production
  • 触发强度：手动触发 + description 路由Trigger strength: Manual activation plus description-based routing
  • 复用范围：跨平台Reuse scope: Cross-platform
+
+
+
+
+ +
+
+
+

原理结构Principle

+

说明入口、参考、脚本、评估和报告如何组成一个稳定闭环。Explains how entrypoint, references, scripts, evals, and reports form a stable loop.

+
+
+
分层结构Layered Structure入口层Entrypoint layer参考层Reference layer脚本层Script layer评估层Evaluation layer报告层Report layer
分层结构展示入口、参考、脚本、评估和报告如何各司其职。The layered structure shows how entrypoint, references, scripts, evals, and reports each carry a distinct role.
+
+
+

执行流程Execution Flow

+
  1. 读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
  2. 对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
  3. 如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
  4. 用 `evals/trigger_cases.jsonl` 检查当前请求是否属于 GEO 内容简报场景。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
  5. 用 `evals/output_cases.jsonl` 检查输出是否包含输入摘要、内容结构、证据要求、禁区和下一步动作。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
+
+
+

调用方式How To Use

+
  1. 当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.
  2. Use this skill when the request matches: 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。Use this skill when the request matches the frontmatter description.
+
+
+
+
+
+ +
+
+
+

契约边界Contract

+

把触发、输入、输出和排除场景放在同一屏。Keeps trigger, inputs, outputs, and exclusions on the same screen.

+
+
+
+

触发描述Trigger

+

将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording.

+

输入材料Inputs

+
  • 用户提供的工作流、提示词、文档、记录或散乱笔记User-provided workflows, prompts, documents, records, or rough notes.
  • 期望沉淀的复用场景、排除项、约束和质量标准The reusable scenario, exclusions, constraints, and quality standards to capture.
+
+
+

输出结果Outputs

+
  • 可路由的 SKILL.mdA routeable SKILL.md.
  • agents/interface.yaml 元数据agents/interface.yaml metadata.
  • 必要的 references、scripts、evals、reports 证据Necessary references, scripts, evals, and reports evidence.
  • 结构化 Skill 目录，共 8 类关键资产。Structured skill directory with 8 key asset groups.
+

不应触发Should Not Trigger

+
  • 只需要一次性回答、没有复用价值的临时请求。One-off requests that do not need reusable skill behavior.
  • 要求直接执行相邻任务，而不是沉淀或使用这个 Skill。Requests to perform an adjacent task directly rather than create or use this skill.
  • 缺少必要事实且用户不允许澄清的场景。Cases that lack required facts and do not allow clarification.
+
+
+
+
+ +
+
+
+

质量评估Quality

+

展示强项、缺口和建议，避免只给分不解释。Shows strengths, gaps, and recommendations instead of scores without explanation.

+
+
+ + + +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap证据充分需要补强：已生成 12 / 20 类报告证据。Evidence depth needs improvement: Generated 12 / 20 evidence report types.纳入下一轮修复Fix next
+
+
+

执行证据Execution Evidence

+
  • 尚未生成输出执行证据报告。The output execution evidence report has not been generated yet.
+
+
+

盲评审定Blind Adjudication

+
  • 尚未生成盲评审定报告。The blind review adjudication report has not been generated yet.
+
+
+
+
+

评审原则Review Rule

+
  • 先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。Record the reviewer's A/B choice before opening the answer key and calculating agreement.
  • 缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.
+
+
+

运行原则Run Rule

+
  • recorded fixture 只能证明可复现样本，不等同于模型执行。A recorded fixture proves reproducible samples only; it is not model execution.
  • 只有 provider runner 返回 model metadata 时才计入 model-executed。Only provider runners that return model metadata count as model-executed.
+
+
+
+
+
+ +
+
+
+

风险治理Risk

+

提前暴露误触发、漂移、证据不足和迁移风险。Surfaces trigger, drift, evidence, and portability risks before the package grows.

+
+
+
风险热力Risk Heatmap000020210发生概率Probability影响程度Impact
风险热力图用影响程度和发生概率标出当前治理重点。The risk heatmap marks governance priorities by impact and probability.
+
+ + + +
风险Risk信号Signal应对Response
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 12 / 20 类报告证据。Generated 12 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 12 / 20 类报告证据。Generated 12 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 864 个词/字。SKILL.md is about 864 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
+
+
+

世界证据World Evidence

未生成 world-class ledger；当前报告不会宣称世界级完成。No world-class ledger was generated; this report does not claim world-class completion.

证据待补Evidence pending
待补证据Pending0仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks0 / 0通过数 / 总检查数。Passed checks / total checks.

尚未生成 world-class ledger；这里只保留反过度承诺提示。No world-class ledger has been generated; this panel keeps the anti-overclaim guard visible.

+
+
+ +
+
+
+

包体资产Assets

+

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

+
+
+
资产分布Asset Distribution42项42 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+ + + +
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
+
+
+
+ +
+
+
+

迭代路线Roadmap

+

把下一步升级收束为少数高价值动作。Keeps next iteration moves focused and actionable.

+
+
+
迭代时间Iteration Timeline下一步 1Next 1Tighten trigger a…Tighten trigger a…下一步 2Next 2Add the first exe…Add the first exe…下一步 3Next 3Promote from scaf…Promote from scaf…
迭代时间线把下一步升级收束成少数可执行动作。The iteration timeline narrows the next upgrade into a few executable moves.
+
下一步 1Next 1

收紧触发与排除边界Tighten trigger and exclusions

在继续扩展前，需要先把相邻但不应触发的场景说清楚。The package needs clearer near-neighbor exclusions before it grows.

  • 增加 3 到 5 个应触发和不应触发的例子。Add 3 to 5 should-trigger and should-not-trigger examples.
  • 精炼 frontmatter description，明确重复任务和非目标。Refine the frontmatter description to name the recurring job and non-goals.
  • 扩展包体前先跑一轮触发评估。Run a first trigger evaluation pass before expanding the package.

路由更清晰，误触发更少。Cleaner routing and fewer accidental activations.

下一步 2Next 2

补上第一个执行资产Add the first execution asset

当前包体仍偏文本说明，应先增加一个能减少重复人工操作的资产。The package is still mostly prose. Add one asset that removes repeated manual work.

  • 如果用户会反复使用某段流程说明，把它沉淀到 references。Move stable procedural guidance into references if users will need it repeatedly.
  • 如果某个重复步骤可以执行而不是描述，就沉淀成一个确定性 helper script。Create one deterministic helper script if a repeated step can be executed instead of described.
  • 保持主 SKILL.md 简洁，并围绕路由与入口组织。Keep the main SKILL.md compact and route-oriented.

在不膨胀入口文件的前提下提升执行质量。Stronger execution quality without bloating the entrypoint.

下一步 3Next 3

从脚手架推进到生产可用Promote from scaffold to production-ready

第一版已经存在，下一步收益通常来自补上最小但有效的质量门禁。The first version exists; the next gain usually comes from adding the smallest useful gates.

  • 判断这个 Skill 是个人使用、团队复用，还是库级基础能力。Decide whether this skill is personal, team-reused, or library-grade.
  • 只添加与风险等级匹配的质量门禁。Add only the gates that match that risk level.
  • 一旦进入真实复用，就记录生命周期元数据和评审节奏。Record lifecycle metadata and review cadence once reuse becomes real.

更清晰地从探索性包体走向可维护资产。A clearer path from exploratory package to maintained asset.

+
+
+
+
+ + + diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.json new file mode 100644 index 00000000..968c3b5b --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/skill-overview.json @@ -0,0 +1,1060 @@ +{ + "skill_summary": { + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "display_name": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "maturity": "production", + "updated_at": "2026-06-15", + "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", + "audience": "Skill 作者、复用团队和后续 reviewer。", + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-ir.json", + "reports/compiled_targets.md", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", + "reports/review-studio.html", + "reports/skill-interpretation.html", + "reports/skill-overview.html" + ], + "flow": [ + "输入材料", + "Skill 包体", + "可复用能力" + ] + }, + "scorecard": { + "completeness_score": { + "label": "完整度", + "score": 92, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 60, + "reasons": [ + "已生成 12 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。", + "证据不足：缺少 output_quality_scorecard.json, conformance_matrix.json, security_trust_report.json。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 85, + "reasons": [ + "SKILL.md 约 864 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 证据不足，重复执行逻辑可能仍靠人工。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 78, + "reasons": [ + "入口约 864 个词/字，references 约 565 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本处于可控区间。" + ] + } + }, + "capability_profile": { + "archetype": "production", + "task_family": "Creative generation", + "maturity": "production", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "principle_model": { + "nodes": [ + { + "title": "意图澄清", + "body": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "contract_boundary": { + "trigger": { + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "已有原始资料，但还没有清晰输入、输出和后续迭代路径。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "quality_review": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。" + ], + "gaps": [ + "证据充分需要补强：已生成 12 / 20 类报告证据。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 85.0, + "highlights": [ + "Primary prompt task family: Creative generation.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 90/100." + ] + }, + "system_model": { + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (77/100).", + "Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + } + }, + "risk_governance": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 2, + "signal": "已生成 12 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 2, + "signal": "已生成 12 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 2, + "signal": "SKILL.md 约 864 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "table", + "report", + "brief" + ], + "score": 4, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "screenshot", + "visual", + "screen" + ], + "score": 3, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "citation", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "copy", + "content" + ], + "score": 2, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + }, + { + "key": "tutorial_quality", + "label": "Tutorial quality", + "matched_keywords": [ + "tutorial" + ], + "score": 1, + "risks": [ + "Generic section headings make the tutorial feel templated instead of fitted to the learner's task.", + "Steps may explain what to do without naming the exact check that proves the step worked.", + "Examples can become too abstract when the input material is concrete." + ], + "constraints": [ + "Write headings from the user's domain nouns and desired outcome, not from generic labels like Overview or Key Points.", + "Pair each major step with a visible success check or expected intermediate output.", + "Use one concrete worked example before adding broad principles." + ], + "self_repair": [ + "Replace generic H2/H3 headings with task-specific headings before final output.", + "Scan every numbered step for a missing verification cue." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "world_class_readiness": { + "ready": false, + "decision": "not-generated", + "entry_count": 0, + "pending_count": 0, + "accepted_count": 0, + "external_pending_count": 0, + "human_pending_count": 0, + "source_check_count": 0, + "source_pass_count": 0, + "conclusion_zh": "未生成 world-class ledger；当前报告不会宣称世界级完成。", + "conclusion_en": "No world-class ledger was generated; this report does not claim world-class completion.", + "entries": [] + }, + "package_assets": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 2 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 0 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 2 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 34 + } + ], + "file_count": 42, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 2 + }, + { + "label": "scripts", + "value": 1 + }, + { + "label": "evals", + "value": 2 + }, + { + "label": "reports", + "value": 34 + } + ] + }, + "iteration_roadmap": { + "items": [ + { + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations." + }, + { + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint." + }, + { + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset." + } + ] + }, + "report_contract": { + "schema_version": "2.0", + "html_report": "reports/skill-overview.html", + "language": "zh-CN", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "layout": "kami-white-audit-v2", + "nav_labels": [ + "技能概述", + "总览指标", + "能力画像", + "原理结构", + "契约边界", + "质量评估", + "风险治理", + "包体资产", + "迭代路线" + ], + "nav_labels_en": [ + "Overview", + "Metrics", + "Profile", + "Principle", + "Contract", + "Quality", + "Risk", + "Assets", + "Roadmap" + ] + }, + "name": "geo-content-brief-skill", + "title": "Geo Content Brief Skill", + "display_name": "Geo Content Brief Skill", + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "logic_steps": [ + "读取用户输入，识别目标受众、内容渠道、关键词、品牌边界、禁止表达和必须覆盖的信息。", + "对照 `references/brief-structure.md` 生成简报骨架：目标、洞察、角度、关键词、结构、证据、风险和验收标准。", + "如果缺少核心输入，先提出最多三个澄清问题，不用假设补全用户没有提供的事实。", + "用 `evals/trigger_cases.jsonl` 检查当前请求是否属于 GEO 内容简报场景。", + "用 `evals/output_cases.jsonl` 检查输出是否包含输入摘要、内容结构、证据要求、禁区和下一步动作。" + ], + "usage_steps": [ + "当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "Use this skill when the request matches: 将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。" + ], + "package_map": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 2 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 0 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 2 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 34 + } + ], + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。" + ], + "scenarios": [ + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：当你需要把访谈纪要、关键词、竞品线索和渠道限制整理成中文 GEO 内容简报时，使用 $geo-content-brief-skill。", + "已有原始资料，但还没有清晰输入、输出和后续迭代路径。" + ], + "trigger_contract": { + "description": "将 GEO 内容访谈、关键词笔记、竞品线索和渠道约束整理成可执行的中文内容简报。用于团队复用、选题规划、内容生产交接和质量评审；不用于直接代写完整长文或替代品牌策略判断。", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "io_contract": { + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ] + }, + "principles": [ + { + "title": "意图澄清", + "body": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。" + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "roadmap": [ + { + "title": "Tighten trigger and exclusions", + "why": "The package needs clearer near-neighbor exclusions before it grows.", + "actions": [ + "Add 3 to 5 should-trigger and should-not-trigger examples.", + "Refine the frontmatter description to name the recurring job and non-goals.", + "Run a first trigger evaluation pass before expanding the package." + ], + "unlocks": "Cleaner routing and fewer accidental activations." + }, + { + "title": "Add the first execution asset", + "why": "The package is still mostly prose. Add one asset that removes repeated manual work.", + "actions": [ + "Move stable procedural guidance into references if users will need it repeatedly.", + "Create one deterministic helper script if a repeated step can be executed instead of described.", + "Keep the main SKILL.md compact and route-oriented." + ], + "unlocks": "Stronger execution quality without bloating the entrypoint." + }, + { + "title": "Promote from scaffold to production-ready", + "why": "The first version exists; the next gain usually comes from adding the smallest useful gates.", + "actions": [ + "Decide whether this skill is personal, team-reused, or library-grade.", + "Add only the gates that match that risk level.", + "Record lifecycle metadata and review cadence once reuse becomes real." + ], + "unlocks": "A clearer path from exploratory package to maintained asset." + } + ], + "cards": [], + "introduction": [ + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" + ], + "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "source_path": "reports/skill-ir.json", + "target_count": 5, + "trigger_samples": 1, + "output_eval_cases": 0 + }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, + "output_quality": {}, + "output_execution": { + "ok": false, + "summary": {}, + "runner": {}, + "failures": [] + }, + "output_blind_review": { + "summary": {}, + "seed": "", + "pair_count": 0, + "answer_key_separate": false + }, + "output_review_kit": { + "ok": false, + "summary": {}, + "artifacts": {}, + "failures": [] + }, + "output_review_adjudication": { + "ok": false, + "summary": {}, + "reviewer": "", + "reviewed_at": "", + "failures": [] + }, + "benchmark_reproducibility": { + "ok": false, + "summary": {}, + "commit": "", + "missing_artifacts": [], + "limitations": [] + }, + "runtime_conformance": {}, + "runtime_permissions": { + "ok": false, + "summary": {}, + "expected_capabilities": [], + "targets": [], + "failures": [] + }, + "trust_security": {}, + "skill_atlas": {}, + "registry_distribution": { + "ok": false, + "package": {}, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": false, + "summary": {}, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": false, + "summary": {}, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": false, + "summary": {}, + "upgrade_diff": {}, + "release_notes": [], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 0, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "no-data", + "event_types": {}, + "failure_types": {}, + "source_types": {}, + "command_counts": {} + }, + "next_iteration_candidates": [ + { + "signal": "no telemetry", + "recommendation": "Start with a small metadata-only sample before using telemetry for release decisions." + } + ], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 0, + "waiverable_open_count": 0, + "non_waivable_count": 0 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] + }, + "world_class_evidence_plan": { + "ok": false, + "summary": {}, + "tasks": [], + "source_audit": {} + }, + "world_class_evidence_ledger": { + "ok": false, + "summary": {}, + "entries": [], + "source_plan": {} + }, + "synthesis_highlights": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 85.0, + "highlights": [ + "Primary prompt task family: Creative generation.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 90/100." + ] + }, + "system_model": { + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (77/100).", + "Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + }, + "metadata": { + "canonical_format": "agent-skills", + "targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "maturity_tier": "production", + "skill_archetype": "production", + "updated_at": "2026-06-15" + } +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.json b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.json new file mode 100644 index 00000000..5a2417a8 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.json @@ -0,0 +1,166 @@ +{ + "skill_name": "geo-content-brief-skill", + "description": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "systems_doctrine": "Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight.", + "stability": { + "score": 77, + "band": "stable-first-pass" + }, + "boundary_map": { + "owned_job": "将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。", + "input_boundary": [ + "user-provided workflow notes, prompts, docs, or examples" + ], + "output_boundary": "a reusable skill output", + "non_goals": [ + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "constraints": [], + "standards": [], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "maturity_assumption": "scaffold" + }, + "feedback_loops": [ + { + "name": "Intent boundary loop", + "signal": "Intent confidence score is 10/100.", + "response": "Ask only the highest-leverage clarification before adding package weight.", + "evidence": "reports/intent-confidence.md and reports/intent-dialogue.md" + }, + { + "name": "Reference synthesis loop", + "signal": "Benchmark patterns are useful only after they are abstracted into borrow and avoid guidance.", + "response": "Borrow one pattern at a time and keep the rest as reviewer-visible evidence.", + "evidence": "reports/reference-synthesis.md", + "current_patterns": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package.", + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not create experimental overhead that exceeds the skill's real risk tier." + ] + }, + { + "name": "Output quality loop", + "signal": "Generated output may fail in recurring domain-specific ways.", + "response": "Apply predicted output-risk families as self-repair checks before final output.", + "evidence": "reports/output-risk-profile.md", + "current_risk_families": [ + "Markdown readability", + "Screenshot and visual capture", + "Citation and footnote clutter", + "Tone and specificity", + "Tutorial quality" + ] + }, + { + "name": "Reviewer feedback loop", + "signal": "Human review catches drift that static checks miss.", + "response": "Capture lightweight feedback and turn repeated findings into gates or references.", + "evidence": "reports/review-viewer.html and feedback records" + }, + { + "name": "Lifecycle loop", + "signal": "As reuse grows, the skill needs stronger gates, ownership, and regression evidence.", + "response": "Promote only when the next gate improves reliability more than context cost.", + "evidence": "manifest.json, reports/iteration-directions.md, and governance checks" + } + ], + "drift_watch": [ + { + "name": "Trigger drift", + "watch_signal": "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "countermeasure": "Add near-neighbor exclusions and route evals before expanding workflow steps.", + "cadence": "per trigger or description change" + }, + { + "name": "Output drift", + "watch_signal": "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "countermeasure": "Refresh output-risk and artifact-design profiles, then add one self-repair check.", + "cadence": "after the first 3-5 real uses", + "risk_families": [ + "Markdown readability", + "Screenshot and visual capture", + "Citation and footnote clutter", + "Tone and specificity", + "Tutorial quality" + ] + }, + { + "name": "Reference drift", + "watch_signal": "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "countermeasure": "Re-run reference synthesis and keep only patterns that improve the current boundary.", + "cadence": "per material benchmark or product assumption change" + }, + { + "name": "Governance drift", + "watch_signal": "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal.", + "countermeasure": "Promote maturity tier and add reviewer-visible lifecycle evidence.", + "cadence": "when reuse becomes real" + } + ], + "failure_pattern_map": [ + { + "family": "Boundary failure", + "symptom": "The skill handles nearby requests that were never part of the recurring job.", + "repair": "Narrow the description and add explicit non-goals before adding more execution steps." + }, + { + "family": "Feedback gap", + "symptom": "The skill has rules but no signal telling authors which rule should change after use.", + "repair": "Turn repeated reviewer feedback into one eval, one reference note, or one self-repair check." + }, + { + "family": "Output degradation", + "symptom": "The result is structurally correct but generic, cluttered, or weakly matched to the user's domain.", + "repair": "Use output-risk families as pre-final checks.", + "current_risk_families": [ + "Markdown readability", + "Screenshot and visual capture", + "Citation and footnote clutter", + "Tone and specificity", + "Tutorial quality" + ] + }, + { + "family": "Prompt-behavior mismatch", + "symptom": "The role, task, and format are copied from a prompt instead of becoming stable skill behavior.", + "repair": "Convert reusable role/task/format assumptions into workflow, reports, or references.", + "watch_axes": [ + "Specificity" + ] + } + ], + "leverage_points": [ + { + "rank": 1, + "point": "Clarify the real job boundary", + "why": "Intent uncertainty creates downstream trigger, output, and governance errors.", + "move": "Ask one focused question and update intent context before adding assets." + }, + { + "rank": 2, + "point": "Tune the frontmatter description", + "why": "The description is the highest-leverage routing surface.", + "move": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + }, + { + "rank": 3, + "point": "Install output self-repair checks", + "why": "The likely failure families are: Markdown readability, Screenshot and visual capture, Citation and footnote clutter.", + "move": "Add only the checks that prevent recurring output mistakes." + }, + { + "rank": 4, + "point": "Borrow one pattern, not a whole product", + "why": "External references improve quality when reduced to structure, not copied as surface style.", + "move": "Start from: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts." + } + ], + "reviewer_rule": "Reviewer should ask whether the skill's structure will keep producing the desired behavior after repeated real use." +} \ No newline at end of file diff --git a/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.md b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.md new file mode 100644 index 00000000..1b59b0a4 --- /dev/null +++ b/.previews/yao-meta-skill-v2-report-demo/geo-content-brief-skill/reports/system-model.md @@ -0,0 +1,156 @@ +# System Model + +Skill: `geo-content-brief-skill` + +- Stability score: `77/100` +- Stability band: `stable-first-pass` +- Doctrine: Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight. + +## System Boundary Map + +- Owned job: 将 GEO 内容访谈、关键词笔记和渠道约束整理成可执行的中文内容简报 Skill，适用于团队复用、触发边界审查和后续质量评估。 +- Output boundary: a reusable skill output +- Maturity assumption: `scaffold` +- Input boundary: + - user-provided workflow notes, prompts, docs, or examples +- Non-goals: + - one-off adjacent requests that do not match the recurring job + - private local material that was not intentionally included +- Human judgment boundary: + - Ask one focused clarification when the real job, output, or exclusion boundary is unclear. + - Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints. + - Do not silently broaden the skill into adjacent jobs just because the examples are nearby. + +## Feedback Loops + +### Intent boundary loop + +- Signal: Intent confidence score is 10/100. +- Response: Ask only the highest-leverage clarification before adding package weight. +- Evidence: reports/intent-confidence.md and reports/intent-dialogue.md + +### Reference synthesis loop + +- Signal: Benchmark patterns are useful only after they are abstracted into borrow and avoid guidance. +- Response: Borrow one pattern at a time and keep the rest as reviewer-visible evidence. +- Evidence: reports/reference-synthesis.md +- Current patterns: + - Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. + - Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. + - Borrow the discipline of defining what the skill should not own before growing the package. + - Do not let packaging or platform concerns swallow the core job boundary. + - Do not create experimental overhead that exceeds the skill's real risk tier. + +### Output quality loop + +- Signal: Generated output may fail in recurring domain-specific ways. +- Response: Apply predicted output-risk families as self-repair checks before final output. +- Evidence: reports/output-risk-profile.md +- Current risk families: + - Markdown readability + - Screenshot and visual capture + - Citation and footnote clutter + - Tone and specificity + - Tutorial quality + +### Reviewer feedback loop + +- Signal: Human review catches drift that static checks miss. +- Response: Capture lightweight feedback and turn repeated findings into gates or references. +- Evidence: reports/review-viewer.html and feedback records + +### Lifecycle loop + +- Signal: As reuse grows, the skill needs stronger gates, ownership, and regression evidence. +- Response: Promote only when the next gate improves reliability more than context cost. +- Evidence: manifest.json, reports/iteration-directions.md, and governance checks + +## Delay And Drift Watch + +### Trigger drift + +- Watch signal: Users start invoking the skill for adjacent one-off or explanation-only requests. +- Countermeasure: Add near-neighbor exclusions and route evals before expanding workflow steps. +- Cadence: per trigger or description change + +### Output drift + +- Watch signal: Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain. +- Countermeasure: Refresh output-risk and artifact-design profiles, then add one self-repair check. +- Cadence: after the first 3-5 real uses +- Risk families: + - Markdown readability + - Screenshot and visual capture + - Citation and footnote clutter + - Tone and specificity + - Tutorial quality + +### Reference drift + +- Watch signal: Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff. +- Countermeasure: Re-run reference synthesis and keep only patterns that improve the current boundary. +- Cadence: per material benchmark or product assumption change + +### Governance drift + +- Watch signal: Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal. +- Countermeasure: Promote maturity tier and add reviewer-visible lifecycle evidence. +- Cadence: when reuse becomes real + +## Failure Pattern Map + +### Boundary failure + +- Symptom: The skill handles nearby requests that were never part of the recurring job. +- Repair: Narrow the description and add explicit non-goals before adding more execution steps. + +### Feedback gap + +- Symptom: The skill has rules but no signal telling authors which rule should change after use. +- Repair: Turn repeated reviewer feedback into one eval, one reference note, or one self-repair check. + +### Output degradation + +- Symptom: The result is structurally correct but generic, cluttered, or weakly matched to the user's domain. +- Repair: Use output-risk families as pre-final checks. +- Current Risk Families: + - Markdown readability + - Screenshot and visual capture + - Citation and footnote clutter + - Tone and specificity + - Tutorial quality + +### Prompt-behavior mismatch + +- Symptom: The role, task, and format are copied from a prompt instead of becoming stable skill behavior. +- Repair: Convert reusable role/task/format assumptions into workflow, reports, or references. +- Watch Axes: + - Specificity + +## Highest Leverage Moves + +### 1. Clarify the real job boundary + +- Why: Intent uncertainty creates downstream trigger, output, and governance errors. +- Move: Ask one focused question and update intent context before adding assets. + +### 2. Tune the frontmatter description + +- Why: The description is the highest-leverage routing surface. +- Move: Name the recurring job, expected input, output, and strongest non-goal in compact language. + +### 3. Install output self-repair checks + +- Why: The likely failure families are: Markdown readability, Screenshot and visual capture, Citation and footnote clutter. +- Move: Add only the checks that prevent recurring output mistakes. + +### 4. Borrow one pattern, not a whole product + +- Why: External references improve quality when reduced to structure, not copied as surface style. +- Move: Start from: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. + +## Reviewer Use + +- Reviewer should ask whether the skill's structure will keep producing the desired behavior after repeated real use. +- Prefer changing the system boundary, feedback loop, or leverage point before adding more prose. +- If a problem repeats, convert it into a named failure pattern and one regression check. diff --git a/AGENTS.md b/AGENTS.md index 47d8fee4..b7511deb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,18 +34,60 @@ After source changes that affect scripts, package contents, trust evidence, Revi GENERATED_AT="${GENERATED_AT:-$(date +%F)}" python3 scripts/run_output_execution.py --runner-command '["python3","scripts/local_output_eval_runner.py"]' python3 scripts/compile_skill.py . --generated-at "$GENERATED_AT" -python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --output-dir dist --zip +python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip python3 scripts/simulate_install.py . --package-dir dist --install-root dist/install-simulation --output-json reports/install_simulation.json --output-md reports/install_simulation.md --generated-at "$GENERATED_AT" python3 scripts/trust_check.py . --output-json reports/security_trust_report.json --output-md reports/security_trust_report.md python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT" python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at "$GENERATED_AT" python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT" python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at "$GENERATED_AT" +python3 scripts/render_adoption_drift_report.py . --generated-at "$GENERATED_AT" +python3 scripts/render_architecture_maintainability.py . --generated-at "$GENERATED_AT" +python3 scripts/python_compat_check.py . --generated-at "$GENERATED_AT" +python3 scripts/probe_runtime_permissions.py . --package-dir dist +python3 scripts/render_review_waivers.py . --generated-at "$GENERATED_AT" +python3 scripts/render_review_annotations.py . +python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json --today "$GENERATED_AT" +python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT" +python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT" +python3 scripts/render_daily_skillops_report.py . --generated-at "$GENERATED_AT" +python3 scripts/render_weekly_curator_report.py . --generated-at "$GENERATED_AT" +python3 scripts/render_skill_os2_audit.py . --generated-at "$GENERATED_AT" +python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT" +python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT" +python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . +python3 scripts/render_skill_interpretation.py . +python3 scripts/render_review_viewer.py . +python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT" python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json +python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT" ``` -Clean test-only scratch directories after verification with `rm -rf tests/tmp_*`. Do not clean unrelated untracked files. +For final release evidence, commit source and generated package evidence first, then run the clean-lock reports from a clean worktree: + +```bash +python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT" +python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT" +python3 scripts/render_daily_skillops_report.py . --generated-at "$GENERATED_AT" +python3 scripts/render_weekly_curator_report.py . --generated-at "$GENERATED_AT" +python3 scripts/render_skill_overview.py . +python3 scripts/render_skill_interpretation.py . +python3 scripts/render_review_viewer.py . +python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT" +python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json +python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT" +``` + +If `reports/benchmark_reproducibility.json` reports `release_lock_ready: false`, do not commit that benchmark as release evidence. Restore the transient dirty-lock reports, commit the source/generated evidence that caused the dirty state, and regenerate the clean-lock reports on the resulting clean tree. + +Local sync into `~/.agents/skills.disabled/yao-meta-skill` or `~/.agents/skills/yao-meta-skill` must keep the install preflight enabled unless the user explicitly requests a diagnostic bypass. `make sync-local-install` and `make sync-active-install` rebuild the package first, then `scripts/sync_local_install.py` refuses to copy files when install simulation or installer permission enforcement fails. + +Clean test-only scratch directories after verification with `find tests -maxdepth 1 \( -name 'tmp' -o -name 'tmp_*' \) -type d -exec rm -rf {} +`. Do not clean unrelated untracked files. ## Boundaries @@ -60,11 +102,28 @@ Clean test-only scratch directories after verification with `rm -rf tests/tmp_*` - `scripts/yao.py`: unified CLI orchestration. Keep command behavior stable; move pure config and side-effect-free helpers into small internal modules. - `scripts/render_skill_overview.py`: v2 bilingual skill overview report. Preserve `reports/skill-overview.html` / `.json`, `body data-report-lang="zh-CN"`, default Simplified Chinese, English switch, and inline-chart/no-external-dependency behavior. +- `scripts/skill_report_layout.py` plus `assets/skill-overview.css` / `assets/skill-overview.js`: overview report layout contract and inline assets. Keep the generated report single-file at render time, but keep long CSS/JS source in `assets/` instead of embedding it inside Python. - `scripts/render_review_studio.py`: Review Studio gate orchestration. Keep gate scoring, evidence links, and action generation separate from layout helpers. -- `scripts/review_studio_layout.py`: Review Studio static layout and CSS contract. +- `scripts/review_studio_layout.py` plus `assets/review-studio.css`: Review Studio static layout and CSS contract. Keep the generated report single-file at render time, but keep long CSS source in `assets/` instead of embedding it inside Python. - `scripts/review_studio_formatting.py`: Review Studio dictionary-to-panel formatting and Chinese metric labels. +- `scripts/review_studio_gates.py`: Review Studio gate evaluation, release decision scoring, and gate status labels. +- `scripts/render_skill_os2_audit.py`: requirement-by-requirement Skill OS 2.0 completion audit. Keep local evidence, human-required gaps, and external-required gaps separate so reports do not overclaim world-class readiness. +- `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete. +- `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence. +- `scripts/render_world_class_evidence_intake.py`: intake validator for external and human world-class evidence packets. Real submissions must reference concrete local aggregate artifacts with matching SHA-256 digests; templates may stay hash-free and must not count as evidence. +- `scripts/world_class_evidence_contract.py`: shared intake contract and artifact-integrity validator. Keep ledger, intake, and submission review aligned so source evidence cannot be accepted without a valid real submission and matching artifact SHA-256 checks. +- `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state. +- `scripts/render_world_class_operator_runbook.py`: operator-facing world-class evidence runbook. Keep it as coordination guidance only; it must not accept evidence or flip world-class readiness. +- `scripts/render_world_class_preflight.py` plus `scripts/world_class_preflight_layout.py`: operator-facing collection preflight for world-class evidence. Keep data assembly and CLI emission in the renderer, keep HTML layout in the layout helper, keep environment and external prerequisite checks redacted, and never let preflight count as accepted evidence. +- `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. +- `scripts/yao_cli_parser.py`: CLI argparse command surface, flags, choices, and command handler binding. +- `scripts/yao_cli_telemetry.py`: opt-in metadata-only CLI run telemetry. Keep it free of prompt, argument, output, transcript, note, or message capture. +- `scripts/import_telemetry_events.py`: external telemetry importer. Validate the whole input before appending events, and keep raw prompt/output/transcript/message/note fields blocked. +- `scripts/emit_telemetry_event.py`: external client telemetry emitter. It may append one normalized metadata event to a local spool, but must never accept or write raw prompt, output, transcript, message, note, argument, or private content. +- `scripts/render_telemetry_hook_recipes.py`: client hook recipe report. Keep recipes metadata-only, mark native auto-capture as unclaimed unless a real client integration exists, and preserve dry-run commands for Browser/Chrome/IDE/wrapper adapters. +- `scripts/telemetry_native_host.py`: Browser/Chrome Native Messaging telemetry bridge. Preserve length-prefixed stdio behavior, raw-content blocking, and launcher/manifest generation tests. New helper modules that are imported by CLI/report scripts but are not standalone commands must declare: diff --git a/Makefile b/Makefile index 659bca15..c3871e02 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check yao-cli-world-class-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -28,20 +28,36 @@ description-optimization-check: promotion-check: $(PYTHON) tests/verify_promotion_checker.py +python-compat-check: + $(PYTHON) tests/verify_python_compat_check.py + +architecture-maintainability-check: + $(PYTHON) tests/verify_architecture_maintainability.py + yao-cli-check: $(PYTHON) tests/verify_yao_cli.py +yao-cli-world-class-check: + $(PYTHON) tests/verify_yao_cli_world_class.py + skill-overview-check: $(PYTHON) tests/verify_skill_overview.py +skill-interpretation-check: + $(PYTHON) tests/verify_skill_interpretation.py + skill-report-metrics-check: $(PYTHON) tests/verify_skill_report_metrics.py skill-report-charts-check: $(PYTHON) tests/verify_skill_report_charts.py +html-rendering-check: + $(PYTHON) tests/verify_html_rendering.py + skill-ir-check: $(PYTHON) tests/verify_skill_ir.py + $(PYTHON) tests/verify_skill_ir_paths.py compiler-check: $(PYTHON) tests/verify_compile_skill.py @@ -52,6 +68,9 @@ output-eval-check: output-execution-check: $(PYTHON) tests/verify_output_execution_runs.py +output-review-kit-check: + $(PYTHON) tests/verify_output_review_kit.py + output-review-adjudication-check: $(PYTHON) tests/verify_output_review_adjudication.py @@ -85,12 +104,72 @@ review-viewer-check: review-studio-check: $(PYTHON) tests/verify_review_studio.py +skill-os2-audit-check: + $(PYTHON) tests/verify_skill_os2_audit.py + +skill-os2-coverage-check: + $(PYTHON) tests/verify_skill_os2_coverage.py + +world-class-evidence-check: + $(PYTHON) tests/verify_world_class_evidence_plan.py + +world-class-ledger-check: + $(PYTHON) tests/verify_world_class_evidence_ledger.py + +world-class-intake-check: + $(PYTHON) tests/verify_world_class_evidence_intake.py + +world-class-submission-kit-check: + $(PYTHON) tests/verify_world_class_submission_kit.py + +world-class-preflight-check: + $(PYTHON) tests/verify_world_class_preflight.py + +world-class-submission-review-check: + $(PYTHON) tests/verify_world_class_submission_review.py + +world-class-runbook-check: + $(PYTHON) tests/verify_world_class_operator_runbook.py + +world-class-claim-guard-check: + $(PYTHON) tests/verify_world_class_claim_guard.py + +benchmark-reproducibility-check: + $(PYTHON) tests/verify_benchmark_reproducibility.py + +evidence-consistency-check: + $(PYTHON) tests/verify_evidence_consistency.py + feedback-check: $(PYTHON) tests/verify_feedback.py +adaptation-safety-check: + $(PYTHON) tests/verify_adaptation_safety.py + +skillops-opportunity-check: + $(PYTHON) tests/verify_skillops_opportunity.py + +daily-skillops-check: + $(PYTHON) tests/verify_daily_skillops.py + +weekly-curator-check: + $(PYTHON) tests/verify_weekly_curator.py + adoption-drift-check: $(PYTHON) tests/verify_adoption_drift.py +telemetry-import-check: + $(PYTHON) tests/verify_telemetry_import.py + +telemetry-emit-check: + $(PYTHON) tests/verify_telemetry_emit.py + +telemetry-hooks-check: + $(PYTHON) tests/verify_telemetry_hooks.py + +telemetry-native-host-check: + $(PYTHON) tests/verify_telemetry_native_host.py + review-waivers-check: $(PYTHON) tests/verify_review_waivers.py @@ -152,7 +231,7 @@ failure-regression-check: $(PYTHON) tests/verify_failure_regressions.py package-check: - $(PYTHON) scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --output-dir dist --zip + $(PYTHON) scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip package-failure-check: $(PYTHON) tests/verify_packager_failures.py @@ -181,17 +260,17 @@ resource-boundary-check: quality-check: $(PYTHON) tests/verify_quality_checks.py -sync-local-install: +sync-local-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(LOCAL_SKILL_INSTALL_DIR)" -sync-active-install: +sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_python_compat tests/tmp_architecture_maintainability tests/tmp_skill_overview tests/tmp_skill_interpretation tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_submission_review tests/tmp_world_class_operator_runbook tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_evidence_consistency tests/tmp_feedback tests/tmp_daily_skillops tests/tmp_weekly_curator tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index bda3dc80..2367431f 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,32 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a silent-by-default GitHub benchmark scan plus reference synthesis that studies top public repositories and world-class pattern tracks, then surfaces only real conflicts or uncertainty to the user - a generated visual HTML overview for each newly initialized skill - a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, adoption drift, reviewer waivers, reviewer annotations, release evidence, and per-warning fix actions +- a Skill OS 2.0 audit that maps each world-class requirement to current evidence, human-required gaps, and external-required gaps +- a Skill OS 2.0 blueprint coverage report that maps the upgrade plan's core modules and recommended PRs to concrete artifacts, commands, and tests +- a world-class evidence plan that turns remaining provider, human, native-permission, and real-client telemetry gaps into executable evidence tasks +- a world-class evidence ledger that records which external and human evidence is accepted or still pending without treating planned work as proof +- a world-class evidence intake contract that validates external and human evidence packets for provenance, privacy, artifact refs, and anti-overclaim rules before ledger review +- a redacted world-class preflight report that checks local files, environment readiness, human/external prerequisites, and source blockers before operators collect evidence +- a world-class submission review queue that compares evidence packets, intake validation, source artifacts, and ledger state without accepting evidence +- a world-class operator runbook that gives reviewers the exact commands, artifacts, and collection checklist needed to close remaining evidence gaps +- a world-class claim guard that scans public claim surfaces and blocks premature completed/true claims while the evidence ledger still has pending external or human evidence +- a benchmark reproducibility manifest that checks methodology sections, required artifacts, failure disclosure, and reproduction commands +- an evidence consistency gate that compares generated reports against each other so benchmark, overview, interpretation, adoption, world-class ledger, coverage, and Review Studio facts do not drift silently - Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports - a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks +- a Python compatibility gate that catches supported-runtime syntax hazards before they reach GitHub Actions or packaged distribution - a side-by-side HTML review studio for first-pass human review - an artifact design profile that defines visual direction, layout patterns, and quality gates for reports, tutorials, dashboards, screenshots, and review pages - a prompt quality profile that abstracts need modeling, RTF mapping, complexity, and quality checks into reviewer-visible evidence instead of bloating `SKILL.md` - a systems-thinking model that maps boundaries, feedback loops, drift risks, recurring failure patterns, and highest-leverage quality moves - three high-value next iteration directions after the first package is created - a lightweight feedback log that does not require a full promotion cycle -- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates +- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates, with optional `yao.py` CLI run capture, external client event emit hooks, hook recipes, and JSONL import that record command names and outcomes without arguments or raw content +- an explicit-source adaptive proposal loop that summarizes redacted repeated user preferences and generates approval-gated adaptation proposals without scanning private logs or writing source files +- a SkillOps opportunity scorer and decision policy that ranks redacted repeated signals, maps them to report-only, AGENTS update, existing-skill patch, or eval-addition actions, and keeps every durable write approval-gated +- a weekly SkillOps curator report that aggregates daily opportunities, Skill Atlas portfolio signals, release lock state, and world-class evidence gaps into a proposal-only maintenance queue +- a Browser/Chrome Native Messaging telemetry host that can receive length-prefixed metadata-only client events and generate a local launcher plus manifest without storing raw content +- a Skill Atlas drift layer that reads aggregate adoption reports and surfaces portfolio-level drift signals without packaging raw telemetry logs - a baseline compare report for with-skill vs baseline review - a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits - Skill IR as the platform-neutral semantic contract, plus compiler reports and client-specific adapters @@ -89,7 +106,7 @@ Weighted score formula: `sum(score / 10 * weight)`. 2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit. 3. Let `quickstart` clarify intent first, then run silent benchmark scan and reference synthesis; it only surfaces explicit questions when intent is still unclear or when there is a real design conflict. 4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode. -5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers, permission approvals, and evidence paths in one page before adding more structure. +5. Review the generated `reports/skill-interpretation.html` first for the bilingual interpretation report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/skill-overview.html` for the audit scorecard and `reports/review-studio.html` to inspect release blockers, permission approvals, and evidence paths in one page before adding more structure. Or use the unified authoring CLI: @@ -100,32 +117,56 @@ python3 scripts/yao.py reference-scan my-skill \ --external-reference "World Class Method::method::Borrow a tight evaluation loop.::Do not copy heavy process." \ --user-reference "A product or repo I admire::taste::Learn the clarity and operating standard.::Do not copy wording." \ --local-constraint "Current Library Naming::structure::Keep naming aligned with the local skill library.::Do not inherit private references." +python3 scripts/yao.py skill-interpretation my-skill python3 scripts/yao.py review-viewer my-skill python3 scripts/yao.py review-studio my-skill python3 scripts/yao.py artifact-design-profile my-skill python3 scripts/yao.py prompt-quality-profile my-skill python3 scripts/yao.py system-model my-skill python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary +python3 scripts/yao.py adapt-scan my-skill --source ./curated-user-signals.jsonl +python3 scripts/yao.py adapt-propose my-skill +python3 scripts/yao.py daily-skillops my-skill --source ./curated-user-signals.jsonl +python3 scripts/yao.py weekly-curator my-skill python3 scripts/yao.py adoption-drift my-skill --record-event skill_activation --activation-type explicit --outcome accepted +YAO_CLI_TELEMETRY=1 python3 scripts/yao.py validate my-skill +python3 scripts/yao.py telemetry-emit my-skill --event skill_activation --activation-type explicit --outcome accepted --command browser-extension +python3 scripts/yao.py telemetry-hooks my-skill +python3 scripts/telemetry_native_host.py my-skill --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/ +python3 scripts/yao.py telemetry-import my-skill --input-jsonl /tmp/external-client-events.jsonl --command browser-extension python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key trust-report --reviewer "Yao Team" --reason "Known warning accepted for this release with bounded follow-up." --expires-at 2026-09-30 python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key permission-gates --reviewer "Yao Team" --reason "Permission warning accepted only for this non-governed release window." --expires-at 2026-09-30 python3 scripts/yao.py review-annotations my-skill --add-annotation --gate-key output-lab --target-path reports/output_quality_scorecard.md --line 1 --body "Clarify recorded fixture vs model-executed evidence before release." python3 scripts/yao.py baseline-compare python3 scripts/yao.py check-update python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json -python3 scripts/yao.py compile-skill . --target openai --target claude --target generic +python3 scripts/yao.py compile-skill . --target openai --target claude --target generic --target vscode python3 scripts/yao.py package . --platform generic --output-dir dist python3 scripts/yao.py output-eval python3 scripts/yao.py output-exec python3 scripts/yao.py output-review python3 scripts/yao.py conformance . python3 scripts/yao.py trust . +python3 scripts/yao.py python-compat . python3 scripts/yao.py runtime-permissions . --package-dir dist python3 scripts/yao.py skill-atlas --workspace-root . python3 scripts/yao.py registry-audit . python3 scripts/yao.py package-verify . --package-dir dist --require-zip python3 scripts/yao.py install-simulate . --package-dir dist python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json +python3 scripts/yao.py world-class-evidence . +SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}" +python3 scripts/yao.py world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" +# Alternative: prefill artifact SHA-256 digests while keeping drafts template-only. +python3 scripts/yao.py world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts +python3 scripts/yao.py world-class-intake . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-runbook . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-claim-guard . +python3 scripts/yao.py benchmark-reproducibility . +python3 scripts/yao.py evidence-consistency . ``` ## Local Development Source @@ -140,7 +181,7 @@ Sync the current source into the disabled mirror: make sync-local-install ``` -The sync command copies Git-tracked files plus new source files in code and guidance directories such as `scripts/`, `tests/`, `references/`, and `docs/`. It skips untracked business-skill folders and untracked private reports by default, so local experiments do not leak into the mirror. +The sync command first rebuilds the package and runs install preflight against `dist/yao-meta-skill.zip`. It refuses to sync when package extraction, adapter readability, or installer permission enforcement fails. After the preflight passes, it copies Git-tracked files plus new source files in code and guidance directories such as `scripts/`, `tests/`, `references/`, and `docs/`. It skips untracked business-skill folders and untracked private reports by default, so local experiments do not leak into the mirror. Restore an active global Codex install only when you intentionally want this skill discoverable outside the development workspace: @@ -177,7 +218,7 @@ python3 scripts/context_sizer.py . python3 scripts/resource_boundary_check.py . python3 scripts/governance_check.py . --require-manifest python3 scripts/compile_skill.py . -python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip +python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --zip python3 scripts/probe_runtime_permissions.py . --package-dir dist python3 tests/verify_packager_failures.py ``` @@ -199,7 +240,7 @@ python3 scripts/yao.py review --target root python3 scripts/yao.py release-snapshot --target root --label release-candidate python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json python3 scripts/yao.py compile-skill . -python3 scripts/yao.py package . --platform openai --platform claude --platform generic --output-dir dist --zip +python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip python3 scripts/yao.py runtime-permissions . --package-dir dist python3 scripts/yao.py package-verify . --package-dir dist --require-zip python3 scripts/yao.py test @@ -247,9 +288,9 @@ The homepage panel below is generated from the current eval suite so the family- Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md) -- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance, semantic parity metadata, and target-native behavior contracts -- target compiler validation: `openai`, `claude`, `generic`, and Agent Skills compatible contracts are compiled from Skill IR with generated-file mappings, adapter modes, native surfaces, permission enforcement notes, and unsupported-feature notes -- runtime permission probes: `openai`, `claude`, and `generic` adapters expose explicit permission contracts; current targets report `0` native-enforcement adapters and `3` metadata fallbacks with residual risks visible to reviewers +- packaging validation: `openai`, `claude`, `generic`, and `vscode` targets pass contract checks and carry IR provenance, semantic parity metadata, and target-native behavior contracts +- target compiler validation: `openai`, `claude`, `generic`, Agent Skills compatible, and VS Code / Copilot contracts are compiled from Skill IR with generated-file mappings, adapter modes, native surfaces, permission enforcement notes, and unsupported-feature notes +- runtime permission probes: `openai`, `claude`, `generic`, and `vscode` adapters expose explicit permission contracts; current targets report `0` native-enforcement adapters and `4` metadata fallbacks with residual risks visible to reviewers - portability score: `100/100` with neutral activation, execution, trust, and degradation metadata preserved across all exported targets - description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately - judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts @@ -258,8 +299,8 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - governance and resource-boundary checks are part of the default test path - root governance maturity score: `90/100`; governed benchmark example: `95/100` - CJK-aware trigger matching is now covered by explicit Chinese build, packaging, eval, and near-neighbor cases -- context budgets: root `987/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` -- quality density: root `131.7`, complex benchmark `164.6`, governed benchmark `171.1` +- context budgets: root `944/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` +- quality density: root `137.7`, complex benchmark `164.6`, governed benchmark `171.1` - regression milestones are tracked in [reports/regression_history.md](reports/regression_history.md) - description drift history is tracked in [reports/description_drift_history.md](reports/description_drift_history.md) - route confusion is tracked in [reports/route_scorecard.md](reports/route_scorecard.md) @@ -269,8 +310,18 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - lightweight with-skill vs baseline comparison is published in [reports/baseline-compare.md](reports/baseline-compare.md) - Review Studio 2.0 gate evidence is published in [reports/review-studio.html](reports/review-studio.html) - Review Studio fix actions are embedded in [reports/review-studio.json](reports/review-studio.json) +- Skill OS 2.0 blueprint coverage is published in [reports/skill_os2_coverage.md](reports/skill_os2_coverage.md) - reviewer waiver evidence is published in [reports/review_waivers.md](reports/review_waivers.md) +- remaining world-class evidence tasks are published in [reports/world_class_evidence_plan.md](reports/world_class_evidence_plan.md) +- current world-class evidence acceptance state is published in [reports/world_class_evidence_ledger.md](reports/world_class_evidence_ledger.md) +- world-class evidence intake readiness is published in [reports/world_class_evidence_intake.md](reports/world_class_evidence_intake.md) +- world-class submission review queue is published in [reports/world_class_submission_review.md](reports/world_class_submission_review.md) +- world-class operator runbook is published in [reports/world_class_operator_runbook.md](reports/world_class_operator_runbook.md) and [reports/world_class_operator_runbook.html](reports/world_class_operator_runbook.html) +- world-class public claim guard status is published in [reports/world_class_claim_guard.md](reports/world_class_claim_guard.md) +- benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md) +- cross-report evidence consistency is published in [reports/evidence_consistency.md](reports/evidence_consistency.md) - target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md) +- Python runtime compatibility evidence is published in [reports/python_compatibility.md](reports/python_compatibility.md) - registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md) - package archive verification is published in [reports/package_verification.md](reports/package_verification.md) - temporary local install simulation is published in [reports/install_simulation.md](reports/install_simulation.md) @@ -385,7 +436,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, python-compat, review, release-snapshot, workspace-flow, report, skill-report, skill-interpretation, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-preflight, world-class-submission-kit, world-class-submission-review, world-class-runbook, world-class-claim-guard, benchmark-reproducibility, evidence-consistency, adapt-scan, adapt-propose, adapt-apply, daily-skillops, weekly-curator, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -394,9 +445,25 @@ Utility scripts that make the meta-skill operational: - `governance_check.py`: validates owner, review cadence, lifecycle stage, and maturity metadata - `render_context_reports.py`: generates root and example context-budget reports plus a shared context summary - `render_regression_history.py`: turns milestone snapshots into a readable regression history report +- `render_skill_os2_audit.py`: renders a requirement-by-requirement Skill OS 2.0 audit that separates landed local evidence from human-required and external-required gaps +- `render_skill_os2_coverage.py`: maps the Skill OS 2.0 upgrade blueprint to local artifacts, commands, tests, and remaining evidence boundaries +- `render_daily_skillops_report.py`: renders an explicit-source Daily SkillOps operations report that summarizes redacted user patterns, proposal-only adaptations, approval state, release evidence, and world-class evidence gaps without scanning private logs or applying patches +- `render_weekly_curator_report.py`: renders a weekly SkillOps curator report from generated daily reports, Skill Atlas, benchmark lock, evidence consistency, and world-class ledger state without scanning private logs or applying patches +- `skillops_opportunity.py`: scores redacted SkillOps opportunities and maps them to approval-gated action types such as report-only, AGENTS update, existing-skill patch, or eval addition +- `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence +- `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts +- `render_world_class_evidence_intake.py`: validates world-class external and human evidence packets against provenance, privacy, artifact, and anti-overclaim requirements before ledger review +- `render_world_class_preflight.py`: renders redacted collection preflight checks for pending provider, human, native-permission, and native-client evidence without accepting evidence +- `render_world_class_submission_review.py`: renders a read-only queue that compares submissions, intake validation, source evidence, and ledger state without accepting evidence +- `render_world_class_operator_runbook.py`: renders an operator-facing checklist and command map for collecting pending world-class evidence without accepting evidence +- `render_world_class_claim_guard.py`: scans README, docs, and reports for premature world-class completion claims while accepted evidence is still pending +- `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims +- `render_evidence_consistency.py`: compares generated report facts across benchmark reproducibility, overview, interpretation, adoption drift, world-class ledger, coverage, and Review Studio artifacts +- `python_compat_check.py`: checks Python source for supported-runtime compatibility hazards such as Python 3.11 f-string expression backslashes - `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap +- `render_skill_interpretation.py`: renders `reports/skill-interpretation.html/json` as the first-class post-creation interpretation report while reusing the Skill Overview v2 model and Kami white layout - `export_skill_ir.py`: exports the 2.0 platform-neutral Skill IR contract from `SKILL.md`, manifest, interface metadata, evals, resources, and reports - `compile_skill.py`: compiles Skill IR into target-specific semantic contracts, generated-file maps, adapter modes, target-native behavior contracts, preserved semantics, warnings, and unsupported-feature notes - `run_output_eval.py`: runs the Output Eval Lab v0 with static with-skill vs baseline assertion grading, blind A/B review pack generation, and separate answer key artifacts @@ -406,12 +473,17 @@ Utility scripts that make the meta-skill operational: - `render_review_annotations.py`: records reviewer annotations tied to Review Studio gates, source/report paths, and optional line numbers, with open blocker annotations reflected in Review Studio decisions - `run_conformance_suite.py`: verifies runtime conformance for OpenAI, Claude, Agent Skills, VS Code/Copilot-style, and generic targets - `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, bounded network host policy, execution-level `--help` smoke checks, permission inputs, trust metadata, and stable source-contract integrity -- `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview for a multi-skill workspace +- `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, aggregate drift signals, and HTML overview for a multi-skill workspace - `registry_audit.py`: builds registry package metadata and audits version, owner, license, checksum, Skill IR source, and compatibility matrix - `verify_package.py`: verifies generated package manifests, target adapters, zip archive safety, archive checksum, and registry parity - `simulate_install.py`: extracts a generated zip into a temporary skill root and verifies entrypoint, manifest, interface, reports, and adapters can be loaded - `upgrade_check.py`: compares current and previous registry package metadata, recommends a version bump, and blocks incompatible upgrade claims - `render_adoption_drift_report.py`: records metadata-only local telemetry and renders adoption, missed-trigger, bad-output, script-error, and review-drift signals without packaging raw event logs +- `import_telemetry_events.py`: imports external metadata-only telemetry JSONL after whole-file privacy validation, then refreshes the aggregate adoption drift report +- `emit_telemetry_event.py`: emits one metadata-only external client event into a local spool for later `telemetry-import`, with dry-run validation and raw-content field blocking +- `render_telemetry_hook_recipes.py`: renders Browser, Chrome, VS Code, CLI wrapper, and provider-adapter telemetry hook recipes with dry-run commands and explicit native-integration caveats +- `telemetry_native_host.py`: receives Browser/Chrome Native Messaging length-prefixed JSON events, rejects raw-content fields, appends metadata-only events, and writes local launcher/manifest files for operator installation +- `yao_cli_telemetry.py`: opt-in metadata-only `yao.py` run capture for command name, source, outcome, and failure class without command arguments or raw content - `render_review_waivers.py`: validates human reviewer risk approvals with gate keys, reasons, expiry dates, and blocker-safe waiver policy - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain - `check_update.py`: checks GitHub for a newer `VERSION` or remote manifest version and reports a reinstall hint without modifying local files diff --git a/SKILL.md b/SKILL.md index 34d49012..89809d3b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,9 +1,8 @@ --- name: yao-meta-skill -description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. metadata: author: Yao Team - philosophy: "structured design, evaluation loop, template ergonomics, operational packaging" --- # Yao Meta Skill @@ -16,44 +15,39 @@ metadata: ## Modes -- `Scaffold`: exploratory or personal. -- `Production`: team reuse. -- `Library`: shared infrastructure. - -Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md). +- `Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical. +- Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md). ## Compact Workflow -1. Decide whether the request should become a skill and choose the lightest fit. -2. Capture job, output, exclusions, constraints, and standards. -3. Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict. -4. Write the `description` early and test route quality before expanding the package. -5. Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter. -6. Add only folders and gates that earn their keep. -7. Surface the top three next iteration directions. +1. For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`. +2. Capture job, output, exclusions, constraints, standards, and the lightest fit. +3. Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict. +4. Write `description` early, test route quality, then add only earned folders and gates. +5. Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful. -Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Registry](references/distribution-registry-method.md), [Telemetry](references/telemetry-drift-method.md), [Waivers](references/review-waiver-method.md), [Review Studio](references/review-studio-method.md). +Playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Review Studio](references/review-studio-method.md). ## Skill OS 2.0 Gates -For production, library, governed, or team-distributed work, use Skill IR, target compiler contracts, trigger + output eval, Skill Atlas, runtime conformance, trust report, registry metadata, package verification, install simulation, upgrade check, adoption/drift telemetry, reviewer waiver evidence, and Review Studio before release. +For production, library, governed, or team-distributed work, run Skill IR, target compiler, trigger + output eval, Skill Atlas, conformance, trust, registry/package/install, upgrade, drift, waiver, and Review Studio gates before release. + +## Governed Package Boundary + +For file-backed, release-critical, or governed packages, name `input_files` as `file-backed fixture` evidence; include `owner`, `review cadence`, `input_files`, `output contract`, `rollback boundary`; require `trust report` and `reports/output_quality_scorecard.md`; mark unavailable telemetry, approvals, metrics, or benchmarks as `missing evidence`; do not fabricate evidence. + +Preserve audit labels literally when they apply: `file-backed fixture`, `input_files`, `output contract`, `rollback boundary`, `trust report`, `reports/output_quality_scorecard.md`, `missing evidence`. ## First-Turn Style -When the skill first activates: - -- start from the user's work and desired outcome before structure -- ask only `2-3` key questions unless the user already gave enough detail -- avoid cold field lists; surface benchmark choices only when uncertainty or conflict needs a call - -Chinese conversations should sound soft and companion-like rather than procedural. - -Opening patterns: [Intent Dialogue](references/intent-dialogue.md). +- Start from the user's work/outcome before structure. +- Ask only `2-3` key questions unless enough detail exists. +- In Chinese, sound soft and companion-like; use [Intent Dialogue](references/intent-dialogue.md). ## Output Contract -Unless the user asks otherwise, produce a working skill directory with `SKILL.md`, aligned `agents/interface.yaml`, justified assets, and a short summary of boundary, exclusions, gates, and next steps. +Unless asked otherwise, produce `SKILL.md`, aligned `agents/interface.yaml`, justified assets, and a short summary of boundary, exclusions, gates, and next steps. ## Reference Map -Primary references: [Method](references/skill-engineering-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md). +Primary: [Method](references/skill-engineering-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md), [SkillOps Decision](references/skillops-decision-policy.md). diff --git a/agents/interface.yaml b/agents/interface.yaml index 599591a9..35912dba 100644 --- a/agents/interface.yaml +++ b/agents/interface.yaml @@ -8,6 +8,7 @@ compatibility: - "openai" - "claude" - "generic" + - "vscode" activation: mode: "manual" paths: [] @@ -22,3 +23,4 @@ compatibility: openai: "metadata-adapter" claude: "neutral-source-plus-adapter" generic: "neutral-source" + vscode: "agent-skills-source-with-vscode-notes" diff --git a/assets/review-studio.css b/assets/review-studio.css new file mode 100644 index 00000000..20bdd878 --- /dev/null +++ b/assets/review-studio.css @@ -0,0 +1,749 @@ +:root { + --ink: #1B365D; + --text: #24201d; + --muted: #746d66; + --line: #e7ded2; + --soft: #faf8f5; + --pass: #1e6b52; + --warn: #9a6718; + --block: #9b2c2c; + } + * { box-sizing: border-box; } + html { scroll-behavior: smooth; } + body { + margin: 0; + background: #ffffff; + color: var(--text); + font-family: Georgia, "Times New Roman", "Songti SC", serif; + line-height: 1.58; + } + nav { + position: sticky; + top: 0; + z-index: 10; + display: flex; + gap: 4px; + justify-content: center; + flex-wrap: wrap; + padding: 10px 16px; + background: rgba(255,255,255,0.94); + border-bottom: 1px solid var(--line); + backdrop-filter: blur(10px); + } + nav a { + color: var(--ink); + text-decoration: none; + font-size: 14px; + padding: 7px 10px; + border-radius: 6px; + } + nav a:hover { background: var(--soft); } + main { max-width: 1180px; margin: 0 auto; padding: 44px 28px 76px; } + header { border-bottom: 1px solid var(--line); padding-bottom: 28px; margin-bottom: 28px; } + .eyebrow { color: var(--ink); font-size: 14px; letter-spacing: .08em; text-transform: uppercase; } + h1, h2, h3 { color: var(--text); font-weight: 500; margin: 0; letter-spacing: 0; } + h1 { font-size: clamp(34px, 5vw, 64px); line-height: 1.03; max-width: 920px; margin-top: 12px; } + h2 { font-size: 30px; margin-bottom: 14px; } + h3 { font-size: 19px; } + p { margin: 0; } + .lede { max-width: 820px; color: var(--muted); font-size: 20px; margin-top: 18px; } + .decision { + display: inline-flex; + align-items: baseline; + gap: 12px; + margin-top: 24px; + padding: 12px 16px; + border: 1px solid var(--line); + border-radius: 8px; + color: var(--ink); + background: var(--soft); + } + .decision strong { font-size: 28px; } + section { padding: 30px 0; border-bottom: 1px solid var(--line); scroll-margin-top: 76px; } + .metrics, .gates { + display: grid; + grid-template-columns: repeat(5, minmax(0, 1fr)); + gap: 14px; + } + .gates { grid-template-columns: repeat(4, minmax(0, 1fr)); } + .metric, .gate { + border: 1px solid var(--line); + border-radius: 8px; + padding: 16px; + background: #fff; + min-width: 0; + } + .metric span, .gate span { display: block; color: var(--muted); font-size: 13px; } + .metric strong { display: block; color: var(--ink); font-size: 34px; line-height: 1.1; margin: 8px 0; } + .metric p, .gate p, .gate footer, .issues span, .evidence span { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; } + .gate { display: flex; flex-direction: column; gap: 10px; } + .gate.pass { border-top: 4px solid var(--pass); } + .gate.warn { border-top: 4px solid var(--warn); } + .gate.block { border-top: 4px solid var(--block); } + .gate footer { border-top: 1px solid var(--line); padding-top: 10px; } + a { color: var(--ink); } + .twocol { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 22px; + align-items: start; + } + .panel { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + } + .panel p { color: var(--muted); } + .kv-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px 12px; + margin: 2px 0 0; + } + .kv-grid div { + min-width: 0; + padding: 9px 0 0; + border-top: 1px solid var(--line); + } + .kv-grid dt { + color: var(--muted); + font-size: 13px; + } + .kv-grid dd { + margin: 2px 0 0; + color: var(--ink); + font-size: 15px; + overflow-wrap: anywhere; + } + .issues, .evidence { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 12px; + } + .issues li, .evidence li { + border-left: 3px solid var(--line); + padding-left: 12px; + display: grid; + gap: 3px; + } + .muted { color: var(--muted); } + .actions-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + } + .annotations-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + } + .action-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 10px; + min-width: 0; + } + .action-card.warn { border-left: 4px solid var(--warn); } + .action-card.block { border-left: 4px solid var(--block); } + .action-card.with-evidence { + grid-column: 1 / -1; + } + .action-evidence-panel { + display: grid; + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + margin-top: 2px; + } + .action-evidence-panel h4 { + margin: 0; + color: var(--ink); + font-size: 16px; + font-weight: 500; + } + .action-evidence-panel p { + color: var(--muted); + font-size: 14px; + } + .action-evidence-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 14px; + } + .action-evidence-item { + border: 1px solid var(--line); + border-radius: 8px; + padding: 14px; + display: grid; + gap: 10px; + min-width: 0; + background: var(--soft); + } + .action-evidence-item.pending { border-left: 4px solid var(--warn); } + .action-evidence-item.accepted { border-left: 4px solid var(--pass); } + .action-evidence-item h4, + .action-evidence-item h5 { + margin: 0; + color: var(--ink); + font-weight: 500; + } + .action-evidence-item h4 { font-size: 16px; } + .action-evidence-item h5 { font-size: 14px; } + .action-evidence-item section { + border-top: 1px solid var(--line); + padding-top: 10px; + display: grid; + gap: 8px; + } + .action-command-details, + .action-phase-details, + .action-runbook-details, + .action-collection-details { + border-top: 1px solid var(--line); + padding-top: 10px; + display: grid; + gap: 8px; + } + .action-command-details summary, + .action-phase-details summary, + .action-runbook-details summary, + .action-collection-details summary { + cursor: pointer; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .action-command-details[open] summary, + .action-phase-details[open] summary, + .action-runbook-details[open] summary, + .action-collection-details[open] summary { + margin-bottom: 8px; + } + .action-collection-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; + } + .action-collection-grid section { + border-top: 1px solid var(--line); + padding-top: 8px; + display: grid; + gap: 6px; + } + .action-collection-grid ul, + .action-collection-grid ol { + margin: 0; + padding-left: 18px; + display: grid; + gap: 5px; + } + .action-collection-grid li { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .action-artifact-roles { + padding: 0; + margin: 0; + list-style: none; + display: grid; + gap: 8px; + } + .action-artifact-roles li { + border: 1px solid var(--line); + border-radius: 8px; + background: #fff; + padding: 10px; + display: grid; + gap: 4px; + min-width: 0; + } + .action-artifact-roles strong { + color: var(--ink); + font-weight: 500; + overflow-wrap: anywhere; + } + .action-artifact-roles span, + .action-artifact-roles code, + .action-artifact-roles small { + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; + white-space: normal; + } + .action-evidence-item dl { + display: grid; + grid-template-columns: 76px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .action-evidence-item dt { + color: var(--ink); + font-size: 14px; + } + .action-evidence-item dd { + margin: 0; + min-width: 0; + } + .action-evidence-checks, + .action-phase-list, + .action-command-list, + .action-repair-list, + .action-runbook-list { + margin: 0; + display: grid; + gap: 8px; + } + .action-evidence-checks, + .action-phase-list, + .action-repair-list, + .action-command-list { + padding: 0; + list-style: none; + } + .action-runbook-list { + padding-left: 20px; + } + .action-evidence-check, + .action-phase-row, + .action-repair-row, + .action-command-list li { + display: grid; + gap: 3px; + min-width: 0; + border-top: 1px solid var(--line); + padding-top: 8px; + } + .action-evidence-check.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .action-evidence-check.pass { border-left: 3px solid var(--pass); padding-left: 8px; } + .action-phase-row.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .action-phase-row.ready { border-left: 3px solid var(--pass); padding-left: 8px; } + .action-repair-row.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .action-repair-row.ready { border-left: 3px solid var(--pass); padding-left: 8px; } + .action-evidence-check span, + .action-phase-row span, + .action-repair-row span, + .action-command-list span { + color: var(--ink); + font-size: 13px; + } + .action-phase-row strong, + .action-repair-row strong { + color: var(--ink); + font-size: 14px; + overflow-wrap: anywhere; + } + .action-phase-row em, + .action-repair-row em { + color: var(--muted); + font-size: 12px; + font-style: normal; + overflow-wrap: anywhere; + } + .action-evidence-check code, + .action-evidence-check small, + .action-phase-row code, + .action-phase-row small, + .action-repair-row code, + .action-repair-row small, + .action-command-list code, + .action-runbook-list li { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .action-command-list code { + display: block; + white-space: normal; + } + .annotation-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 10px; + min-width: 0; + } + .annotation-card.warning { border-left: 4px solid var(--warn); } + .annotation-card.blocker { border-left: 4px solid var(--block); } + .annotation-card.resolved { opacity: .72; } + .world-evidence-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + margin-top: 16px; + } + .world-evidence-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .world-evidence-card.pending { border-left: 4px solid var(--warn); } + .world-evidence-card.accepted { border-left: 4px solid var(--pass); } + .world-evidence-card span, + .world-evidence-card p, + .world-evidence-card dd, + .world-evidence-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .world-evidence-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .world-evidence-card dt { color: var(--ink); font-size: 14px; } + .world-evidence-card dd { margin: 0; } + .world-evidence-columns { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-evidence-columns h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-evidence-columns ul { + margin: 0; + padding-left: 18px; + } + .world-source-panel { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-runbook-panel { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-runbook-panel h4, + .world-source-panel h4 { + margin: 0 0 8px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-runbook-panel ul { + margin: 0; + padding-left: 18px; + } + .world-source-checks { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 8px; + } + .world-source-check { + display: grid; + grid-template-columns: minmax(0, .7fr) minmax(0, .6fr) minmax(0, 1fr); + gap: 10px; + align-items: start; + min-width: 0; + padding: 9px 0 0; + border-top: 1px solid var(--line); + } + .world-source-check span { + color: var(--ink); + font-size: 14px; + } + .world-source-check code, + .world-source-check small { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .world-source-check.pass { border-left: 3px solid var(--pass); padding-left: 8px; } + .world-source-check.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .output-review-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + margin-top: 16px; + } + .output-review-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .output-review-card.awaiting-decision, + .output-review-card.fix-decision { border-left: 4px solid var(--warn); } + .output-review-card.adjudicated { border-left: 4px solid var(--pass); } + .output-review-card span, + .output-review-card p, + .output-review-card dd, + .output-review-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .output-review-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .output-review-card dt { color: var(--ink); font-size: 14px; } + .output-review-card dd { margin: 0; min-width: 0; } + .output-review-steps { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .output-review-steps h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .output-review-steps ul { + margin: 0; + padding-left: 18px; + } + .output-review-commands, + .output-review-fields { + list-style: none; + padding: 0; + display: grid; + gap: 8px; + } + .output-review-commands li, + .output-review-fields li { + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + } + .output-review-commands code { + overflow-wrap: anywhere; + white-space: normal; + display: block; + } + .world-intake-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + margin-top: 16px; + } + .world-intake-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .world-intake-card.awaiting-submission, + .world-intake-card.fix-submission, + .world-intake-card.fix-template { border-left: 4px solid var(--warn); } + .world-intake-card.ready-for-ledger-review, + .world-intake-card.accepted { border-left: 4px solid var(--pass); } + .world-intake-card span, + .world-intake-card p, + .world-intake-card dd, + .world-intake-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .world-intake-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .world-intake-card dt { color: var(--ink); font-size: 14px; } + .world-intake-card dd { margin: 0; min-width: 0; } + .world-intake-steps { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-intake-steps h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-intake-steps ul { + margin: 0; + padding-left: 18px; + } + .world-intake-commands { + list-style: none; + padding: 0; + display: grid; + gap: 8px; + } + .world-intake-commands li { + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + } + .world-intake-commands code { + overflow-wrap: anywhere; + white-space: normal; + display: block; + } + .section-subtitle { + margin-top: 22px; + margin-bottom: 12px; + color: var(--ink); + } + .waiver-summary { margin-top: 16px; } + .waiver-candidate-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + } + .waiver-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .waiver-card.allowed { border-left: 4px solid var(--warn); } + .waiver-card.blocked { border-left: 4px solid var(--block); } + .waiver-card span, + .waiver-card p, + .waiver-card dd, + .waiver-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .waiver-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .waiver-card dt { + color: var(--ink); + font-size: 14px; + } + .waiver-card dd { margin: 0; min-width: 0; } + .waiver-review { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .waiver-review h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .waiver-review ul { + margin: 0; + padding-left: 18px; + } + .waiver-command { + display: grid; + gap: 4px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .waiver-command span { + color: var(--ink); + font-size: 14px; + } + .waiver-command code { + display: block; + max-width: 100%; + overflow-wrap: anywhere; + white-space: normal; + } + .action-card span, + .annotation-card span, + .action-card small, + .annotation-card small, + .action-card footer, + .annotation-card footer, + .action-card dd { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .annotation-card dd { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; } + .action-card dl, .annotation-card dl { + display: grid; + grid-template-columns: 80px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .action-card dt, .annotation-card dt { color: var(--ink); font-size: 14px; } + .action-card dd, .annotation-card dd { margin: 0; } + .source-ref-list { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 6px; + } + .source-ref-list li { + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + } + .source-ref-list a, + .source-ref-list span { + overflow-wrap: anywhere; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + } + .source-ref-list small { + font-size: 12px; + color: var(--muted); + } + .source-ref-list blockquote { + margin: 2px 0 0; + padding: 7px 9px; + border-left: 3px solid var(--line); + background: var(--soft); + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; + } + code { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + } + @media (max-width: 980px) { + .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .action-evidence-grid, .action-evidence-item dl, .action-collection-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-source-check, .world-intake-grid, .world-intake-steps, .waiver-candidate-grid, .waiver-card dl, .kv-grid { grid-template-columns: 1fr; } + main { padding: 32px 18px 60px; } + nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; } + nav a { flex: 0 0 auto; } + } diff --git a/assets/review-viewer.css b/assets/review-viewer.css new file mode 100644 index 00000000..11abdb53 --- /dev/null +++ b/assets/review-viewer.css @@ -0,0 +1,176 @@ +:root { + --text: #111111; + --muted: #666666; + --line: #e8e8e8; + --soft: #f6f6f4; + --white: #ffffff; +} +* { box-sizing: border-box; } +body { + margin: 0; + background: var(--white); + color: var(--text); + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + line-height: 1.6; +} +.page { + max-width: 1120px; + margin: 0 auto; + padding: 48px 32px 72px; +} +.hero { + padding-bottom: 28px; + border-bottom: 1px solid var(--line); + margin-bottom: 28px; +} +h1, h2, h3 { + margin: 0 0 12px; + letter-spacing: -0.02em; + font-weight: 600; +} +h1 { font-size: 40px; line-height: 1.08; } +h2 { font-size: 22px; margin-top: 34px; } +h3 { font-size: 16px; } +p, li, span { font-size: 15px; } +.lede { + max-width: 860px; + font-size: 18px; + color: var(--muted); + margin: 0 0 18px; +} +.meta { + display: flex; + gap: 10px; + flex-wrap: wrap; + margin-top: 16px; +} +.meta span { + border: 1px solid var(--line); + padding: 6px 10px; + background: var(--soft); +} +.arch-grid { + display: grid; + grid-template-columns: repeat(5, minmax(0, 1fr)); + gap: 12px; + margin-top: 16px; +} +.arch-step, .panel, .direction-card, .baseline-box { + border: 1px solid var(--line); + background: var(--white); +} +.arch-step { + padding: 14px; + min-height: 132px; +} +.step-label { + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--muted); + margin-bottom: 10px; +} +.step-detail { + font-size: 14px; +} +.grid { + display: grid; + grid-template-columns: 1.1fr 0.9fr; + gap: 18px; + margin-top: 16px; +} +.panel { + padding: 18px; +} +.panel ul { + margin: 0; + padding-left: 18px; +} +.direction-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 16px; + margin-top: 16px; +} +.variant-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 16px; + margin-top: 16px; +} +.direction-card { + padding: 18px; +} +.direction-card ul { + margin: 12px 0; + padding-left: 18px; +} +.minor { + color: var(--muted); + font-size: 13px; +} +.variant-card { + border: 1px solid var(--line); + background: var(--white); + padding: 18px; +} +.variant-head { + display: flex; + justify-content: space-between; + gap: 12px; + align-items: baseline; +} +.variant-head span { + color: var(--muted); + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.06em; +} +.variant-description { + margin: 14px 0; + padding-left: 14px; + border-left: 2px solid var(--line); + color: var(--text); +} +.variant-metrics { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-bottom: 14px; +} +.variant-metrics span { + border: 1px solid var(--line); + background: var(--soft); + padding: 6px 10px; + font-size: 12px; +} +.variant-cues p { + margin: 8px 0 6px; +} +.variant-cues ul { + margin: 0 0 12px; + padding-left: 18px; +} +table { + width: 100%; + border-collapse: collapse; + margin-top: 14px; + font-size: 14px; +} +th, td { + border-top: 1px solid var(--line); + text-align: left; + padding: 10px 8px; + vertical-align: top; +} +th { + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--muted); +} +@media (max-width: 1000px) { + .arch-grid, .direction-grid, .variant-grid, .grid { + grid-template-columns: 1fr; + } +} diff --git a/assets/skill-overview.css b/assets/skill-overview.css new file mode 100644 index 00000000..b626f5e1 --- /dev/null +++ b/assets/skill-overview.css @@ -0,0 +1,735 @@ +:root { + --paper: #ffffff; + --wash: #f8fafc; + --wash-strong: #f2f5f8; + --line: #e6e0d4; + --line-soft: #eee9df; + --brand: #1B365D; + --brand-soft: #EEF3F8; + --brand-mid: #315982; + --ink: #151515; + --text: #2f2d29; + --muted: #68625a; + --faint: #8b857b; + --success: #2f6f5e; + --warn: #8a5a19; + --serif: "TsangerJinKai02", "Source Han Serif SC", "Noto Serif CJK SC", "Songti SC", "STSong", Charter, Georgia, serif; + --mono: "JetBrains Mono", "SF Mono", ui-monospace, Menlo, Consolas, monospace; + --shadow-soft: 0 1px 2px rgba(27, 54, 93, 0.06), 0 16px 44px rgba(27, 54, 93, 0.08); + } + * { box-sizing: border-box; } + html { scroll-behavior: smooth; } + body { + margin: 0; + background: #ffffff; + color: var(--ink); + font-family: var(--serif); + line-height: 1.62; + letter-spacing: 0; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + text-rendering: optimizeLegibility; + } + body[data-report-lang="zh-CN"] [data-lang="en"], + body[data-report-lang="en"] [data-lang="zh-CN"] { display: none !important; } + .skip-link { + position: fixed; + left: 18px; + top: 10px; + z-index: 40; + transform: translateY(-140%); + padding: 8px 12px; + border-radius: 8px; + background: var(--brand); + color: #fff; + text-decoration: none; + transition: transform 160ms cubic-bezier(0.16, 1, 0.3, 1); + } + .skip-link:focus-visible { transform: translateY(0); outline: 2px solid var(--brand-mid); outline-offset: 2px; } + .topbar { + position: sticky; + top: 0; + z-index: 20; + background: rgba(255, 255, 255, 0.96); + border-bottom: 1px solid var(--line); + } + .progress-track { + height: 2px; + background: transparent; + } + .progress-bar { + display: block; + width: 100%; + height: 100%; + background: var(--brand); + transition: transform 160ms cubic-bezier(0.16, 1, 0.3, 1); + transform-origin: left center; + transform: scaleX(0); + } + .topbar-inner { + max-width: 1240px; + margin: 0 auto; + padding: 9px 28px; + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 18px; + align-items: center; + } + .nav-shell { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 18px; + align-items: center; + min-width: 0; + } + .report-mark { + color: var(--brand); + font-family: var(--mono); + font-size: 11px; + line-height: 1; + text-transform: uppercase; + white-space: nowrap; + } + .report-nav { + display: flex; + gap: 4px; + overflow-x: auto; + scrollbar-width: none; + min-width: 0; + } + .report-nav::-webkit-scrollbar { display: none; } + .report-nav a { + flex: 0 0 auto; + min-width: 76px; + min-height: 40px; + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0 10px; + color: var(--brand); + text-decoration: none; + text-align: center; + font-size: 13px; + border-radius: 8px; + white-space: nowrap; + transition-property: background-color, color, transform; + transition-duration: 160ms; + transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1); + } + @media (hover: hover) { + .report-nav a:hover { background: var(--brand-soft); } + } + .report-nav a:focus-visible { + outline: 2px solid var(--brand-mid); + outline-offset: 2px; + background: var(--brand-soft); + } + .report-nav a[aria-current="true"] { + background: var(--brand); + color: #ffffff; + } + .report-nav a[aria-current="true"] span { color: #ffffff; } + .language-switch { + display: inline-flex; + gap: 3px; + padding: 3px; + border: 1px solid var(--line); + border-radius: 8px; + background: var(--paper); + box-shadow: 0 1px 2px rgba(27, 54, 93, 0.05); + } + .language-switch button { + appearance: none; + min-width: 42px; + min-height: 34px; + border: 0; + border-radius: 6px; + padding: 0 10px; + background: transparent; + color: var(--muted); + font: inherit; + font-size: 13px; + cursor: pointer; + transition-property: background-color, color, transform; + transition-duration: 160ms; + transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1); + } + .language-switch button:active { transform: scale(0.96); } + .language-switch button:focus-visible { outline: 2px solid var(--brand-mid); outline-offset: 2px; } + .language-switch button[aria-pressed="true"] { + background: var(--brand-soft); + color: var(--brand); + } + .wrap { + max-width: 1240px; + margin: 0 auto; + padding: 58px 28px 92px; + } + .hero { + padding: 16px 0 34px; + } + .hero-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) 360px; + gap: 44px; + align-items: end; + } + .eyebrow, .report-mark { + letter-spacing: 0; + } + .eyebrow { + margin: 0 0 12px; + color: var(--brand); + font-family: var(--mono); + font-size: 12px; + text-transform: uppercase; + } + .slug { + margin: 0 0 18px; + color: var(--faint); + font-family: var(--mono); + font-size: 13px; + overflow-wrap: anywhere; + } + h1, h2, h3 { + margin: 0; + font-weight: 500; + letter-spacing: 0; + text-wrap: balance; + } + h1 { + max-width: 8em; + color: var(--ink); + font-size: 4rem; + line-height: 1.02; + } + .lead { + max-width: 760px; + margin: 20px 0 0; + color: var(--text); + font-size: 1.08rem; + text-wrap: pretty; + } + .hero-meta, .badges { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 20px; + } + .hero-meta span, .badges span, .tag { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 5px 10px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 13px; + font-variant-numeric: tabular-nums; + } + .hero-card { + position: relative; + padding: 22px 22px 24px; + border-radius: 8px; + background: linear-gradient(180deg, #ffffff 0%, #fbfaf7 100%); + box-shadow: var(--shadow-soft); + border: 1px solid var(--line-soft); + } + .hero-card::before { + content: ""; + position: absolute; + top: 18px; + right: 18px; + width: 38px; + height: 2px; + background: var(--brand); + } + .score-strip { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 10px; + margin-top: 32px; + padding-top: 20px; + border-top: 1px solid var(--line); + } + .score-chip { + padding: 14px 14px 12px; + border-radius: 8px; + background: var(--wash); + border: 1px solid var(--line-soft); + } + .score-chip span { + display: block; + color: var(--muted); + font-size: 12px; + } + .score-chip strong { + display: block; + margin: 5px 0 8px; + color: var(--brand); + font-family: var(--mono); + font-size: 1.65rem; + font-weight: 500; + line-height: 1; + font-variant-numeric: tabular-nums; + } + .score-chip i { + display: block; + height: 3px; + border-radius: 999px; + background: linear-gradient(90deg, var(--brand) var(--score), #dfe6ee var(--score)); + } + .score-chip small { + display: block; + margin-top: 8px; + color: var(--muted); + font-size: 12px; + line-height: 1.45; + } + section { + scroll-margin-top: 78px; + padding-top: 44px; + margin-top: 44px; + border-top: 1px solid var(--line); + } + section.hero { + scroll-margin-top: 0; + padding-top: 16px; + margin-top: 0; + border-top: 0; + } + .section-head { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 22px; + align-items: start; + } + h2 { + color: var(--ink); + font-size: 1.5rem; + line-height: 1.18; + } + h2::before { + content: ""; + display: block; + width: 32px; + height: 2px; + margin-bottom: 12px; + background: var(--brand); + } + h3 { + color: var(--ink); + font-size: 1.02rem; + line-height: 1.3; + margin-bottom: 10px; + } + .section-head > div:first-child p { + margin: 12px 0 0; + color: var(--muted); + max-width: 68ch; + text-wrap: pretty; + } + .section-body { + margin-top: 24px; + } + .two-col, .metric-grid, .chart-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + align-items: stretch; + } + .quality-panels { + margin-top: 16px; + } + .metrics-flow { + display: grid; + gap: 30px; + } + .metrics-primary { + display: grid; + grid-template-columns: minmax(360px, 0.92fr) minmax(0, 1.08fr); + gap: 22px; + align-items: start; + } + .metrics-primary .chart-figure { + min-height: 0; + display: grid; + align-content: center; + } + .metrics-primary .chart-figure svg { + max-height: 520px; + } + .metrics-note { + display: grid; + align-content: start; + gap: 14px; + } + .metrics-note p { + margin: 0; + max-width: none; + color: var(--text); + } + .metric-summary-list { + margin: 0; + padding: 0; + list-style: none; + display: grid; + gap: 10px; + } + .metric-summary-list li { + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; + gap: 8px 10px; + align-items: baseline; + padding: 10px 0; + border-bottom: 1px solid var(--line-soft); + } + .metric-summary-list li:last-child { border-bottom: 0; } + .metric-summary-list b { + color: var(--ink); + font-weight: 500; + } + .metric-summary-list em { + color: var(--brand); + font-family: var(--mono); + font-style: normal; + font-variant-numeric: tabular-nums; + } + .metric-summary-list small { + grid-column: 2 / 4; + color: var(--muted); + line-height: 1.5; + overflow-wrap: break-word; + } + .metric-detail-section { + display: grid; + gap: 14px; + } + .detail-section-kicker { + display: inline-flex; + width: fit-content; + min-height: 28px; + align-items: center; + padding: 3px 8px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 12px; + } + .metric-status { + display: inline-flex; + align-items: center; + min-height: 24px; + padding: 2px 7px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 12px; + } + .metric-detail-grid { + grid-template-columns: repeat(2, minmax(min(100%, 420px), 1fr)); + align-items: stretch; + } + .list, .compact-list, .step-list { + margin: 0; + padding-left: 1.15em; + display: grid; + gap: 8px; + } + .list li::marker, .compact-list li::marker, .step-list li::marker { color: var(--brand); } + .compact-list { + gap: 6px; + font-size: 13.5px; + color: var(--muted); + line-height: 1.55; + } + .compact-list li, .list li, .step-list li { + min-width: 0; + overflow-wrap: break-word; + } + .panel, .metric-card, .roadmap-item { + background: #ffffff; + border: 1px solid var(--line-soft); + border-radius: 8px; + padding: 22px; + box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04); + min-width: 0; + } + .metric-card { + display: grid; + grid-template-columns: minmax(112px, 0.28fr) minmax(0, 1fr); + gap: 18px; + align-content: start; + min-height: 0; + } + .metric-card-head { + display: grid; + gap: 10px; + align-content: start; + } + .metric-card strong { + display: block; + margin: 0; + color: var(--brand); + font-family: var(--mono); + font-size: 2rem; + font-weight: 500; + line-height: 1; + font-variant-numeric: tabular-nums; + } + .metric-label { + color: var(--muted); + font-size: 13px; + line-height: 1.45; + overflow-wrap: break-word; + } + .metric-card-body { + min-width: 0; + } + .metric-card-body .compact-list { + gap: 7px; + max-width: 76ch; + } + .metric-card-body .compact-list li { + overflow-wrap: break-word; + word-break: normal; + } + .chart-figure { + margin: 0; + padding: 18px; + border: 1px solid var(--line-soft); + border-radius: 8px; + background: #ffffff; + box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04); + } + .chart-figure svg { + width: 100%; + height: auto; + display: block; + overflow: visible; + } + .chart-figure text { + fill: var(--ink); + font-family: var(--serif); + font-size: 13px; + } + .chart-title { + fill: var(--brand); + font-size: 16px; + font-weight: 500; + } + .chart-line { + fill: none; + stroke: var(--brand); + stroke-width: 2; + } + figcaption { + margin-top: 12px; + padding-top: 10px; + border-top: 1px solid var(--line-soft); + color: var(--muted); + font-size: 13px; + line-height: 1.5; + text-wrap: pretty; + } + table { + width: 100%; + border-collapse: collapse; + font-size: 14px; + } + th, td { + padding: 12px 10px; + text-align: left; + border-bottom: 1px solid var(--line-soft); + vertical-align: top; + } + th { + color: var(--brand); + font-weight: 500; + font-size: 12px; + } + td:first-child, th:first-child { width: 96px; } + .world-readiness { + margin-top: 16px; + display: grid; + gap: 16px; + } + .section-head > .world-readiness { grid-column: 1; } + .world-readiness-head { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 16px; + align-items: start; + } + .world-readiness-head p { + margin-top: 8px; + max-width: 72ch; + } + .world-status { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 4px 10px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 13px; + white-space: nowrap; + } + .evidence-kpis { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 10px; + } + .evidence-kpi { + padding: 14px; + border: 1px solid var(--line-soft); + border-radius: 8px; + background: var(--wash); + min-width: 0; + } + .evidence-kpi span { + display: block; + color: var(--muted); + font-size: 12px; + } + .evidence-kpi strong { + display: block; + margin-top: 4px; + color: var(--brand); + font-family: var(--mono); + font-size: 1.5rem; + font-weight: 500; + line-height: 1; + font-variant-numeric: tabular-nums; + } + .evidence-kpi small { + display: block; + margin-top: 8px; + color: var(--muted); + line-height: 1.45; + overflow-wrap: break-word; + } + .evidence-list { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; + } + .evidence-item { + padding: 14px; + border: 1px solid var(--line-soft); + border-radius: 8px; + background: #ffffff; + min-width: 0; + } + .evidence-item span { + color: var(--brand); + font-family: var(--mono); + font-size: 11px; + text-transform: uppercase; + } + .evidence-item h4 { + margin: 4px 0 8px; + color: var(--ink); + font-size: 1rem; + font-weight: 500; + line-height: 1.3; + } + .evidence-item h5 { + margin: 12px 0 6px; + color: var(--brand); + font-size: 12px; + font-weight: 500; + } + .evidence-item p { + margin: 0; + max-width: none; + color: var(--muted); + overflow-wrap: break-word; + } + .blocked-checks { + margin: 0; + padding-left: 1.05em; + color: var(--muted); + font-size: 13px; + line-height: 1.45; + } + .blocked-checks li { + overflow-wrap: break-word; + } + .blocked-checks li::marker { + color: var(--brand); + } + .blocked-checks-empty { + font-size: 13px; + } + .roadmap { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 16px; + } + .step { + display: inline-flex; + margin-bottom: 12px; + min-height: 28px; + align-items: center; + padding: 3px 8px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 12px; + } + .unlock { + margin-top: 14px; + color: var(--muted); + font-size: 13px; + } + @media (max-width: 980px) { + .topbar-inner { + grid-template-columns: minmax(0, 1fr) auto; + padding: 8px 16px; + } + .nav-shell { grid-template-columns: 1fr; gap: 6px; } + .report-mark { display: none; } + .report-nav a { min-width: 72px; } + .hero-grid, .section-head { grid-template-columns: 1fr; } + .section-head > .world-readiness { grid-column: auto; } + .hero-grid { gap: 24px; } + .score-strip { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .metrics-primary, .metric-detail-grid, .two-col, .metric-grid, .chart-grid, .roadmap { grid-template-columns: 1fr; } + .evidence-list { grid-template-columns: 1fr; } + .metric-card { grid-template-columns: 1fr; } + h1 { font-size: 2.75rem; } + } + @media (max-width: 540px) { + .wrap { padding: 34px 16px 72px; } + .topbar-inner { gap: 10px; } + .report-nav { + display: flex; + gap: 2px; + } + .report-nav a { + min-width: 66px; + min-height: 38px; + padding: 0 8px; + font-size: 12px; + } + .language-switch button { min-width: 38px; min-height: 32px; padding: 0 8px; } + .hero { padding-top: 8px; } + h1 { max-width: 8em; font-size: 2.35rem; } + .lead { font-size: 1rem; } + .hero-meta span, .badges span { font-size: 12px; } + .score-strip { grid-template-columns: 1fr; } + section { margin-top: 34px; padding-top: 34px; } + .panel, .metric-card, .roadmap-item, .chart-figure, .hero-card { padding: 18px; } + .world-readiness-head, .evidence-kpis { grid-template-columns: 1fr; } + .metric-card { grid-template-columns: 1fr; gap: 10px; } + .metric-card strong { margin-bottom: 2px; } + .metric-summary-list li { grid-template-columns: auto minmax(0, 1fr) auto; } + table { font-size: 13px; } + th, td { padding: 10px 7px; } + } + @media (prefers-reduced-motion: reduce) { + html { scroll-behavior: auto; } + *, *::before, *::after { transition-duration: 0.01ms !important; } + } diff --git a/assets/skill-overview.js b/assets/skill-overview.js new file mode 100644 index 00000000..eaf9ef4f --- /dev/null +++ b/assets/skill-overview.js @@ -0,0 +1,37 @@ +(function () { + var buttons = Array.prototype.slice.call(document.querySelectorAll("[data-set-lang]")); + var navLinks = Array.prototype.slice.call(document.querySelectorAll(".report-nav a")); + var sections = navLinks + .map(function (link) { return document.querySelector(link.getAttribute("href")); }) + .filter(Boolean); + var progressBar = document.querySelector(".progress-bar"); + function setLanguage(lang) { + document.body.setAttribute("data-report-lang", lang); + document.documentElement.setAttribute("lang", lang); + buttons.forEach(function (button) { + button.setAttribute("aria-pressed", button.getAttribute("data-set-lang") === lang ? "true" : "false"); + }); + } + function updateProgress() { + var scrollTop = window.scrollY || document.documentElement.scrollTop; + var height = Math.max(1, document.documentElement.scrollHeight - window.innerHeight); + var pct = Math.min(100, Math.max(0, (scrollTop / height) * 100)); + if (progressBar) progressBar.style.transform = "scaleX(" + pct / 100 + ")"; + var active = sections[0]; + sections.forEach(function (section) { + if (section.getBoundingClientRect().top <= 110) active = section; + }); + navLinks.forEach(function (link) { + link.setAttribute("aria-current", link.getAttribute("href") === "#" + active.id ? "true" : "false"); + }); + } + buttons.forEach(function (button) { + button.addEventListener("click", function () { + setLanguage(button.getAttribute("data-set-lang")); + }); + }); + window.addEventListener("scroll", updateProgress, { passive: true }); + window.addEventListener("resize", updateProgress); + setLanguage("zh-CN"); + updateProgress(); + })(); diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 0b4a349b..7189997b 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -41,6 +41,8 @@ This document tracks the first migration path from Yao Meta Skill 1.x to the Ski python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json ``` + This also writes `skill_atlas/drift_signals.json` from aggregate `reports/adoption_drift_report.json` files. Do not migrate or publish raw `reports/telemetry_events.jsonl` logs. + 5. Keep legacy trigger eval gates intact while output eval coverage grows. 6. Move adapter-specific logic toward future compiler commands after IR fields are stable. diff --git a/evals/adaptation/user_signals.example.jsonl b/evals/adaptation/user_signals.example.jsonl new file mode 100644 index 00000000..ed06eda9 --- /dev/null +++ b/evals/adaptation/user_signals.example.jsonl @@ -0,0 +1,8 @@ +{"text":"新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。"} +{"text":"HTML 报告需要双语能力，但默认内容应该保持中文简体。"} +{"text":"报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。"} +{"text":"Skill 概览报告需要更多图表模块，避免右侧文本挤在一起。"} +{"text":"自适应升级必须先生成提案，不能直接自动修改源文件。"} +{"text":"用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。"} +{"text":"每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。"} +{"text":"涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。"} diff --git a/evals/packaging_expectations.json b/evals/packaging_expectations.json index 374c9059..f39792d4 100644 --- a/evals/packaging_expectations.json +++ b/evals/packaging_expectations.json @@ -1,5 +1,5 @@ { - "required_targets": ["openai", "claude", "generic"], + "required_targets": ["openai", "claude", "generic", "vscode"], "required_fields": [ "name", "description", @@ -30,5 +30,6 @@ ], "openai_required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], "claude_required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], - "generic_required_files": ["targets/generic/adapter.json"] + "generic_required_files": ["targets/generic/adapter.json"], + "vscode_required_files": ["targets/vscode/adapter.json", "targets/vscode/README.md"] } diff --git a/evidence/world_class/README.md b/evidence/world_class/README.md new file mode 100644 index 00000000..a6013f8d --- /dev/null +++ b/evidence/world_class/README.md @@ -0,0 +1,45 @@ +# World-Class Evidence Intake + +This directory defines the intake contract for external and human evidence required before `yao-meta-skill` can honestly claim public world-class completion. + +The templates in `templates/` are review aids only. They do not count as accepted evidence. Real submissions belong in `evidence/world_class/submissions/`, which is intentionally gitignored by default so provider metadata, reviewer identity, or client integration notes can be reviewed before anything is committed. + +Run: + +```bash +SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}" +python3 scripts/yao.py world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" +# Alternative: prefill artifact SHA-256 digests while keeping drafts template-only. +python3 scripts/yao.py world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts +python3 scripts/yao.py world-class-intake . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR" +python3 scripts/yao.py world-class-runbook . --submissions-dir "$SUBMISSIONS_DIR" +``` + +The intake validator checks: + +- the evidence key matches the current world-class ledger +- the category and source type match the expected human or external evidence path +- artifact references are declared +- real submissions reference concrete files inside the skill directory and include a matching SHA-256 digest for each artifact +- real submissions use the canonical `.json` filename expected by the ledger +- credentials, secrets, raw user content, and raw provider prompts are explicitly excluded +- raw prompt, output, transcript, message, credential, secret, token, and API-key fields are rejected even when nested +- human adjudication packets must preserve reviewer identity, review date, A/B winner, confidence, and a required rationale before answer-key reveal can count +- planned work, local command-only output, and metadata fallback are not claimed as completion evidence + +Run `world-class-preflight` before assigning external or human work. It checks local files, redacted environment readiness, human/external prerequisites, and source-evidence blockers without accepting evidence or printing secrets. + +The generated intake report also includes an `operator_checklist` for each pending evidence item. Use it to find the template path, target submission path, preparation command, validation command, required provenance, success checks, and privacy boundary before asking a reviewer or external operator to submit evidence. + +The submission kit command creates editable JSON drafts plus a local README for an external operator or human reviewer. Use `--prefill-artifacts` when you want the kit to insert SHA-256 digests for currently available local aggregate artifacts. Prefill is operator convenience only: those drafts still keep `template_only: true` and do not count as evidence until the real run or review exists, the packet is edited truthfully, every artifact ref points to a local aggregate evidence file with a matching `sha256`, and `world-class-intake` validates it. + +The submission kit separates artifact rows into `submission-ref` and `supporting-evidence`. `submission-ref` rows are the concrete paths expected in a real packet's `artifact_refs`; `supporting-evidence` rows help reviewers audit the packet and do not all need to be copied into the submission. + +The submission review command renders a read-only queue that compares valid packets with the source evidence checks and current ledger state. It is for reviewer triage only; it does not accept evidence or make the world-class claim true. + +The operator runbook is the step-by-step coordination cockpit for finishing the remaining real-world work. It includes a Coordination Plan that separates user-required provider, reviewer, native-client, and telemetry actions from assistant-run commands, plus a Release Gate that mirrors the ledger, claim guard, benchmark, Review Studio, evidence consistency, and final CI checks. The runbook is operational guidance only: planning rows, submission drafts, and release-gate rows keep `counts_as_completion: false`. + +Accepted intake means "ready for ledger review", not evidence that the final public claim is ready. The ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/evidence/world_class/intake.schema.json b/evidence/world_class/intake.schema.json new file mode 100644 index 00000000..8194e251 --- /dev/null +++ b/evidence/world_class/intake.schema.json @@ -0,0 +1,440 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://example.local/yao-meta-skill/world-class-evidence-intake.schema.json", + "title": "Yao World-Class Evidence Intake", + "description": "Real submissions must use the canonical .json filename and the runtime validator rejects nested raw prompt, output, transcript, message, credential, secret, token, and API-key fields.", + "type": "object", + "required": [ + "schema_version", + "evidence_key", + "template_only", + "category", + "source_type", + "submitted_by", + "submitted_at", + "summary", + "artifact_refs", + "provenance", + "privacy", + "anti_overclaim", + "attestation" + ], + "properties": { + "schema_version": { + "const": "1.0" + }, + "evidence_key": { + "enum": [ + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry" + ] + }, + "template_only": { + "type": "boolean" + }, + "category": { + "enum": [ + "human", + "external" + ] + }, + "source_type": { + "type": "string", + "minLength": 3 + }, + "submitted_by": { + "type": "string", + "minLength": 1 + }, + "submitted_at": { + "type": "string", + "minLength": 1 + }, + "summary": { + "type": "string", + "minLength": 1 + }, + "artifact_refs": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "path", + "kind", + "contains_raw_content" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "kind": { + "type": "string", + "minLength": 1 + }, + "contains_raw_content": { + "type": "boolean" + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-fA-F]{64}$" + }, + "note": { + "type": "string" + } + }, + "additionalProperties": true + } + }, + "provenance": { + "type": "object", + "additionalProperties": true + }, + "privacy": { + "type": "object", + "required": [ + "raw_user_content_included", + "raw_provider_prompt_included", + "credentials_included", + "secrets_included" + ], + "properties": { + "raw_user_content_included": { + "type": "boolean" + }, + "raw_provider_prompt_included": { + "type": "boolean" + }, + "credentials_included": { + "type": "boolean" + }, + "secrets_included": { + "type": "boolean" + }, + "notes": { + "type": "string" + } + }, + "additionalProperties": true + }, + "anti_overclaim": { + "type": "object", + "required": [ + "planned_work_counts_as_evidence", + "metadata_fallback_counts_as_native_enforcement", + "pending_review_counts_as_human_decision", + "local_command_runner_counts_as_provider_model" + ], + "properties": { + "planned_work_counts_as_evidence": { + "const": false + }, + "metadata_fallback_counts_as_native_enforcement": { + "const": false + }, + "pending_review_counts_as_human_decision": { + "const": false + }, + "local_command_runner_counts_as_provider_model": { + "const": false + } + }, + "additionalProperties": false + }, + "attestation": { + "type": "object", + "required": [ + "real_external_or_human_evidence", + "reviewer_or_operator_identity_present", + "artifact_refs_reviewed", + "privacy_contract_satisfied", + "ledger_reviewer_approved", + "ledger_reviewer", + "ledger_reviewed_at" + ], + "properties": { + "real_external_or_human_evidence": { + "type": "boolean" + }, + "reviewer_or_operator_identity_present": { + "type": "boolean" + }, + "artifact_refs_reviewed": { + "type": "boolean" + }, + "privacy_contract_satisfied": { + "type": "boolean" + }, + "ledger_reviewer_approved": { + "type": "boolean" + }, + "ledger_reviewer": { + "type": "string" + }, + "ledger_reviewed_at": { + "type": "string" + }, + "notes": { + "type": "string" + } + }, + "additionalProperties": true + } + }, + "allOf": [ + { + "if": { + "properties": { + "template_only": { + "const": false + } + }, + "required": [ + "template_only" + ] + }, + "then": { + "properties": { + "submitted_at": { + "pattern": "^\\d{4}-\\d{2}-\\d{2}(?:T\\d{2}:\\d{2}:\\d{2}Z)?$" + }, + "submitted_by": { + "not": { + "enum": [ + "operator with provider credentials", + "human reviewer", + "target client or installer integrator", + "Browser/Chrome/IDE/provider client integrator" + ] + } + }, + "artifact_refs": { + "items": { + "required": [ + "path", + "kind", + "contains_raw_content", + "sha256" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "template_only": { + "const": false + }, + "evidence_key": { + "const": "provider-holdout" + } + }, + "required": [ + "template_only", + "evidence_key" + ] + }, + "then": { + "properties": { + "artifact_refs": { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/output_execution_runs.json" + }, + "kind": { + "const": "aggregate-report" + } + }, + "required": [ + "path", + "kind" + ] + } + } + } + } + }, + { + "if": { + "properties": { + "template_only": { + "const": false + }, + "evidence_key": { + "const": "human-adjudication" + } + }, + "required": [ + "template_only", + "evidence_key" + ] + }, + "then": { + "properties": { + "artifact_refs": { + "allOf": [ + { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/output_review_adjudication.json" + }, + "kind": { + "const": "adjudication-report" + } + }, + "required": [ + "path", + "kind" + ] + } + }, + { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/output_review_decisions.json" + }, + "kind": { + "const": "review-decisions" + } + }, + "required": [ + "path", + "kind" + ] + } + } + ] + } + } + } + }, + { + "if": { + "properties": { + "template_only": { + "const": false + }, + "evidence_key": { + "const": "native-permission-enforcement" + } + }, + "required": [ + "template_only", + "evidence_key" + ] + }, + "then": { + "properties": { + "artifact_refs": { + "allOf": [ + { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/runtime_permission_probes.json" + }, + "kind": { + "const": "runtime-probe-report" + } + }, + "required": [ + "path", + "kind" + ] + } + }, + { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/install_simulation.json" + }, + "kind": { + "const": "installer-enforcement-report" + } + }, + "required": [ + "path", + "kind" + ] + } + } + ] + } + } + } + }, + { + "if": { + "properties": { + "template_only": { + "const": false + }, + "evidence_key": { + "const": "native-client-telemetry" + } + }, + "required": [ + "template_only", + "evidence_key" + ] + }, + "then": { + "properties": { + "artifact_refs": { + "allOf": [ + { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/adoption_drift_report.json" + }, + "kind": { + "const": "adoption-drift-report" + } + }, + "required": [ + "path", + "kind" + ] + } + }, + { + "contains": { + "type": "object", + "properties": { + "path": { + "const": "reports/telemetry_hook_recipes.json" + }, + "kind": { + "const": "hook-recipes" + } + }, + "required": [ + "path", + "kind" + ] + } + } + ] + } + } + } + } + ], + "additionalProperties": true +} diff --git a/evidence/world_class/templates/human-adjudication.intake.json b/evidence/world_class/templates/human-adjudication.intake.json new file mode 100644 index 00000000..f6cfd728 --- /dev/null +++ b/evidence/world_class/templates/human-adjudication.intake.json @@ -0,0 +1,64 @@ +{ + "schema_version": "1.0", + "evidence_key": "human-adjudication", + "template_only": true, + "category": "human", + "source_type": "blind-ab-review", + "submitted_by": "human reviewer", + "submitted_at": "YYYY-MM-DD", + "summary": "Real blind A/B reviewer decisions recorded before the answer key was opened.", + "artifact_refs": [ + { + "path": "reports/output_review_adjudication.json", + "kind": "adjudication-report", + "contains_raw_content": false, + "note": "Must show pending_count == 0, invalid_decision_count == 0, blind_review_attested == true, and review_integrity.blind_pack_sha256." + }, + { + "path": "reports/output_review_decisions.json", + "kind": "review-decisions", + "contains_raw_content": false, + "note": "Reviewer identity, reviewed_at, winner_variant, confidence, required reason, reviewer_attestation, and review_integrity only." + } + ], + "provenance": { + "reviewer": "name or team handle", + "reviewed_at": "YYYY-MM-DD", + "blind_pack_path": "reports/output_blind_review_pack.md", + "blind_pack_sha256": "sha256 from reports/output_review_adjudication.json review_integrity.blind_pack_sha256", + "answer_key_opened_after_decisions": true, + "blind_review_completed_before_answer_key": true, + "decision_fields": [ + "case_id", + "winner_variant", + "confidence", + "reason", + "reviewer_attestation", + "review_integrity" + ], + "reviewer_reason_required": true + }, + "privacy": { + "raw_user_content_included": false, + "raw_provider_prompt_included": false, + "credentials_included": false, + "secrets_included": false, + "notes": "Reviewer reasons must be based on the visible rubric and must not include private customer data." + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "attestation": { + "real_external_or_human_evidence": false, + "reviewer_or_operator_identity_present": false, + "artifact_refs_reviewed": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "notes": "Copy this template into submissions/ after a real reviewer completes blind decisions." + } +} diff --git a/evidence/world_class/templates/native-client-telemetry.intake.json b/evidence/world_class/templates/native-client-telemetry.intake.json new file mode 100644 index 00000000..9dfdfa27 --- /dev/null +++ b/evidence/world_class/templates/native-client-telemetry.intake.json @@ -0,0 +1,53 @@ +{ + "schema_version": "1.0", + "evidence_key": "native-client-telemetry", + "template_only": true, + "category": "external", + "source_type": "native-client-telemetry", + "submitted_by": "Browser/Chrome/IDE/provider client integrator", + "submitted_at": "YYYY-MM-DD", + "summary": "Metadata-only production events imported from a real external client into the local drift loop.", + "artifact_refs": [ + { + "path": "reports/adoption_drift_report.json", + "kind": "adoption-drift-report", + "contains_raw_content": false, + "note": "Must show summary.source_types.external > 0 and adoption_sample_count > 0." + }, + { + "path": "reports/telemetry_hook_recipes.json", + "kind": "hook-recipes", + "contains_raw_content": false, + "note": "Documents local-first metadata-only client integration." + } + ], + "provenance": { + "client": "Browser/Chrome/IDE/provider client", + "native_host_manifest": "/local/path/not/committed", + "event_source": "external", + "metadata_only": true + }, + "privacy": { + "raw_user_content_included": false, + "raw_provider_prompt_included": false, + "credentials_included": false, + "secrets_included": false, + "notes": "Never commit telemetry JSONL with raw prompts, outputs, transcripts, notes, or messages." + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "attestation": { + "real_external_or_human_evidence": false, + "reviewer_or_operator_identity_present": false, + "artifact_refs_reviewed": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "notes": "Copy this template into submissions/ after real client events have been imported." + } +} diff --git a/evidence/world_class/templates/native-permission-enforcement.intake.json b/evidence/world_class/templates/native-permission-enforcement.intake.json new file mode 100644 index 00000000..e16db354 --- /dev/null +++ b/evidence/world_class/templates/native-permission-enforcement.intake.json @@ -0,0 +1,61 @@ +{ + "schema_version": "1.0", + "evidence_key": "native-permission-enforcement", + "template_only": true, + "category": "external", + "source_type": "runtime-permission-guard", + "submitted_by": "target client or installer integrator", + "submitted_at": "YYYY-MM-DD", + "summary": "A real target or installer guard enforces approved high-permission capabilities at runtime.", + "artifact_refs": [ + { + "path": "reports/runtime_permission_probes.json", + "kind": "runtime-probe-report", + "contains_raw_content": false, + "note": "Must show native_enforcement_count > 0 and failure_count == 0 for target-native completion; installer_enforcement_pass_count is supporting local evidence only." + }, + { + "path": "reports/install_simulation.json", + "kind": "installer-enforcement-report", + "contains_raw_content": false, + "note": "Shows local package installer enforcement. This does not replace target-native or external runtime guard evidence." + }, + { + "path": "dist/targets//adapter.json", + "kind": "target-adapter", + "contains_raw_content": false, + "note": "Adapter may claim native enforcement only when the target really enforces it." + } + ], + "provenance": { + "target": "openai|claude|generic|vscode|other", + "guard_location": "client or installer component", + "guard_scope": "target-client-native", + "guard_blocks_undeclared_capability": true, + "metadata_fallback_retained_for_other_targets": true, + "notes": "Use target-client-native for client-enforced guards or external-installer-runtime-guard for a real external installer guard." + }, + "privacy": { + "raw_user_content_included": false, + "raw_provider_prompt_included": false, + "credentials_included": false, + "secrets_included": false, + "notes": "Runtime guard proof should avoid environment dumps." + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "attestation": { + "real_external_or_human_evidence": false, + "reviewer_or_operator_identity_present": false, + "artifact_refs_reviewed": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "notes": "Copy this template into submissions/ only after a real client or installer guard exists." + } +} diff --git a/evidence/world_class/templates/provider-holdout.intake.json b/evidence/world_class/templates/provider-holdout.intake.json new file mode 100644 index 00000000..631982b8 --- /dev/null +++ b/evidence/world_class/templates/provider-holdout.intake.json @@ -0,0 +1,50 @@ +{ + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": true, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "operator with provider credentials", + "submitted_at": "YYYY-MM-DD", + "summary": "Provider-backed holdout run with model, timing, token metadata, and aggregate output hashes.", + "artifact_refs": [ + { + "path": "reports/output_execution_runs.json", + "kind": "aggregate-report", + "contains_raw_content": false, + "note": "Must show model_executed_count > 0 and token_observed_count > 0." + } + ], + "provenance": { + "provider": "", + "model": "", + "run_command": "python3 scripts/yao.py output-exec --provider-runner --provider-model --timeout-seconds 60", + "credential_env": "", + "model_env": "YAO_OUTPUT_EVAL_MODEL", + "model_default": "", + "credential_material_committed": false + }, + "privacy": { + "raw_user_content_included": false, + "raw_provider_prompt_included": false, + "credentials_included": false, + "secrets_included": false, + "notes": "Commit aggregate metadata and hashes only." + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "attestation": { + "real_external_or_human_evidence": false, + "reviewer_or_operator_identity_present": false, + "artifact_refs_reviewed": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "notes": "Copy this template into submissions/ and set all booleans truthfully after the real provider run." + } +} diff --git a/examples/complex-release-orchestrator/generated-skill/reports/context_budget.json b/examples/complex-release-orchestrator/generated-skill/reports/context_budget.json index 5c287256..660f62f5 100644 --- a/examples/complex-release-orchestrator/generated-skill/reports/context_budget.json +++ b/examples/complex-release-orchestrator/generated-skill/reports/context_budget.json @@ -6,10 +6,47 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 718, - "other_text_tokens": 1946, + "other_text_tokens": 1842, "estimated_initial_load_tokens": 790, - "estimated_total_text_tokens": 2664, - "relevant_file_count": 12, + "estimated_total_text_tokens": 2560, + "deferred_resource_tokens": 1657, + "deferred_resource_warn_threshold": 120000, + "deferred_resource_dirs": [ + { + "path": "references", + "estimated_tokens": 554, + "file_count": 4 + }, + { + "path": "scripts", + "estimated_tokens": 367, + "file_count": 1 + }, + { + "path": "input", + "estimated_tokens": 315, + "file_count": 1 + }, + { + "path": "outputs", + "estimated_tokens": 263, + "file_count": 1 + }, + { + "path": "evals", + "estimated_tokens": 158, + "file_count": 1 + } + ], + "large_deferred_resource_dirs": [], + "deferred_resource_governance": { + "status": "not-required", + "large_dir_count": 0, + "governed_large_dir_count": 0, + "directories": [], + "summary": "No large deferred resource directory exceeds the per-dir threshold." + }, + "relevant_file_count": 11, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 164.6 diff --git a/examples/governed-incident-command/generated-skill/reports/context_budget.json b/examples/governed-incident-command/generated-skill/reports/context_budget.json index 98b114e9..c0471b88 100644 --- a/examples/governed-incident-command/generated-skill/reports/context_budget.json +++ b/examples/governed-incident-command/generated-skill/reports/context_budget.json @@ -6,10 +6,47 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 658, - "other_text_tokens": 1603, + "other_text_tokens": 1499, "estimated_initial_load_tokens": 760, - "estimated_total_text_tokens": 2261, - "relevant_file_count": 13, + "estimated_total_text_tokens": 2157, + "deferred_resource_tokens": 1030, + "deferred_resource_warn_threshold": 120000, + "deferred_resource_dirs": [ + { + "path": "scripts", + "estimated_tokens": 278, + "file_count": 1 + }, + { + "path": "references", + "estimated_tokens": 266, + "file_count": 3 + }, + { + "path": "outputs", + "estimated_tokens": 203, + "file_count": 1 + }, + { + "path": "input", + "estimated_tokens": 153, + "file_count": 1 + }, + { + "path": "evals", + "estimated_tokens": 130, + "file_count": 1 + } + ], + "large_deferred_resource_dirs": [], + "deferred_resource_governance": { + "status": "not-required", + "large_dir_count": 0, + "governed_large_dir_count": 0, + "directories": [], + "summary": "No large deferred resource directory exceeds the per-dir threshold." + }, + "relevant_file_count": 12, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 171.1 diff --git a/examples/governed-incident-command/optimization/reports/description_optimization.json b/examples/governed-incident-command/optimization/reports/description_optimization.json index 6af793f3..8707247d 100644 --- a/examples/governed-incident-command/optimization/reports/description_optimization.json +++ b/examples/governed-incident-command/optimization/reports/description_optimization.json @@ -8,19 +8,19 @@ "estimated_tokens": 37, "dev": { "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.667, + "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, - 1, + 0, + -1.0, -1.0, -1.0, -1.0, - -0.667, 37 ], "holdout": { @@ -39,14 +39,14 @@ "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.667, + "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 3, - "passed": 2, - "pass_rate": 0.667 + "passed": 3, + "pass_rate": 1.0 }, "should_not_trigger": { "total": 3, @@ -69,10 +69,10 @@ }, "governed_workflow": { "total": 2, - "passed": 1, + "passed": 2, "false_positives": 0, - "false_negatives": 1, - "pass_rate": 0.5 + "false_negatives": 0, + "pass_rate": 1.0 }, "explain_only": { "total": 1, @@ -110,21 +110,7 @@ "pass_rate": 1.0 } }, - "misfires": [ - { - "bucket": "should_trigger", - "family": "governed_workflow", - "prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.", - "score": 0.392, - "reason": "false_negative", - "matched_desired_concepts": [ - "governed_workflow", - "reusable_packaging", - "severity_assessment" - ], - "matched_negative_concepts": [] - } - ], + "misfires": [], "results": { "should_trigger": [ { @@ -212,7 +198,8 @@ "outage workflow" ], "incident_packet": [ - "incident review skill" + "incident review skill", + "incident review" ], "reusable_packaging": [ "reusable", @@ -221,7 +208,8 @@ ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "incident review" ] }, "negative": {} @@ -231,10 +219,10 @@ { "prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.", "family": "governed_workflow", - "score": 0.392, - "predicted_trigger": false, + "score": 0.539, + "predicted_trigger": true, "expected_trigger": true, - "passed": false, + "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ @@ -246,6 +234,7 @@ "reusable_packaging" ], "matched_desired_concepts": [ + "comms_and_owners", "governed_workflow", "reusable_packaging", "severity_assessment" @@ -253,13 +242,16 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.32, + "semantic_coverage": 0.48, "support_score": 0, "lexical_support": 0.467, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { + "comms_and_owners": [ + "owners" + ], "governed_workflow": [ "maintained", "operational skill" @@ -394,7 +386,7 @@ { "prompt": "Summarize this incident but do not package it as a reusable skill.", "family": "summary_only", - "score": 0.07, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, @@ -412,12 +404,16 @@ "reusable_packaging" ], "extra_positive_concepts": [], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.417, - "negative_penalty": 0, + "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { @@ -427,7 +423,11 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } } }, @@ -467,7 +467,8 @@ "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "reusable_packaging": [ "workflow" @@ -488,14 +489,14 @@ "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, - "false_negatives": 2, - "precision": null, - "recall": 0.0, + "false_negatives": 1, + "precision": 1.0, + "recall": 0.5, "bucket_stats": { "should_trigger": { "total": 2, - "passed": 0, - "pass_rate": 0.0 + "passed": 1, + "pass_rate": 0.5 }, "should_not_trigger": { "total": 2, @@ -511,10 +512,10 @@ "family_stats": { "governed_workflow": { "total": 1, - "passed": 0, + "passed": 1, "false_positives": 0, - "false_negatives": 1, - "pass_rate": 0.0 + "false_negatives": 0, + "pass_rate": 1.0 }, "packet_assembly": { "total": 1, @@ -553,19 +554,6 @@ } }, "misfires": [ - { - "bucket": "should_trigger", - "family": "governed_workflow", - "prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.", - "score": 0.399, - "reason": "false_negative", - "matched_desired_concepts": [ - "governed_workflow", - "reusable_packaging", - "severity_assessment" - ], - "matched_negative_concepts": [] - }, { "bucket": "should_trigger", "family": "packet_assembly", @@ -584,10 +572,10 @@ { "prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.", "family": "governed_workflow", - "score": 0.399, - "predicted_trigger": false, + "score": 0.694, + "predicted_trigger": true, "expected_trigger": true, - "passed": false, + "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ @@ -600,13 +588,14 @@ ], "matched_desired_concepts": [ "governed_workflow", + "incident_packet", "reusable_packaging", "severity_assessment" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.32, + "semantic_coverage": 0.64, "support_score": 0, "lexical_support": 0.562, "negative_penalty": 0, @@ -614,19 +603,23 @@ "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "standardize" + ], + "incident_packet": [ + "incident review" ], "reusable_packaging": [ "workflow" ], "severity_assessment": [ - "severity" + "severity", + "incident review" ] }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.", @@ -829,19 +822,19 @@ "estimated_tokens": 37, "dev": { "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.667, + "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, - 1, + 0, + -1.0, -1.0, -1.0, -1.0, - -0.667, 37 ], "holdout": { @@ -857,14 +850,14 @@ "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.667, + "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 3, - "passed": 2, - "pass_rate": 0.667 + "passed": 3, + "pass_rate": 1.0 }, "should_not_trigger": { "total": 3, @@ -880,10 +873,10 @@ "family_stats": { "packet_assembly": { "total": 1, - "passed": 0, + "passed": 1, "false_positives": 0, - "false_negatives": 1, - "pass_rate": 0.0 + "false_negatives": 0, + "pass_rate": 1.0 }, "governed_workflow": { "total": 2, @@ -928,49 +921,38 @@ "pass_rate": 1.0 } }, - "misfires": [ - { - "bucket": "should_trigger", - "family": "packet_assembly", - "prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.", - "score": 0.348, - "reason": "false_negative", - "matched_desired_concepts": [ - "governed_workflow" - ], - "matched_negative_concepts": [] - } - ], + "misfires": [], "results": { "should_trigger": [ { "prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.", "family": "packet_assembly", - "score": 0.348, - "predicted_trigger": false, + "score": 0.617, + "predicted_trigger": true, "expected_trigger": true, - "passed": false, + "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ - "governed_workflow" + "governed_workflow", + "incident_packet" ], "extra_positive_concepts": [ - "incident_inputs", - "incident_packet" + "incident_inputs" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.227, - "support_score": 0.52, + "semantic_coverage": 0.553, + "support_score": 0.2, "lexical_support": 0.231, "negative_penalty": 0, - "coverage_boost": 0.0, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "governed_workflow": [ @@ -992,39 +974,41 @@ { "prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.", "family": "governed_workflow", - "score": 0.521, + "score": 0.73, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ "incident_inputs", - "incident_packet", "reusable_packaging" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.409, - "support_score": 0.56, + "semantic_coverage": 0.658, + "support_score": 0.24, "lexical_support": 0.312, "negative_penalty": 0, - "coverage_boost": 0.0, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "incident_inputs": [ "outage workflow" ], "incident_packet": [ - "incident review skill" + "incident review skill", + "incident review" ], "reusable_packaging": [ "reusable", @@ -1033,29 +1017,31 @@ ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "incident review" ] }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.", "family": "governed_workflow", - "score": 0.651, + "score": 0.619, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ + "comms_and_owners", "governed_workflow", "severity_assessment" ], @@ -1064,13 +1050,16 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.636, + "semantic_coverage": 0.579, "support_score": 0.04, "lexical_support": 0.2, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { + "comms_and_owners": [ + "owners" + ], "governed_workflow": [ "maintained", "operational skill" @@ -1092,13 +1081,14 @@ { "prompt": "Explain what incident severity means.", "family": "explain_only", - "score": 0.408, + "score": 0.25, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1109,7 +1099,7 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.409, + "semantic_coverage": 0.237, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, @@ -1122,8 +1112,7 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Draft one short outage update for customers.", @@ -1135,6 +1124,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1172,6 +1162,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1196,13 +1187,14 @@ { "prompt": "Summarize this incident but do not package it as a reusable skill.", "family": "summary_only", - "score": 0.017, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1211,12 +1203,16 @@ "extra_positive_concepts": [ "reusable_packaging" ], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "semantic_coverage": 0.0, "support_score": 0.04, "lexical_support": 0.083, - "negative_penalty": 0, + "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { @@ -1226,7 +1222,11 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } } }, @@ -1240,6 +1240,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1256,7 +1257,7 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.227, + "semantic_coverage": 0.132, "support_score": 0.04, "lexical_support": 0.286, "negative_penalty": 0.24, @@ -1264,7 +1265,8 @@ "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "reusable_packaging": [ "workflow" @@ -1365,19 +1367,21 @@ { "prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.", "family": "governed_workflow", - "score": 0.66, + "score": 0.821, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ "governed_workflow", + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ @@ -1385,21 +1389,26 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.636, + "semantic_coverage": 0.789, "support_score": 0.04, "lexical_support": 0.312, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "standardize" + ], + "incident_packet": [ + "incident review" ], "reusable_packaging": [ "workflow" ], "severity_assessment": [ - "severity" + "severity", + "incident review" ] }, "negative": {} @@ -1416,6 +1425,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1458,6 +1468,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1487,6 +1498,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1526,6 +1538,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1561,6 +1574,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1587,14 +1601,14 @@ "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.667, + "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 3, - "passed": 2, - "pass_rate": 0.667 + "passed": 3, + "pass_rate": 1.0 }, "should_not_trigger": { "total": 3, @@ -1610,10 +1624,10 @@ "family_stats": { "packet_assembly": { "total": 1, - "passed": 0, + "passed": 1, "false_positives": 0, - "false_negatives": 1, - "pass_rate": 0.0 + "false_negatives": 0, + "pass_rate": 1.0 }, "governed_workflow": { "total": 2, @@ -1658,49 +1672,38 @@ "pass_rate": 1.0 } }, - "misfires": [ - { - "bucket": "should_trigger", - "family": "packet_assembly", - "prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.", - "score": 0.348, - "reason": "false_negative", - "matched_desired_concepts": [ - "governed_workflow" - ], - "matched_negative_concepts": [] - } - ], + "misfires": [], "results": { "should_trigger": [ { "prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.", "family": "packet_assembly", - "score": 0.348, - "predicted_trigger": false, + "score": 0.617, + "predicted_trigger": true, "expected_trigger": true, - "passed": false, + "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ - "governed_workflow" + "governed_workflow", + "incident_packet" ], "extra_positive_concepts": [ - "incident_inputs", - "incident_packet" + "incident_inputs" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.227, - "support_score": 0.52, + "semantic_coverage": 0.553, + "support_score": 0.2, "lexical_support": 0.231, "negative_penalty": 0, - "coverage_boost": 0.0, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "governed_workflow": [ @@ -1722,39 +1725,41 @@ { "prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.", "family": "governed_workflow", - "score": 0.521, + "score": 0.73, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ "incident_inputs", - "incident_packet", "reusable_packaging" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.409, - "support_score": 0.56, + "semantic_coverage": 0.658, + "support_score": 0.24, "lexical_support": 0.312, "negative_penalty": 0, - "coverage_boost": 0.0, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "incident_inputs": [ "outage workflow" ], "incident_packet": [ - "incident review skill" + "incident review skill", + "incident review" ], "reusable_packaging": [ "reusable", @@ -1763,29 +1768,31 @@ ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "incident review" ] }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.", "family": "governed_workflow", - "score": 0.651, + "score": 0.619, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ + "comms_and_owners", "governed_workflow", "severity_assessment" ], @@ -1794,13 +1801,16 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.636, + "semantic_coverage": 0.579, "support_score": 0.04, "lexical_support": 0.2, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { + "comms_and_owners": [ + "owners" + ], "governed_workflow": [ "maintained", "operational skill" @@ -1822,13 +1832,14 @@ { "prompt": "Explain what incident severity means.", "family": "explain_only", - "score": 0.408, + "score": 0.25, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1839,7 +1850,7 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.409, + "semantic_coverage": 0.237, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, @@ -1852,8 +1863,7 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Draft one short outage update for customers.", @@ -1865,6 +1875,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1902,6 +1913,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1926,13 +1938,14 @@ { "prompt": "Summarize this incident but do not package it as a reusable skill.", "family": "summary_only", - "score": 0.017, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1941,12 +1954,16 @@ "extra_positive_concepts": [ "reusable_packaging" ], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "semantic_coverage": 0.0, "support_score": 0.04, "lexical_support": 0.083, - "negative_penalty": 0, + "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { @@ -1956,7 +1973,11 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } } }, @@ -1970,6 +1991,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -1986,7 +2008,7 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.227, + "semantic_coverage": 0.132, "support_score": 0.04, "lexical_support": 0.286, "negative_penalty": 0.24, @@ -1994,7 +2016,8 @@ "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "reusable_packaging": [ "workflow" @@ -2095,19 +2118,21 @@ { "prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.", "family": "governed_workflow", - "score": 0.66, + "score": 0.821, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ "governed_workflow", + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ @@ -2115,21 +2140,26 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.636, + "semantic_coverage": 0.789, "support_score": 0.04, "lexical_support": 0.312, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "standardize" + ], + "incident_packet": [ + "incident review" ], "reusable_packaging": [ "workflow" ], "severity_assessment": [ - "severity" + "severity", + "incident review" ] }, "negative": {} @@ -2146,6 +2176,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2188,6 +2219,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2217,6 +2249,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2256,6 +2289,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2291,6 +2325,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2387,13 +2422,14 @@ { "prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.", "family": "blind_packet_assembly", - "score": 1.0, + "score": 0.994, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2401,22 +2437,22 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], - "extra_positive_concepts": [ - "incident_packet" - ], + "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 1.0, - "support_score": 0.32, + "support_score": 0, "lexical_support": 0.176, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -2435,19 +2471,21 @@ { "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", "family": "blind_governed_workflow", - "score": 0.804, + "score": 0.645, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ "comms_and_owners", + "governed_workflow", "severity_assessment" ], "extra_positive_concepts": [ @@ -2455,23 +2493,27 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.773, + "semantic_coverage": 0.579, "support_score": 0.04, "lexical_support": 0.533, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { "comms_and_owners": [ "incident communication" ], + "governed_workflow": [ + "standardize" + ], "reusable_packaging": [ "reusable", "workflow" ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "outage review" ] }, "negative": {} @@ -2483,13 +2525,14 @@ { "prompt": "Explain how severity levels are decided during incidents.", "family": "blind_explain_only", - "score": 0.386, + "score": 0.228, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2500,7 +2543,7 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.409, + "semantic_coverage": 0.237, "support_score": 0, "lexical_support": 0.125, "negative_penalty": 0, @@ -2513,8 +2556,7 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Write a single incident update for customers from these notes.", @@ -2526,6 +2568,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2550,13 +2593,14 @@ { "prompt": "Summarize the outage timeline, but do not package a governed workflow.", "family": "blind_summary_only", - "score": 0.276, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2568,17 +2612,22 @@ "incident_inputs", "reusable_packaging" ], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.227, + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.132, "support_score": 0.24, "lexical_support": 0.091, - "negative_penalty": 0, + "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "incident_inputs": [ "timeline" @@ -2588,7 +2637,11 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } } }, @@ -2602,6 +2655,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2706,13 +2760,14 @@ { "prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.", "family": "blind_packet_assembly", - "score": 1.0, + "score": 0.994, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2720,22 +2775,22 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], - "extra_positive_concepts": [ - "incident_packet" - ], + "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 1.0, - "support_score": 0.32, + "support_score": 0, "lexical_support": 0.176, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -2754,19 +2809,21 @@ { "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", "family": "blind_governed_workflow", - "score": 0.804, + "score": 0.645, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ "comms_and_owners", + "governed_workflow", "severity_assessment" ], "extra_positive_concepts": [ @@ -2774,23 +2831,27 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.773, + "semantic_coverage": 0.579, "support_score": 0.04, "lexical_support": 0.533, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { "comms_and_owners": [ "incident communication" ], + "governed_workflow": [ + "standardize" + ], "reusable_packaging": [ "reusable", "workflow" ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "outage review" ] }, "negative": {} @@ -2802,13 +2863,14 @@ { "prompt": "Explain how severity levels are decided during incidents.", "family": "blind_explain_only", - "score": 0.386, + "score": 0.228, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2819,7 +2881,7 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.409, + "semantic_coverage": 0.237, "support_score": 0, "lexical_support": 0.125, "negative_penalty": 0, @@ -2832,8 +2894,7 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Write a single incident update for customers from these notes.", @@ -2845,6 +2906,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2869,13 +2931,14 @@ { "prompt": "Summarize the outage timeline, but do not package a governed workflow.", "family": "blind_summary_only", - "score": 0.276, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2887,17 +2950,22 @@ "incident_inputs", "reusable_packaging" ], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.227, + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.132, "support_score": 0.24, "lexical_support": 0.091, - "negative_penalty": 0, + "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "incident_inputs": [ "timeline" @@ -2907,7 +2975,11 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } } }, @@ -2921,6 +2993,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -2955,14 +3028,14 @@ "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.5, + "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, - "passed": 1, - "pass_rate": 0.5 + "passed": 2, + "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, @@ -2985,10 +3058,10 @@ }, "blind_governed_workflow": { "total": 1, - "passed": 0, + "passed": 1, "false_positives": 0, - "false_negatives": 1, - "pass_rate": 0.0 + "false_negatives": 0, + "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, @@ -3019,21 +3092,7 @@ "pass_rate": 1.0 } }, - "misfires": [ - { - "bucket": "should_trigger", - "family": "blind_governed_workflow", - "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", - "score": 0.458, - "reason": "false_negative", - "matched_desired_concepts": [ - "comms_and_owners", - "reusable_packaging", - "severity_assessment" - ], - "matched_negative_concepts": [] - } - ], + "misfires": [], "results": { "should_trigger": [ { @@ -3070,7 +3129,8 @@ "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -3089,10 +3149,10 @@ { "prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.", "family": "blind_governed_workflow", - "score": 0.458, - "predicted_trigger": false, + "score": 0.55, + "predicted_trigger": true, "expected_trigger": true, - "passed": false, + "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ @@ -3105,13 +3165,14 @@ ], "matched_desired_concepts": [ "comms_and_owners", + "governed_workflow", "reusable_packaging", "severity_assessment" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.38, + "semantic_coverage": 0.48, "support_score": 0, "lexical_support": 0.6, "negative_penalty": 0, @@ -3121,13 +3182,17 @@ "comms_and_owners": [ "incident communication" ], + "governed_workflow": [ + "standardize" + ], "reusable_packaging": [ "reusable", "workflow" ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "outage review" ] }, "negative": {} @@ -3212,7 +3277,7 @@ { "prompt": "Summarize the outage timeline, but do not package a governed workflow.", "family": "blind_summary_only", - "score": 0.395, + "score": 0.005, "predicted_trigger": false, "expected_trigger": false, "passed": true, @@ -3232,17 +3297,22 @@ "reusable_packaging" ], "extra_positive_concepts": [], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "semantic_coverage": 0.34, "support_score": 0, "lexical_support": 0.273, - "negative_penalty": 0, + "negative_penalty": 0.24, "coverage_boost": 0.06, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "incident_inputs": [ "timeline" @@ -3252,10 +3322,13 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } - }, - "boundary_case": true + } }, { "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", @@ -3380,11 +3453,13 @@ "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "strong_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners" ], @@ -3403,7 +3478,8 @@ "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -3429,7 +3505,8 @@ "mode": "judge-rubric", "focused_positive_concepts": [ "severity_assessment", - "comms_and_owners" + "comms_and_owners", + "governed_workflow" ], "strong_positive_concepts": [ "severity_assessment", @@ -3446,22 +3523,26 @@ "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], - "positive_vote": 5, + "positive_vote": 6, "negative_vote": 0, - "margin": 5, - "confidence": 0.96, + "margin": 6, + "confidence": 1.0, "concept_evidence": { "positive": { "comms_and_owners": [ "incident communication" ], + "governed_workflow": [ + "standardize" + ], "reusable_packaging": [ "reusable", "workflow" ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "outage review" ] }, "negative": {} @@ -3556,20 +3637,23 @@ "input_hits": [], "artifact_hits": [], "capability_overlap": 1, - "exclusive_negative_concepts": [], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "summarize" ], "positive_vote": 1, - "negative_vote": 1, - "margin": 0, - "confidence": 0.5, + "negative_vote": 3, + "margin": -2, + "confidence": 0.58, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "incident_inputs": [ "timeline" @@ -3579,10 +3663,13 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } - }, - "boundary_case": true + } }, { "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", @@ -3624,7 +3711,7 @@ }, "judge_summary": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -3708,11 +3795,13 @@ "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "strong_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners" ], @@ -3731,7 +3820,8 @@ "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -3757,7 +3847,8 @@ "mode": "judge-rubric", "focused_positive_concepts": [ "severity_assessment", - "comms_and_owners" + "comms_and_owners", + "governed_workflow" ], "strong_positive_concepts": [ "severity_assessment", @@ -3774,22 +3865,26 @@ "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], - "positive_vote": 5, + "positive_vote": 6, "negative_vote": 0, - "margin": 5, - "confidence": 0.96, + "margin": 6, + "confidence": 1.0, "concept_evidence": { "positive": { "comms_and_owners": [ "incident communication" ], + "governed_workflow": [ + "standardize" + ], "reusable_packaging": [ "reusable", "workflow" ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "outage review" ] }, "negative": {} @@ -3884,20 +3979,23 @@ "input_hits": [], "artifact_hits": [], "capability_overlap": 1, - "exclusive_negative_concepts": [], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "summarize" ], "positive_vote": 1, - "negative_vote": 1, - "margin": 0, - "confidence": 0.5, + "negative_vote": 3, + "margin": -2, + "confidence": 0.58, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "incident_inputs": [ "timeline" @@ -3907,10 +4005,13 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } - }, - "boundary_case": true + } }, { "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", @@ -3952,7 +4053,7 @@ }, "judge_summary": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -4061,7 +4162,8 @@ "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -4088,6 +4190,7 @@ "focused_positive_concepts": [ "severity_assessment", "comms_and_owners", + "governed_workflow", "reusable_packaging" ], "strong_positive_concepts": [ @@ -4114,13 +4217,17 @@ "comms_and_owners": [ "incident communication" ], + "governed_workflow": [ + "standardize" + ], "reusable_packaging": [ "reusable", "workflow" ], "severity_assessment": [ "severity assessment", - "severity" + "severity", + "outage review" ] }, "negative": {} @@ -4219,20 +4326,23 @@ "input_hits": [], "artifact_hits": [], "capability_overlap": 1, - "exclusive_negative_concepts": [], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "summarize" ], "positive_vote": 3, - "negative_vote": 1, - "margin": 2, - "confidence": 0.66, + "negative_vote": 3, + "margin": 0, + "confidence": 0.58, "concept_evidence": { "positive": { "governed_workflow": [ - "governed" + "governed", + "governed workflow" ], "incident_inputs": [ "timeline" @@ -4242,9 +4352,14 @@ "package" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "do not package" + ] + } } - } + }, + "boundary_case": true }, { "prompt": "Brainstorm how incident handling could improve next quarter without building the packet.", @@ -4286,7 +4401,7 @@ }, "judge_summary": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -4371,6 +4486,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4378,23 +4494,24 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ - "incident_inputs", - "incident_packet" + "incident_inputs" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 1.0, - "support_score": 0.52, + "support_score": 0.2, "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -4419,19 +4536,21 @@ { "prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.", "family": "adversarial_operational_skill_route", - "score": 0.654, + "score": 0.815, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ "governed_workflow", + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ @@ -4439,23 +4558,28 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.636, + "semantic_coverage": 0.789, "support_score": 0.04, "lexical_support": 0.235, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { "governed_workflow": [ + "standardize", "maintained", "operational skill" ], + "incident_packet": [ + "incident review" + ], "reusable_packaging": [ "skill", "package" ], "severity_assessment": [ - "severity" + "severity", + "incident review" ] }, "negative": {} @@ -4474,6 +4598,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4502,13 +4627,14 @@ { "prompt": "Explain incident severity and summarize the outage timeline for training.", "family": "adversarial_explain_and_summary_collision", - "score": 0.032, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4525,7 +4651,7 @@ "exclusive_negative_concepts": [ "explain_only" ], - "semantic_coverage": 0.409, + "semantic_coverage": 0.237, "support_score": 0.2, "lexical_support": 0.2, "negative_penalty": 0.26, @@ -4552,39 +4678,61 @@ { "prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.", "family": "adversarial_future_playbook_not_packet", - "score": 0.021, + "score": 0.277, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], - "matched_desired_concepts": [], + "matched_desired_concepts": [ + "incident_packet", + "severity_assessment" + ], "extra_positive_concepts": [ "reusable_packaging" ], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.0, + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.658, "support_score": 0.04, "lexical_support": 0.143, - "negative_penalty": 0, - "coverage_boost": 0.0, + "negative_penalty": 0.24, + "coverage_boost": 0.04, "concept_evidence": { "positive": { + "incident_packet": [ + "incident review" + ], "reusable_packaging": [ "reusable", "workflow", "package" + ], + "severity_assessment": [ + "incident review" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "sketch a future", + "for later", + "do not package", + "do not package a reusable workflow" + ] + } } - } + }, + "boundary_case": true }, { "prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.", @@ -4596,6 +4744,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4713,6 +4862,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4720,23 +4870,24 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ - "incident_inputs", - "incident_packet" + "incident_inputs" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 1.0, - "support_score": 0.52, + "support_score": 0.2, "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -4761,19 +4912,21 @@ { "prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.", "family": "adversarial_operational_skill_route", - "score": 0.654, + "score": 0.815, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], "matched_desired_concepts": [ "governed_workflow", + "incident_packet", "severity_assessment" ], "extra_positive_concepts": [ @@ -4781,23 +4934,28 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.636, + "semantic_coverage": 0.789, "support_score": 0.04, "lexical_support": 0.235, "negative_penalty": 0, - "coverage_boost": 0.04, + "coverage_boost": 0.06, "concept_evidence": { "positive": { "governed_workflow": [ + "standardize", "maintained", "operational skill" ], + "incident_packet": [ + "incident review" + ], "reusable_packaging": [ "skill", "package" ], "severity_assessment": [ - "severity" + "severity", + "incident review" ] }, "negative": {} @@ -4816,6 +4974,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4844,13 +5003,14 @@ { "prompt": "Explain incident severity and summarize the outage timeline for training.", "family": "adversarial_explain_and_summary_collision", - "score": 0.032, + "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -4867,7 +5027,7 @@ "exclusive_negative_concepts": [ "explain_only" ], - "semantic_coverage": 0.409, + "semantic_coverage": 0.237, "support_score": 0.2, "lexical_support": 0.2, "negative_penalty": 0.26, @@ -4894,39 +5054,61 @@ { "prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.", "family": "adversarial_future_playbook_not_packet", - "score": 0.021, + "score": 0.277, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" ], - "matched_desired_concepts": [], + "matched_desired_concepts": [ + "incident_packet", + "severity_assessment" + ], "extra_positive_concepts": [ "reusable_packaging" ], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.0, + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.658, "support_score": 0.04, "lexical_support": 0.143, - "negative_penalty": 0, - "coverage_boost": 0.0, + "negative_penalty": 0.24, + "coverage_boost": 0.04, "concept_evidence": { "positive": { + "incident_packet": [ + "incident review" + ], "reusable_packaging": [ "reusable", "workflow", "package" + ], + "severity_assessment": [ + "incident review" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "sketch a future", + "for later", + "do not package", + "do not package a reusable workflow" + ] + } } - } + }, + "boundary_case": true }, { "prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.", @@ -4938,6 +5120,7 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ + "incident_packet", "severity_assessment", "comms_and_owners", "governed_workflow" @@ -5080,7 +5263,8 @@ "concept_evidence": { "positive": { "comms_and_owners": [ - "owner" + "owner", + "owner actions" ], "governed_workflow": [ "governed" @@ -5105,7 +5289,7 @@ { "prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.", "family": "adversarial_operational_skill_route", - "score": 0.387, + "score": 0.682, "predicted_trigger": true, "expected_trigger": true, "passed": true, @@ -5121,13 +5305,14 @@ ], "matched_desired_concepts": [ "governed_workflow", + "incident_packet", "reusable_packaging", "severity_assessment" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.32, + "semantic_coverage": 0.64, "support_score": 0, "lexical_support": 0.412, "negative_penalty": 0, @@ -5135,21 +5320,25 @@ "concept_evidence": { "positive": { "governed_workflow": [ + "standardize", "maintained", "operational skill" ], + "incident_packet": [ + "incident review" + ], "reusable_packaging": [ "skill", "package" ], "severity_assessment": [ - "severity" + "severity", + "incident review" ] }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ @@ -5246,7 +5435,7 @@ { "prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.", "family": "adversarial_future_playbook_not_packet", - "score": 0.071, + "score": 0.201, "predicted_trigger": false, "expected_trigger": false, "passed": true, @@ -5261,25 +5450,44 @@ "reusable_packaging" ], "matched_desired_concepts": [ - "reusable_packaging" + "incident_packet", + "reusable_packaging", + "severity_assessment" ], "extra_positive_concepts": [], - "matched_negative_concepts": [], - "exclusive_negative_concepts": [], - "semantic_coverage": 0.04, + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.54, "support_score": 0, "lexical_support": 0.429, - "negative_penalty": 0, - "coverage_boost": 0.0, + "negative_penalty": 0.24, + "coverage_boost": 0.06, "concept_evidence": { "positive": { + "incident_packet": [ + "incident review" + ], "reusable_packaging": [ "reusable", "workflow", "package" + ], + "severity_assessment": [ + "incident review" ] }, - "negative": {} + "negative": { + "brainstorm_only": [ + "sketch a future", + "for later", + "do not package", + "do not package a reusable workflow" + ] + } } } }, @@ -5341,19 +5549,19 @@ "estimated_tokens": 37, "dev": { "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.667, + "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, - 1, + 0, + -1.0, -1.0, -1.0, -1.0, - -0.667, 37 ], "holdout": { @@ -5365,6 +5573,102 @@ "should_not_trigger_pass_rate": 1.0 } }, + { + "id": "balanced", + "label": "Balanced", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "strategy": "balanced_template", + "estimated_tokens": 54, + "dev": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "selection_key": [ + 0, + 0, + -1.0, + -1.0, + -1.0, + -1.0, + 54 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 1, + "precision": 1.0, + "recall": 0.5, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "boundary", + "label": "Boundary", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "strategy": "boundary_template", + "estimated_tokens": 78, + "dev": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "selection_key": [ + 0, + 0, + -1.0, + -1.0, + -1.0, + -1.0, + 78 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 1, + "precision": 1.0, + "recall": 0.5, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "artifact_aware", + "label": "Artifact Aware", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", + "strategy": "artifact_template", + "estimated_tokens": 78, + "dev": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "selection_key": [ + 0, + 0, + -1.0, + -1.0, + -1.0, + -1.0, + 78 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 1, + "precision": 1.0, + "recall": 0.5, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, { "id": "guardrail", "label": "Guardrail", @@ -5388,102 +5692,6 @@ -0.667, 51 ], - "holdout": { - "false_positives": 1, - "false_negatives": 1, - "precision": 0.5, - "recall": 0.5, - "near_neighbor_pass_rate": 0.5, - "should_not_trigger_pass_rate": 1.0 - } - }, - { - "id": "balanced", - "label": "Balanced", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", - "strategy": "balanced_template", - "estimated_tokens": 54, - "dev": { - "false_positives": 0, - "false_negatives": 1, - "precision": 1.0, - "recall": 0.667, - "near_neighbor_pass_rate": 1.0, - "should_not_trigger_pass_rate": 1.0 - }, - "selection_key": [ - 0, - 1, - -1.0, - -1.0, - -1.0, - -0.667, - 54 - ], - "holdout": { - "false_positives": 0, - "false_negatives": 1, - "precision": 1.0, - "recall": 0.5, - "near_neighbor_pass_rate": 1.0, - "should_not_trigger_pass_rate": 1.0 - } - }, - { - "id": "boundary", - "label": "Boundary", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "strategy": "boundary_template", - "estimated_tokens": 78, - "dev": { - "false_positives": 0, - "false_negatives": 1, - "precision": 1.0, - "recall": 0.667, - "near_neighbor_pass_rate": 1.0, - "should_not_trigger_pass_rate": 1.0 - }, - "selection_key": [ - 0, - 1, - -1.0, - -1.0, - -1.0, - -0.667, - 78 - ], - "holdout": { - "false_positives": 0, - "false_negatives": 1, - "precision": 1.0, - "recall": 0.5, - "near_neighbor_pass_rate": 1.0, - "should_not_trigger_pass_rate": 1.0 - } - }, - { - "id": "artifact_aware", - "label": "Artifact Aware", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", - "strategy": "artifact_template", - "estimated_tokens": 78, - "dev": { - "false_positives": 0, - "false_negatives": 1, - "precision": 1.0, - "recall": 0.667, - "near_neighbor_pass_rate": 1.0, - "should_not_trigger_pass_rate": 1.0 - }, - "selection_key": [ - 0, - 1, - -1.0, - -1.0, - -1.0, - -0.667, - 78 - ], "holdout": { "false_positives": 0, "false_negatives": 1, @@ -5508,15 +5716,15 @@ "comparison": { "winner_vs_current_dev": { "baseline_false_positives": 0, - "baseline_false_negatives": 1, + "baseline_false_negatives": 0, "improved_false_positives": 0, - "improved_false_negatives": 1, + "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, - "baseline_recall": 0.667, - "improved_recall": 0.667 + "baseline_recall": 1.0, + "improved_recall": 1.0 }, "winner_vs_current_holdout": { "baseline_false_positives": 0, @@ -5543,6 +5751,18 @@ "improved_recall": 1.0 }, "winner_vs_baseline_dev": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 1, "improved_false_positives": 0, @@ -5551,31 +5771,19 @@ "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, - "baseline_recall": 0.667, - "improved_recall": 0.667 - }, - "winner_vs_baseline_holdout": { - "baseline_false_positives": 0, - "baseline_false_negatives": 2, - "improved_false_positives": 0, - "improved_false_negatives": 1, - "false_positive_delta": 0, - "false_negative_delta": -1, - "baseline_precision": null, - "improved_precision": 1.0, - "baseline_recall": 0.0, + "baseline_recall": 0.5, "improved_recall": 0.5 }, "winner_vs_baseline_blind_holdout": { "baseline_false_positives": 0, - "baseline_false_negatives": 1, + "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, - "false_negative_delta": -1, + "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, - "baseline_recall": 0.5, + "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_judge_blind_holdout": { @@ -5648,15 +5856,15 @@ }, "baseline": { "false_positives": 0, - "false_negatives": 2, - "precision": null, - "recall": 0.0, + "false_negatives": 1, + "precision": 1.0, + "recall": 0.5, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -5671,7 +5879,7 @@ }, "current_calibration": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -5686,7 +5894,7 @@ }, "baseline_calibration": { "threshold": 0.48, - "mean_positive_score": 0.36, + "mean_positive_score": 0.507, "mean_non_trigger_score": 0.059, "mean_near_neighbor_score": 0.119, "min_positive_score": 0.321, @@ -5695,8 +5903,8 @@ "threshold_margin": -0.159, "positive_threshold_buffer": -0.159, "negative_threshold_buffer": 0.26, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "overlap" }, "winner_family_health": { @@ -5733,13 +5941,8 @@ }, "baseline_family_health": { "family_count": 6, - "clean_family_count": 4, + "clean_family_count": 5, "failing_families": [ - { - "family": "governed_workflow", - "errors": 1, - "pass_rate": 0.0 - }, { "family": "packet_assembly", "errors": 1, @@ -5772,56 +5975,56 @@ }, "baseline": { "false_positives": 0, - "false_negatives": 1, + "false_negatives": 0, "precision": 1.0, - "recall": 0.5, + "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "baseline_calibration": { "threshold": 0.48, - "mean_positive_score": 0.625, - "mean_non_trigger_score": 0.151, - "mean_near_neighbor_score": 0.198, - "min_positive_score": 0.458, - "max_non_trigger_score": 0.395, - "score_gap": 0.063, - "threshold_margin": -0.022, - "positive_threshold_buffer": -0.022, - "negative_threshold_buffer": 0.085, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "overlap" + "mean_positive_score": 0.671, + "mean_non_trigger_score": 0.053, + "mean_near_neighbor_score": 0.003, + "min_positive_score": 0.55, + "max_non_trigger_score": 0.176, + "score_gap": 0.374, + "threshold_margin": 0.07, + "positive_threshold_buffer": 0.07, + "negative_threshold_buffer": 0.304, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" }, "winner_family_health": { "family_count": 6, @@ -5845,18 +6048,12 @@ }, "baseline_family_health": { "family_count": 6, - "clean_family_count": 5, - "failing_families": [ - { - "family": "blind_governed_workflow", - "errors": 1, - "pass_rate": 0.0 - } - ], + "clean_family_count": 6, + "failing_families": [], "weakest_family": { - "family": "blind_governed_workflow", - "pass_rate": 0.0, - "errors": 1 + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 } } }, @@ -5870,7 +6067,7 @@ "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -5883,7 +6080,7 @@ "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -5896,7 +6093,7 @@ "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -5961,49 +6158,49 @@ }, "winner_calibration": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" - }, - "current_calibration": { - "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" - }, - "baseline_calibration": { - "threshold": 0.34, - "mean_positive_score": 0.689, - "mean_non_trigger_score": 0.074, - "mean_near_neighbor_score": 0.035, - "min_positive_score": 0.387, - "max_non_trigger_score": 0.215, - "score_gap": 0.172, - "threshold_margin": 0.047, - "positive_threshold_buffer": 0.047, - "negative_threshold_buffer": 0.125, + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, "boundary_case_count": 1, "boundary_case_rate": 0.167, "risk_band": "watch" }, + "current_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" + }, + "baseline_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.836, + "mean_non_trigger_score": 0.107, + "mean_near_neighbor_score": 0.101, + "min_positive_score": 0.682, + "max_non_trigger_score": 0.215, + "score_gap": 0.467, + "threshold_margin": 0.125, + "positive_threshold_buffer": 0.342, + "negative_threshold_buffer": 0.125, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, "winner_family_health": { "family_count": 6, "clean_family_count": 6, @@ -6040,8 +6237,8 @@ "winner_label": "Current", "winner_tokens": 37, "current_tokens": 37, - "winner_dev_total_errors": 1, - "current_dev_total_errors": 1, + "winner_dev_total_errors": 0, + "current_dev_total_errors": 0, "winner_holdout_total_errors": 1, "current_holdout_total_errors": 1, "winner_blind_holdout_total_errors": 0, @@ -6051,13 +6248,13 @@ "winner_adversarial_holdout_total_errors": 0, "current_adversarial_holdout_total_errors": 0, "winner_judge_blind_agreement_rate": 1.0, - "winner_adversarial_risk_band": "healthy", - "winner_adversarial_score_gap": 0.598, + "winner_adversarial_risk_band": "watch", + "winner_adversarial_score_gap": 0.538, "candidate_count": 5, "baseline_tokens": 93, - "baseline_dev_total_errors": 1, - "baseline_holdout_total_errors": 2, - "baseline_blind_holdout_total_errors": 1, + "baseline_dev_total_errors": 0, + "baseline_holdout_total_errors": 1, + "baseline_blind_holdout_total_errors": 0, "baseline_judge_blind_holdout_total_errors": 0, "baseline_adversarial_holdout_total_errors": 0 } diff --git a/examples/governed-incident-command/optimization/reports/description_optimization.md b/examples/governed-incident-command/optimization/reports/description_optimization.md index 154dab1a..874d859a 100644 --- a/examples/governed-incident-command/optimization/reports/description_optimization.md +++ b/examples/governed-incident-command/optimization/reports/description_optimization.md @@ -14,18 +14,18 @@ Build governed incident command packets. Use when asked to standardize incident | Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `Current` | 37 | 0 | 1 | 1.0 | 0 | 1 | -| `Guardrail` | 51 | 0 | 1 | 1.0 | 1 | 1 | -| `Balanced` | 54 | 0 | 1 | 1.0 | 0 | 1 | -| `Boundary` | 78 | 0 | 1 | 1.0 | 0 | 1 | -| `Artifact Aware` | 78 | 0 | 1 | 1.0 | 0 | 1 | +| `Current` | 37 | 0 | 0 | 1.0 | 0 | 1 | +| `Balanced` | 54 | 0 | 0 | 1.0 | 0 | 1 | +| `Boundary` | 78 | 0 | 0 | 1.0 | 0 | 1 | +| `Artifact Aware` | 78 | 0 | 0 | 1.0 | 0 | 1 | +| `Guardrail` | 51 | 0 | 1 | 1.0 | 0 | 1 | ## Acceptance Gates | Gate | Winner FP | Winner FN | Current FP | Current FN | Baseline FP | Baseline FN | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| Holdout | 0 | 1 | 0 | 1 | 0 | 2 | -| Blind Holdout | 0 | 0 | 0 | 0 | 0 | 1 | +| Holdout | 0 | 1 | 0 | 1 | 0 | 1 | +| Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Judge Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | @@ -34,21 +34,21 @@ Build governed incident command packets. Use when asked to standardize incident | Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap | | --- | ---: | --- | ---: | ---: | ---: | | Holdout | 0.023 | overlap | 0.0 | 0.023 | 0.101 | -| Blind Holdout | 0.418 | healthy | 0.167 | 0.418 | 0.063 | -| Adversarial Holdout | 0.598 | healthy | 0.0 | 0.598 | 0.172 | +| Blind Holdout | 0.417 | healthy | 0.0 | 0.417 | 0.374 | +| Adversarial Holdout | 0.538 | watch | 0.167 | 0.538 | 0.467 | ## Judge Blind Summary | Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement | | --- | ---: | ---: | ---: | ---: | -| Judge Blind Holdout | 1.0 | 0.657 | 1.0 | 1.0 | +| Judge Blind Holdout | 1.0 | 0.677 | 1.0 | 1.0 | ## Family Health | Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | | --- | --- | --- | --- | --- | -| Holdout | 5/6 | packet_assembly (1 errors) | 5/6 | 4/6 | -| Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 5/6 | +| Holdout | 5/6 | packet_assembly (1 errors) | 5/6 | 5/6 | +| Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 6/6 | | Judge Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 6/6 | | Adversarial Holdout | 6/6 | adversarial_single_update_collision (0 errors) | 6/6 | 6/6 | diff --git a/examples/governed-incident-command/optimization/semantic_config.json b/examples/governed-incident-command/optimization/semantic_config.json index 8cd90699..20cdcf21 100644 --- a/examples/governed-incident-command/optimization/semantic_config.json +++ b/examples/governed-incident-command/optimization/semantic_config.json @@ -37,9 +37,12 @@ "weight": 0.32, "phrases": [ "incident command packet", + "incident command packets", "governed incident packet", "incident packet assembly", - "incident review skill" + "incident review skill", + "incident review", + "standardize incident review" ] }, "incident_inputs": { @@ -59,6 +62,8 @@ "severity assessment", "severity", "impact scope", + "incident review", + "outage review", "sev" ] }, @@ -68,6 +73,8 @@ "stakeholder update", "incident communication", "owner", + "owners", + "owner actions", "action ownership", "next actions" ] @@ -76,6 +83,9 @@ "weight": 0.1, "phrases": [ "governed", + "governed workflow", + "reusable governed workflow", + "standardize", "reviewed", "audited", "maintained", @@ -128,9 +138,13 @@ "exclusive": true, "phrases": [ "brainstorm", + "sketch a future", + "for later", "possible response ideas", "improve incident handling", - "ideas for later" + "ideas for later", + "do not package", + "do not package a reusable workflow" ] }, "summary_only": { diff --git a/manifest.json b/manifest.json index 91767255..317a1a11 100644 --- a/manifest.json +++ b/manifest.json @@ -12,7 +12,8 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "factory_components": [ "templates", diff --git a/references/autonomous-adaptation.md b/references/autonomous-adaptation.md new file mode 100644 index 00000000..6f3684dd --- /dev/null +++ b/references/autonomous-adaptation.md @@ -0,0 +1,65 @@ +# Autonomous Adaptation Method + +This reference defines the safe foundation for adaptive self-iteration. + +## Scope + +Adaptive iteration is proposal-only until a human explicitly approves a patch application workflow. The current implementation may: + +- read one user-provided local source file; +- redact sensitive text before storing evidence excerpts; +- summarize repeated preferences and operational signals; +- produce adaptation proposals with target files, risks, tests, and rollback plans. +- draft a pending approval ledger entry from a reviewed patch, including patch SHA-256, target files, target baseline hashes, verifier commands, and rollback metadata. +- dry-run an approved patch through `adapt-apply`, after patch hash, approval, target allowlist, and target baseline hash checks pass. +- apply a patch only when the operator passes `--apply` and the approval ledger names the reviewer, reason, patch hash, target files, target file SHA-256 baselines, verification commands, and rollback plan. +- automatically reverse an applied patch when `--run-verification` fails, unless the operator explicitly passes `--no-rollback-on-failure`. + +It must not: + +- scan shell history, browser history, chat logs, mail, or private folders by default; +- infer permanent user memory from a single comment; +- write source files as part of scan or proposal generation; +- write source files through `adapt-apply` without explicit `--apply`; +- apply a patch whose target files are outside both the proposal and approval allowlists; +- apply a patch when an approved target file has changed since the reviewer recorded its baseline SHA-256; +- leave a failed verified apply in place by default; +- count proposals as completed implementation evidence. + +## Flow + +1. `adapt-scan` reads an explicit source path and writes `reports/user_patterns.json` plus `reports/user_patterns.md`. +2. `adapt-propose` reads the pattern report and writes `reports/adaptation_proposals.json` plus `reports/adaptation_proposals.md`. +3. A reviewer decides whether any proposal is worth implementing. +4. `adapt-apply --write-template` creates `reports/adaptation_approval_ledger.json` and `reports/adaptation_regression_report.json` so the review surface exists before any patch is applied. +5. `adapt-apply --prepare-approval --proposal-id --patch-file ` drafts a `pending-review` approval entry. It does not approve or apply the patch. +6. A human reviewer changes the draft decision to `approved`, fills reviewer, reason, approval date, and optional expiry, then keeps the generated patch and target baseline hashes intact. +7. `adapt-apply --proposal-id --patch-file ` defaults to a dry-run and records patch, target, approval, regression, and rollback evidence. +8. `adapt-apply --apply --run-verification` may write files only after approval, patch hash, allowlist, target baseline hash, `git apply --check`, and safe regression command checks pass. +9. If a verification command fails after a patch is applied, `adapt-apply` runs `git apply -R ` by default and records `failed-rolled-back` plus rollback evidence in `reports/adaptation_regression_report.json`. + +## Evidence Standard + +Each proposal should include: + +- the repeated pattern that triggered it; +- redacted excerpts, never unredacted raw content; +- target files and change intent; +- risk level and boundary; +- verification commands; +- rollback plan; +- a clear `proposal-only` status. + +Each approved application should include: + +- reviewer, reason, approval date, and optional expiry; +- exact patch SHA-256; +- target file allowlist; +- target file SHA-256 baselines for every patch target, or `__absent__` for approved new files; +- regression commands restricted to local `make` targets or local Python verifier scripts; +- rollback command or plan. +- rollback result if regression failed after an apply attempt. + +## Review Boundary + +The adaptive loop improves iteration quality, but it does not replace normal review. Any proposal touching trigger behavior, reports, packaging, telemetry, privacy, or governance must still pass the same tests and release gates as a manually designed change. `adapt-apply` evidence proves that an approved patch path was checked or applied; it does not make world-class external or human evidence complete. diff --git a/references/distribution-registry-method.md b/references/distribution-registry-method.md index 68bb0521..4511b58e 100644 --- a/references/distribution-registry-method.md +++ b/references/distribution-registry-method.md @@ -25,7 +25,7 @@ Do not publish a team package when registry audit reports missing version, hash, For an installable archive, first build the distribution and run package verification: ```bash -python3 scripts/yao.py package . --platform openai --platform claude --platform generic --output-dir dist --zip +python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip python3 scripts/yao.py package-verify . --package-dir dist --require-zip python3 scripts/yao.py install-simulate . --package-dir dist python3 scripts/yao.py registry-audit . @@ -36,6 +36,8 @@ Do not claim archive readiness when package verification reports unsafe zip path Do not claim install readiness when install simulation cannot extract the archive into a temporary skill root, load `SKILL.md` frontmatter, read `manifest.json`, read `agents/interface.yaml`, find the overview and Review Studio reports, or load each generated adapter. +Do not sync a local or active install from source until the same package has passed install preflight. `scripts/sync_local_install.py` must run install simulation against the configured package directory and fail before copying files when any target/capability pair lacks active permission approval or target-specific enforcement evidence. Use `--skip-install-preflight` only for isolated diagnostics, not for release or active install. + Do not include raw `reports/telemetry_events.jsonl` in a distributed package. Include only aggregate adoption drift reports, and block release review when telemetry contains raw prompts, outputs, transcripts, notes, or messages. Review waiver evidence may be distributed as `reports/review_waivers.md/json` because it is metadata-only reviewer accountability. Do not store raw prompts, outputs, transcripts, credentials, or private customer detail in waiver reasons. @@ -53,5 +55,6 @@ A reviewer should be able to answer: 5. Which reports prove trust and runtime readiness? 6. Was the installable archive verified, and which checksum identifies it? 7. Was the archive install-simulated in a temporary local skill root? -8. What changed since the previous package, and does the declared version bump match the recommended bump? -9. Are adoption and drift signals summarized without packaging raw local telemetry? +8. Did local or active install sync preserve that preflight and installer permission gate? +9. What changed since the previous package, and does the declared version bump match the recommended bump? +10. Are adoption and drift signals summarized without packaging raw local telemetry? diff --git a/references/operating-modes.md b/references/operating-modes.md index 766b4d1e..a6e3c9de 100644 --- a/references/operating-modes.md +++ b/references/operating-modes.md @@ -68,11 +68,37 @@ Minimum gates: - `trigger_eval.py` - `cross_packager.py` for requested targets +## Governed + +Use when: + +- the skill affects incident, release, compliance, security, or organizational standards +- external distribution, public claims, or high-permission scripts require reviewable evidence +- wrong output or wrong activation can cause operational, legal, trust, or reputational harm + +Default deliverables: + +- everything required for Library +- explicit owner, lifecycle, review cadence, and expiry-aware approvals +- trust/security reports for scripts, dependencies, permissions, secrets, and package hash +- output eval evidence with blind review status and reviewer-visible boundaries +- world-class or public-claim evidence ledger when public readiness is claimed + +Minimum gates: + +- Library gates +- `trust_check.py` +- runtime permission probes for packaged adapters +- review waiver ledger for accepted warning-level risk +- Review Studio before release +- claim guard before public world-class language + ## Escalation Rules - stay in Scaffold unless reuse is clearly real - move to Production when team reuse or route confusion matters -- move to Library when the skill becomes shared infrastructure or a governed asset +- move to Library when the skill becomes shared infrastructure +- move to Governed when the skill needs explicit risk ownership, high-permission review, or public-claim evidence ## Context Discipline diff --git a/references/output-eval-method.md b/references/output-eval-method.md index d9d999c3..dfb8d443 100644 --- a/references/output-eval-method.md +++ b/references/output-eval-method.md @@ -69,6 +69,16 @@ python3 scripts/yao.py output-exec --runner-command '["python3","scripts/local_o This verifies the command-runner contract, timing capture, grading path, and failure handling. It must not be described as provider-backed model evidence. +For provider-backed evidence, use the bundled provider runner with real credentials: + +```bash +YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini \ +OPENAI_API_KEY=... \ +python3 scripts/yao.py output-exec --provider-runner openai +``` + +The provider runner calls an OpenAI Responses API compatible endpoint, reads input files relative to `evals/output/`, returns `execution_kind: "model"`, and records observed token usage when the provider returns usage fields. If the API key or model is missing, the runner must fail instead of falling back to fixtures or pretending model evidence exists. Use `--provider-base-url` only for reviewed compatible endpoints; non-default HTTPS hosts require `--allow-custom-base-url`, and plain HTTP is allowed only with `--allow-insecure-localhost` for local test servers. + ## Blind A/B Review Every output eval run should also generate: @@ -81,7 +91,7 @@ The review pack must hide whether Variant A or Variant B came from the baseline ## Reviewer Adjudication -After blind review, record reviewer choices in `reports/output_review_decisions.json` and run: +After blind review, record reviewer choices in `reports/output_review_decisions.json` with `reviewer`, `reviewed_at`, `winner_variant`, optional `confidence`, and a required rubric-based `reason`, then run: ```bash python3 scripts/adjudicate_output_review.py --write-template @@ -90,10 +100,13 @@ python3 scripts/yao.py output-review The adjudication report writes: +- `reports/output_review_decisions.json` - `reports/output_review_adjudication.json` - `reports/output_review_adjudication.md` -When no reviewer decisions exist, the report should say the cases are pending. Do not count pending cases as human agreement. Only a real `winner_variant` of `A` or `B` should contribute to agreement rate, disagreement count, and reviewer judgment count. +When no reviewer decisions exist, the report should say the cases are pending and Review Studio should link to the decisions template. Do not count pending cases as human agreement. Only a real `winner_variant` of `A` or `B` with reviewer metadata and a non-empty `reason` should contribute to agreement rate, disagreement count, and reviewer judgment count. + +The adjudication report must preserve blind-review integrity: pending and invalid decisions should show the expected winner as hidden. Only reveal `expected_winner_variant` after a valid reviewer decision with rationale exists for that case. ## Anti-Overfitting diff --git a/references/platform-capability-matrix.md b/references/platform-capability-matrix.md index 101763db..57f5fa94 100644 --- a/references/platform-capability-matrix.md +++ b/references/platform-capability-matrix.md @@ -8,6 +8,7 @@ This matrix describes the current packaging targets and their support level. | `claude` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Generates `targets/claude/README.md` plus adapter metadata | | `generic` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Uses neutral adapter metadata only | | `agent-skills-compatible` | Neutral source | Yes | Yes | Source-compatible | Yes | activation, execution, trust, permissions, degradation, native behavior | Keeps canonical `SKILL.md` plus `agents/interface.yaml` source shape | +| `vscode` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior, install scope | Generates `targets/vscode/README.md` plus adapter metadata for VS Code / Copilot Agent Skills review | ## Current Support Model @@ -15,6 +16,7 @@ This matrix describes the current packaging targets and their support level. - `claude`: lightweight compatibility adapter with an explicit compiler contract and fallback notes. - `generic`: lowest-friction export for neutral Agent Skills consumers. - `agent-skills-compatible`: canonical source shape with compiler evidence for review and distribution. +- `vscode`: VS Code / Copilot Agent Skills adapter that preserves the neutral source package and documents user/project scope plus workspace-trust review notes. - runtime permission probes currently report metadata fallback for generated targets; no target is claimed as native-enforced until a client or installer integration can actually enforce the permission model. ## Portable Semantics diff --git a/references/review-studio-method.md b/references/review-studio-method.md index da67a4cb..4674fbd4 100644 --- a/references/review-studio-method.md +++ b/references/review-studio-method.md @@ -40,7 +40,7 @@ Production, library, and governed reviews should also show a blind A/B review pa When `reports/output_execution_runs.json` exists, Review Studio should show the number of variant runs, command-executed runs, model-executed runs, recorded fixtures, timing-observed runs, and token-estimated runs. Recorded fixtures are valid reproducibility evidence, but they must not be described as model-executed output evidence. -When `reports/output_review_adjudication.json` exists, Review Studio should show reviewed pairs and pending pairs. Pending reviewer decisions are acceptable as an explicit state, but they must not be counted as agreement or human review evidence. Invalid adjudication records should block release because they make the blind review audit untrustworthy. +When `reports/output_review_adjudication.json` exists, Review Studio should show reviewed pairs and pending pairs. Pending reviewer decisions are acceptable as an explicit state, but they must not be counted as agreement or human review evidence. For production, library, and governed packages, pending reviewer decisions should keep the Output Lab in `warn` until reviewer decisions are recorded or the warning is explicitly accepted in the waiver ledger. Invalid adjudication records should block release because they make the blind review audit untrustworthy. The Operations Loop must never display raw telemetry logs. It should link only to `reports/adoption_drift_report.md`; privacy or schema violations are blockers. @@ -65,10 +65,12 @@ Each action must include: - `evidence` - `verification_command` -`source_refs` must be structured entries with relative `path`, human label, kind, existence flag, best-effort line number, and relative link when the file exists. They should point to the smallest useful report or source file, not just a broad directory. +`source_refs` must be structured entries with relative `path`, human label, kind, existence flag, best-effort line number, matched pattern, short source excerpt, and relative link when the file exists. They should point to the smallest useful report or source file, not just a broad directory. The HTML page should render the excerpt next to the link so reviewers can understand why a line anchor matters before opening the full artifact. The HTML page should render these actions before the detailed supporting sections so a reviewer can move directly from warning to fix. Action entries do not change gate count or score; they make the current decision more operational. +For `world-class-evidence`, the action should also expose an evidence-step card for every pending evidence key. Each card should show the submission path, template path, blocked source checks, command handoff, first runbook steps, provenance requirements, success checks, evidence artifacts, and privacy boundary. These cards are collection guidance only; they must not count as accepted evidence or change world-class readiness. + ## Review Annotations `reports/review_annotations.json` is the structured ledger, and `reports/review_annotations.md` is the human-readable review note surface. Each annotation should include: diff --git a/references/review-waiver-method.md b/references/review-waiver-method.md index 5872d830..84d3fc78 100644 --- a/references/review-waiver-method.md +++ b/references/review-waiver-method.md @@ -25,6 +25,16 @@ Every waiver must include: - `evidence`: optional path or note that explains the decision. - `scope`: default `current-release`. +## Gate Key Policy + +The waiver ledger must track the Review Studio gate universe explicitly: + +- `review_studio_gate_keys`: every gate Review Studio can render. +- `waiverable_gate_keys`: warning gates that may receive bounded human acceptance. +- `non_waivable_gate_keys`: gates that must not be accepted through a waiver. + +When Review Studio adds or renames a gate, update the waiver gate policy and tests in the same change. `review-waivers` and `world-class-evidence` stay non-waivable: the first is the waiver mechanism itself, and the second can only be satisfied by accepted ledger evidence. + ## Release Semantics - Invalid waiver records block Review Studio. @@ -55,3 +65,12 @@ python3 scripts/yao.py review-waivers . \ For a non-governed release where `permission-gates` is only a warning, the same command can name `--gate-key permission-gates`. Governed releases must instead provide reviewer, scope, reason, expiry, evidence, and target-enforcement fields in `security/permission_policy.json`. Review Studio reads `reports/review_waivers.json` and links to `reports/review_waivers.md`. + +## Candidate Actions + +The waiver report also surfaces current candidate actions from local evidence: + +- waiverable warning candidates, such as an `output-lab` warning caused by pending reviewer decisions or missing provider-backed runs +- non-waivable boundaries, especially `world-class-evidence`, where pending ledger evidence cannot be converted into completion by a waiver + +A waiver can make a bounded warning auditable for a release window. It cannot count as provider-backed evidence, human adjudication, native runtime enforcement, external telemetry, or public world-class readiness. diff --git a/references/skill-atlas-method.md b/references/skill-atlas-method.md index a0f0455a..9c580434 100644 --- a/references/skill-atlas-method.md +++ b/references/skill-atlas-method.md @@ -16,6 +16,7 @@ Single-skill quality is not enough for a team library. A skill portfolio also ne - Flag missing owner or review metadata. - Flag stale skills based on `updated_at` and `review_cadence`. - Extract no-route opportunities from failure notes. +- Read aggregate adoption drift reports and flag telemetry drift without reading raw telemetry logs. ## Scope Policy @@ -23,6 +24,12 @@ Atlas keeps a full catalog, but release gates should distinguish actionable libr Use `skill_atlas/policy.json` to mark path prefixes as non-actionable when they are intentionally retained as examples, evolution snapshots, embedded generated skills, or validator fixtures. Non-actionable items still appear in the full report, route matrix, stale list, and owner gap list, but Review Studio should use the actionable counts for release readiness. +## Telemetry Link + +Atlas may read each skill's aggregate `reports/adoption_drift_report.json` to surface portfolio drift signals such as no telemetry for production/library/governed skills, missed triggers, bad outputs, missing resources, script errors, and review-overdue counts. It must not read or package `reports/telemetry_events.jsonl`; raw telemetry remains local-only evidence owned by the skill. + +Write drift output to `skill_atlas/drift_signals.json`. Non-actionable scopes stay visible in that file and in the HTML report, but only actionable drift signals should affect release readiness. + ## Reviewer Gate -Use Atlas before promoting a single skill into a shared library. If an actionable route collision, missing owner, or stale governed skill appears, fix the portfolio boundary before adding more local complexity to one skill. Non-actionable issues should stay visible as evidence, not as release blockers. +Use Atlas before promoting a single skill into a shared library. If an actionable route collision, missing owner, stale governed skill, or telemetry drift signal appears, fix the portfolio boundary before adding more local complexity to one skill. Non-actionable issues should stay visible as evidence, not as release blockers. diff --git a/references/skillops-decision-policy.md b/references/skillops-decision-policy.md new file mode 100644 index 00000000..9e3e7471 --- /dev/null +++ b/references/skillops-decision-policy.md @@ -0,0 +1,53 @@ +# SkillOps Decision Policy + +Use this policy when turning explicit-source conversation evidence into SkillOps opportunities, proposals, or release work. The goal is to make repeated user signals actionable without letting automation write durable instructions, skills, scripts, or evals without review. + +## Decision Order + +1. Classify the signal as no action, report only, Memory, AGENTS.md, existing Skill patch, candidate Skill, script, eval, report, merge, split, or archive. +2. Prefer the smallest durable surface that fixes the repeated friction. +3. Require evidence for every write action. +4. Require a proposal or approval ledger entry before any source-file write. +5. Map every proposed change to at least one verification command. + +## Score Bands + +SkillOps opportunities use a `0-100` score. Scores are advisory and never bypass approval. + +| Score | Decision | +| ---: | --- | +| `85-100` | Ready for approval review | +| `70-84` | Proposal review | +| `50-69` | Observe more evidence | +| `0-49` | Report only or no action | + +High-risk items stay proposal-only even when their score is high. + +## Action Mapping + +| Pattern | Default Action | Durable Surface | +| --- | --- | --- | +| `language_default` | Patch existing skill | Report template or artifact doctrine | +| `report_ui` | Patch existing skill | Report renderer, artifact doctrine, visual test | +| `approval_safety` | AGENTS update | Governance guidance or approval policy | +| `delivery_format` | Patch existing skill | CLI output, README, generated summary copy | +| `evidence_testing` | Add eval | Focused regression, report-quality, or release gate | +| Unknown pattern | Report only | Manual review queue | + +## Safety Rules + +- Do not scan private logs implicitly; only use explicit user-supplied sources. +- Do not store raw conversation content in reports; use redacted excerpts and aggregate counts. +- Do not write source files from a daily report run. +- Do not count SkillOps reports as public world-class evidence. +- Do not treat planned work, draft submissions, or generated proposals as accepted evidence. + +## Verification + +Every implementation that changes this policy should run: + +```bash +python3 tests/verify_skillops_opportunity.py +python3 tests/verify_daily_skillops.py +python3 tests/verify_yao_cli.py +``` diff --git a/references/telemetry-drift-method.md b/references/telemetry-drift-method.md index 86331084..32e1bf1d 100644 --- a/references/telemetry-drift-method.md +++ b/references/telemetry-drift-method.md @@ -17,6 +17,8 @@ The local event stream is `reports/telemetry_events.jsonl`. It is intentionally "event": "skill_activation", "skill": "example-skill", "version": "2.0.0", + "source": "yao_cli", + "command": "quickstart", "activation_type": "implicit", "outcome": "accepted", "failure_type": "none", @@ -26,10 +28,125 @@ The local event stream is `reports/telemetry_events.jsonl`. It is intentionally Allowed events: `skill_activation`, `skill_output`, `script_run`, `review_event`. +Allowed sources: `manual`, `yao_cli`, `external`, `unknown`. + Allowed outcomes: `accepted`, `edited`, `rejected`, `missed`, `failed`, `reviewed`, `unknown`. Allowed failure types: `wrong_trigger`, `under_trigger`, `bad_output`, `missing_resource`, `script_error`, `review_overdue`, `none`. +`source` and `command` are metadata fields. They may identify that `yao.py` ran `quickstart`, `validate`, `output-exec`, or another subcommand, but they must not include arguments, prompt text, file content, model output, transcripts, or reviewer notes. + +## CLI Capture + +`scripts/yao.py` can record metadata-only `script_run` events automatically. It is opt-in to keep release evidence reproducible and avoid surprising local writes: + +```bash +YAO_CLI_TELEMETRY=1 python3 scripts/yao.py validate . +``` + +Optional destination override: + +```bash +YAO_CLI_TELEMETRY=1 \ +YAO_CLI_TELEMETRY_EVENTS=/tmp/yao-telemetry.jsonl \ +python3 scripts/yao.py output-exec +``` + +Equivalent global flags are available before the subcommand: + +```bash +python3 scripts/yao.py --record-cli-telemetry validate . +python3 scripts/yao.py --no-cli-telemetry validate . +``` + +Successful CLI runs record `event=script_run`, `source=yao_cli`, `outcome=accepted`, and `failure_type=none`. Failed CLI runs record `outcome=failed` and `failure_type=script_error`. The command name is normalized to the subcommand only; command arguments are never recorded. + +## External Client Emit + +External clients, browser extensions, editor adapters, or wrapper scripts can emit one sanitized event at a time into a local spool before importing it into the aggregate drift report: + +```bash +python3 scripts/yao.py telemetry-emit . \ + --event skill_activation \ + --activation-type explicit \ + --outcome accepted \ + --command browser-extension +``` + +By default this writes to `.yao/telemetry_spool/external_events.jsonl`. Use `--output-jsonl` when a client needs a different local handoff path: + +```bash +python3 scripts/yao.py telemetry-emit . \ + --output-jsonl /tmp/external-client-events.jsonl \ + --event skill_output \ + --activation-type manual \ + --outcome edited \ + --command browser-plugin +``` + +Use `--dry-run` to validate a proposed event without writing to the spool. The emitter uses the same metadata-only contract as import: no prompt, input, output, transcript, message, note, raw text, arguments, or unknown fields are accepted. + +After a client finishes a batch, import the spool: + +```bash +python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl +``` + +## Client Hook Recipes + +Use `telemetry-hooks` to generate auditable Browser, Chrome, VS Code, CLI wrapper, and provider-adapter hook recipes: + +```bash +python3 scripts/yao.py telemetry-hooks . +``` + +The report is written to: + +- `reports/telemetry_hook_recipes.json` +- `reports/telemetry_hook_recipes.md` + +Each recipe includes a dry-run command, an emit command, the target local spool, trigger points, and the privacy contract. The report intentionally sets `native_auto_capture=false`; it proves the local hook contract and metadata-only command shape, not that a host client is already natively integrated. + +## Browser Native Host + +`scripts/telemetry_native_host.py` implements the local side of Browser/Chrome Native Messaging. It accepts length-prefixed JSON messages on stdio, validates them with the same metadata-only telemetry contract, appends accepted events to the local spool, and rejects raw prompt/output/transcript/message/note fields. + +Smoke-test one message without Browser installation: + +```bash +python3 scripts/telemetry_native_host.py . \ + --message-json '{"event":"skill_activation","activation_type":"explicit","outcome":"accepted","failure_type":"none","command":"chrome-native-host"}' +``` + +Generate a local launcher and Chrome native messaging manifest for an operator-installed extension: + +```bash +python3 scripts/telemetry_native_host.py . \ + --write-launcher /tmp/yao-telemetry-host.sh \ + --write-manifest /tmp/yao-telemetry-host.json \ + --allowed-origin chrome-extension://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/ +``` + +This is an executable native host bridge and manifest generator. It still does not prove that a specific Browser/Chrome extension is installed or sending events in the user's environment. + +## External Client Import + +External clients, browser extensions, editor adapters, or wrapper scripts may hand off already-sanitized JSONL through `telemetry-import`: + +```bash +python3 scripts/yao.py telemetry-import . \ + --input-jsonl /tmp/external-client-events.jsonl \ + --command browser-extension +``` + +The importer defaults missing `source` to `external` and missing `command` to `external-client`. It validates the entire JSONL file before writing anything. If any line includes a raw content field, unsupported source, unsupported outcome, unsupported failure type, unknown field, malformed JSON, or an unsafe command name, the whole import is rejected and the existing local event stream is left untouched. + +Use `--dry-run` to validate an external batch without writing `reports/telemetry_events.jsonl` or refreshing aggregate reports: + +```bash +python3 scripts/yao.py telemetry-import . --input-jsonl /tmp/external-client-events.jsonl --dry-run +``` + ## Privacy Rule The raw JSONL event log is local evidence and should not be distributed in skill packages. The distributable artifact is the aggregate report: @@ -48,13 +165,18 @@ Package builders should exclude `reports/telemetry_events.jsonl`. The root repos ## Iteration Loop -1. Capture metadata-only events locally. +1. Capture metadata-only events locally, either manually with `adoption-drift --record-event`, automatically with opt-in `yao.py` CLI capture, through `telemetry-emit` client hooks, through generated `telemetry-hooks` client recipes, or through validated external JSONL import. 2. Render `reports/adoption_drift_report.md`. 3. Convert missed triggers into trigger eval cases. 4. Convert bad outputs into Output Eval assertions and failure taxonomy entries. 5. Convert script errors into non-interactive smoke tests. 6. Feed review-overdue signals back into Skill Atlas and owner review. +7. Let Skill Atlas read only `reports/adoption_drift_report.json` and publish portfolio-level `skill_atlas/drift_signals.json`. ## Review Studio Role Review Studio should show the aggregate telemetry gate as an operating loop, not as raw logs. A blocker means the telemetry contract was violated. A warning means the evidence is absent or the drift signal needs a follow-up case. + +## Skill Atlas Role + +Skill Atlas uses aggregate adoption drift reports to rank portfolio work. It should surface no-data warnings for actionable production/library/governed skills, and drift warnings for missed triggers, wrong triggers, bad outputs, missing resources, script errors, and review-overdue counts. It must not inspect raw JSONL telemetry or use non-actionable example/fixture signals as release blockers. diff --git a/references/trust-security-method.md b/references/trust-security-method.md index ad6c9f54..a55e0d88 100644 --- a/references/trust-security-method.md +++ b/references/trust-security-method.md @@ -64,6 +64,7 @@ Permission approval validates reviewer intent. Runtime permission probes validat - Run `python3 scripts/probe_runtime_permissions.py . --package-dir dist` after `cross_packager.py`. - The probe writes `reports/runtime_permission_probes.json` and `reports/runtime_permission_probes.md`. - A passing probe requires every target adapter to carry `permission_contract`, `target_permission_contract`, declared capabilities, a native-enforcement boolean, representation notes, and operator notes. +- When `reports/install_simulation.json` matches the same package directory, the probe also reports installer enforcement counts from the install simulation. This proves the local package installer gate is wired, but it does not count as target-client native enforcement. - If a target has no native enforcement, the probe must mark an explicit metadata fallback and keep residual risk reviewer-visible. - Review Studio surfaces this as the `permission-runtime` gate. diff --git a/references/user-memory-policy.md b/references/user-memory-policy.md new file mode 100644 index 00000000..fd363856 --- /dev/null +++ b/references/user-memory-policy.md @@ -0,0 +1,35 @@ +# User Memory Policy + +This skill treats user preference memory as local, explicit, and reviewable. + +## Principles + +- **Explicit source only**: adaptive scans require a user-provided file path. +- **Local first**: no network access is needed for preference extraction. +- **No implicit private logs**: shell history, browser history, mail, and hidden chat logs are blocked by default. +- **Repeated signals only**: one-off statements are recorded as discarded signals unless they meet the configured support threshold. +- **Redacted evidence**: stored excerpts must remove secrets, tokens, email addresses, and local absolute paths. +- **Proposal before patch**: preference memory can generate proposals, not automatic source edits. + +## Allowed Inputs + +Recommended inputs are curated JSONL, Markdown, or text files prepared for review. JSONL records should use a field such as `text`, `message`, `content`, `excerpt`, `prompt`, `note`, or `body`. + +## Blocked By Default + +The scanner refuses common shell history files such as `.zsh_history`, `.bash_history`, and `.fish_history` unless an explicit override is added for a controlled test. Even with an override, the output remains redacted and proposal-only. + +## Retention + +Generated reports store only summarized patterns and short redacted excerpts. They should not be treated as a full transcript, chat archive, or durable personal memory store. + +## Upgrade Path + +A future patch-application stage must add: + +- human approval ledger; +- allowlisted target files; +- dry-run diffs; +- regression command execution; +- rollback artifacts; +- reviewer-visible audit trail. diff --git a/registry/index.json b/registry/index.json index e4bc9b50..94e14ef1 100644 --- a/registry/index.json +++ b/registry/index.json @@ -12,10 +12,11 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 4fa9e8c6..28c2a608 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -2,12 +2,13 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "maturity": "governed", "owner": "Yao Team", @@ -15,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f", - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { "openai": "pass", @@ -47,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adaptation_approval_ledger.json b/reports/adaptation_approval_ledger.json new file mode 100644 index 00000000..78e9d2c7 --- /dev/null +++ b/reports/adaptation_approval_ledger.json @@ -0,0 +1,63 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15T00:00:00Z", + "summary": { + "approval_count": 0, + "active_approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0 + }, + "approval_contract": { + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true + }, + "entries": [], + "approval_count": 0, + "active_approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true, + "report_contract": { + "schema_version": "1.0", + "contract": "adaptation-approval-ledger", + "top_level_mirrors_summary": true, + "top_level_mirrors_approval_contract": true, + "summary_fields": [ + "approval_count", + "active_approval_count", + "pending_review_count", + "applied_count", + "rollback_count" + ], + "approval_contract_fields": [ + "approval_required", + "patch_sha256_required", + "allowlisted_targets_required", + "target_file_sha256_required", + "approval_draft_supported", + "dry_run_default", + "writes_repository_files_only_with_apply", + "rollback_required" + ], + "source_of_truth": [ + "summary", + "approval_contract" + ] + } +} diff --git a/reports/adaptation_proposals.json b/reports/adaptation_proposals.json new file mode 100644 index 00000000..55892afa --- /dev/null +++ b/reports/adaptation_proposals.json @@ -0,0 +1,248 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17T08:58:54Z", + "skill_dir": ".", + "source_patterns": "reports/user_patterns.json", + "pattern_count": 5, + "proposal_count": 5, + "apply_supported": true, + "failure_count": 0, + "proposal_only": true, + "approval_required": true, + "writes_repository_files": false, + "allowlisted_targets_required": true, + "target_file_sha256_required_for_apply": true, + "approval_draft_supported": true, + "rollback_required_for_apply": true, + "apply_command_available": true, + "summary": { + "pattern_count": 5, + "proposal_count": 5, + "apply_supported": true, + "failure_count": 0 + }, + "proposal_contract": { + "proposal_only": true, + "approval_required": true, + "writes_repository_files": false, + "allowlisted_targets_required": true, + "target_file_sha256_required_for_apply": true, + "approval_draft_supported": true, + "rollback_required_for_apply": true, + "apply_command_available": true + }, + "report_contract": { + "schema_version": "1.0", + "contract": "adaptation-proposals", + "top_level_mirrors_summary": true, + "top_level_mirrors_proposal_contract": true, + "summary_fields": [ + "pattern_count", + "proposal_count", + "apply_supported", + "failure_count" + ], + "proposal_contract_fields": [ + "proposal_only", + "approval_required", + "writes_repository_files", + "allowlisted_targets_required", + "target_file_sha256_required_for_apply", + "approval_draft_supported", + "rollback_required_for_apply", + "apply_command_available" + ], + "source_of_truth": [ + "summary", + "proposal_contract" + ] + }, + "proposals": [ + { + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "change_type": "report-default-language", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "low", + "reason": "2 redacted records matched repeated report-language signals.", + "support_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "suggested_changes": [ + "Keep user-facing report copy Simplified Chinese by default.", + "Expose English through the existing language switch instead of mixing languages in the default view." + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ], + "rollback_plan": "Revert report language template changes and rerun the overview verifier.", + "evidence_refs": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + } + ] + }, + { + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "change_type": "artifact-ui-polish", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "medium", + "reason": "5 redacted records matched repeated artifact-design signals.", + "support_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "suggested_changes": [ + "Prefer vertical narrative sections with limited two-column layouts only when content has enough width.", + "Keep charts inline SVG, with captions and stable responsive constraints." + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ], + "rollback_plan": "Restore the previous report renderer and regenerate the demo report.", + "evidence_refs": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-3", + "excerpt": "报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。" + } + ] + }, + { + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "change_type": "privacy-governance", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "low", + "reason": "2 redacted records matched repeated governance signals.", + "support_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "suggested_changes": [ + "Require explicit source paths for memory scans.", + "Generate proposals before any source patching.", + "Reserve automatic apply for a future approval ledger and rollback implementation." + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ], + "rollback_plan": "Remove the adaptive proposal artifacts and keep feedback/adoption drift as the only iteration inputs.", + "evidence_refs": [ + { + "record_id": "line-5", + "excerpt": "自适应升级必须先生成提案，不能直接自动修改源文件。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "change_type": "artifact-discoverability", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "low", + "reason": "2 redacted records matched repeated artifact-format signals.", + "support_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "suggested_changes": [ + "Include stable report paths in command output.", + "Document which artifacts are meant for human review." + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ], + "rollback_plan": "Revert CLI copy/documentation changes and keep artifact paths unchanged.", + "evidence_refs": [ + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "change_type": "quality-gate", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "medium", + "reason": "2 redacted records matched repeated quality-gate signals.", + "support_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "suggested_changes": [ + "Add focused verifier coverage for every new adaptive behavior.", + "Refresh Skill OS 2.0 coverage so planned, partial, and covered states remain visible." + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ], + "rollback_plan": "Revert the new verifier and coverage status updates, then regenerate coverage reports.", + "evidence_refs": [ + { + "record_id": "line-7", + "excerpt": "每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。" + }, + { + "record_id": "line-8", + "excerpt": "涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。" + } + ] + } + ], + "failures": [], + "artifacts": { + "json": "reports/adaptation_proposals.json", + "markdown": "reports/adaptation_proposals.md" + } +} diff --git a/reports/adaptation_proposals.md b/reports/adaptation_proposals.md new file mode 100644 index 00000000..86f10bec --- /dev/null +++ b/reports/adaptation_proposals.md @@ -0,0 +1,119 @@ +# Adaptation Proposals + +- Generated at: `2026-06-17T08:58:54Z` +- Pattern report: `reports/user_patterns.json` +- Proposal only: `true` +- Writes repository files: `false` +- Proposals: `5` + +## Keep reports Chinese-first with optional English + +- ID: `adapt-18c7517f3d` +- Status: `proposal-only` +- Pattern: `language_default` +- Risk: `low` +- Requires approval: `true` +- Reason: 2 redacted records matched repeated report-language signals. +- Target files: + - `scripts/render_skill_overview.py` + - `references/artifact-design-doctrine.md` +- Suggested changes: + - Keep user-facing report copy Simplified Chinese by default. + - Expose English through the existing language switch instead of mixing languages in the default view. +- Verification: + - `python3 tests/verify_skill_overview.py` +- Rollback: Revert report language template changes and rerun the overview verifier. +- Redacted evidence refs: + - `line-1`: 新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。 + - `line-2`: HTML 报告需要双语能力，但默认内容应该保持中文简体。 + +## Improve report layout, visual hierarchy, and chart readability + +- ID: `adapt-fbfe921ba5` +- Status: `proposal-only` +- Pattern: `report_ui` +- Risk: `medium` +- Requires approval: `true` +- Reason: 5 redacted records matched repeated artifact-design signals. +- Target files: + - `scripts/render_skill_overview.py` + - `references/artifact-design-doctrine.md` + - `tests/verify_skill_overview.py` +- Suggested changes: + - Prefer vertical narrative sections with limited two-column layouts only when content has enough width. + - Keep charts inline SVG, with captions and stable responsive constraints. +- Verification: + - `python3 tests/verify_skill_overview.py` + - `python3 tests/verify_skill_report_charts.py` +- Rollback: Restore the previous report renderer and regenerate the demo report. +- Redacted evidence refs: + - `line-1`: 新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。 + - `line-2`: HTML 报告需要双语能力，但默认内容应该保持中文简体。 + - `line-3`: 报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。 + +## Keep adaptive iteration approval-gated + +- ID: `adapt-59d219a1fb` +- Status: `proposal-only` +- Pattern: `approval_safety` +- Risk: `low` +- Requires approval: `true` +- Reason: 2 redacted records matched repeated governance signals. +- Target files: + - `references/user-memory-policy.md` + - `references/autonomous-adaptation.md` + - `schemas/adaptation-proposal.schema.json` +- Suggested changes: + - Require explicit source paths for memory scans. + - Generate proposals before any source patching. + - Reserve automatic apply for a future approval ledger and rollback implementation. +- Verification: + - `python3 tests/verify_adaptation_safety.py` +- Rollback: Remove the adaptive proposal artifacts and keep feedback/adoption drift as the only iteration inputs. +- Redacted evidence refs: + - `line-5`: 自适应升级必须先生成提案，不能直接自动修改源文件。 + - `line-6`: 用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。 + +## Make generated artifact paths explicit in CLI output + +- ID: `adapt-457baca160` +- Status: `proposal-only` +- Pattern: `delivery_format` +- Risk: `low` +- Requires approval: `true` +- Reason: 2 redacted records matched repeated artifact-format signals. +- Target files: + - `scripts/yao.py` + - `README.md` +- Suggested changes: + - Include stable report paths in command output. + - Document which artifacts are meant for human review. +- Verification: + - `python3 tests/verify_yao_cli.py` +- Rollback: Revert CLI copy/documentation changes and keep artifact paths unchanged. +- Redacted evidence refs: + - `line-2`: HTML 报告需要双语能力，但默认内容应该保持中文简体。 + - `line-6`: 用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。 + +## Attach tests and evidence refresh to each upgrade + +- ID: `adapt-abfee25d3a` +- Status: `proposal-only` +- Pattern: `evidence_testing` +- Risk: `medium` +- Requires approval: `true` +- Reason: 2 redacted records matched repeated quality-gate signals. +- Target files: + - `tests/verify_adaptation_safety.py` + - `scripts/render_skill_os2_coverage.py` + - `reports/skill_os2_coverage.json` +- Suggested changes: + - Add focused verifier coverage for every new adaptive behavior. + - Refresh Skill OS 2.0 coverage so planned, partial, and covered states remain visible. +- Verification: + - `python3 tests/verify_adaptation_safety.py` + - `python3 tests/verify_skill_os2_coverage.py` +- Rollback: Revert the new verifier and coverage status updates, then regenerate coverage reports. +- Redacted evidence refs: + - `line-7`: 每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。 + - `line-8`: 涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。 diff --git a/reports/adaptation_regression_report.json b/reports/adaptation_regression_report.json new file mode 100644 index 00000000..a0308041 --- /dev/null +++ b/reports/adaptation_regression_report.json @@ -0,0 +1,87 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15T00:00:00Z", + "skill_dir": ".", + "summary": { + "apply_supported": true, + "attempt_count": 0, + "approval_draft_count": 0, + "applied_count": 0, + "dry_run_count": 0, + "rollback_count": 0, + "regression_run_count": 0, + "regression_pass_count": 0, + "failure_count": 0 + }, + "apply_contract": { + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true, + "safe_regression_commands_only": true, + "rollback_on_failure_default": true + }, + "attempts": [], + "failures": [], + "artifacts": { + "json": "reports/adaptation_regression_report.json", + "approval_ledger": "reports/adaptation_approval_ledger.json" + }, + "apply_supported": true, + "attempt_count": 0, + "approval_draft_count": 0, + "applied_count": 0, + "dry_run_count": 0, + "rollback_count": 0, + "regression_run_count": 0, + "regression_pass_count": 0, + "failure_count": 0, + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true, + "safe_regression_commands_only": true, + "rollback_on_failure_default": true, + "report_contract": { + "schema_version": "1.0", + "contract": "adaptation-regression-report", + "top_level_mirrors_summary": true, + "top_level_mirrors_apply_contract": true, + "summary_fields": [ + "apply_supported", + "attempt_count", + "approval_draft_count", + "applied_count", + "dry_run_count", + "rollback_count", + "regression_run_count", + "regression_pass_count", + "failure_count" + ], + "apply_contract_fields": [ + "approval_required", + "patch_sha256_required", + "allowlisted_targets_required", + "target_file_sha256_required", + "approval_draft_supported", + "dry_run_default", + "writes_repository_files_only_with_apply", + "rollback_required", + "safe_regression_commands_only", + "rollback_on_failure_default" + ], + "source_of_truth": [ + "summary", + "apply_contract" + ] + } +} diff --git a/reports/adaptation_regression_report.md b/reports/adaptation_regression_report.md new file mode 100644 index 00000000..cdcaa683 --- /dev/null +++ b/reports/adaptation_regression_report.md @@ -0,0 +1,11 @@ +# Adaptation Regression Report + +- generated_at: `2026-06-15T00:00:00Z` +- apply_supported: `true` +- attempts: `0` +- applied: `0` +- dry runs: `0` +- rollbacks: `0` +- failures: `0` + +This report proves the adaptive apply harness behavior. It does not count proposals as applied changes. diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 7f931d0b..da4a16c2 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T10:57:52Z", + "generated_at": "2026-06-17T08:58:53Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -25,13 +25,14 @@ }, "summary": { "event_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -40,36 +41,43 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, - "failure_types": {} + "failure_types": {}, + "source_types": { + "manual": 1 + }, + "command_counts": {} }, "adoption_by_skill": [ { "skill": "yao-meta-skill", "events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { - "event": "skill_activation", + "command": "unknown", + "event": "review_event", "skill": "yao-meta-skill", + "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } diff --git a/reports/adoption_drift_report.md b/reports/adoption_drift_report.md index 743dd610..91b31983 100644 --- a/reports/adoption_drift_report.md +++ b/reports/adoption_drift_report.md @@ -5,8 +5,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Summary - Events: `1` -- Activation events: `1` -- Adoption rate: `100.0` +- Adoption samples: `0` +- Activation events: `0` +- Adoption rate: `0` - Missed trigger signals: `0` - Bad output signals: `0` - Script error signals: `0` @@ -22,9 +23,9 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Adoption By Skill -| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `yao-meta-skill` | 1 | 1 | 0 | 0 | 0 | 100.0 | +| Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `yao-meta-skill` | 1 | 0 | 0 | 0 | 0 | 0 | 0 | ## Next Iteration Candidates @@ -32,4 +33,4 @@ Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, ## Recent Metadata Events -- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` activation=`explicit` outcome=`accepted` failure=`none` +- `2026-06-13T12:00:00Z` `yao-meta-skill` event=`review_event` source=`manual` command=`unknown` activation=`manual` outcome=`reviewed` failure=`none` diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json new file mode 100644 index 00000000..6951d03a --- /dev/null +++ b/reports/architecture_maintainability.json @@ -0,0 +1,181 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "python_file_count": 229, + "script_file_count": 153, + "test_file_count": 76, + "internal_module_count": 69, + "cli_script_count": 86, + "command_handler_count": 68, + "entrypoint_command_handler_count": 18, + "command_module_count": 6, + "warn_line_threshold": 900, + "watch_line_threshold": 720, + "early_watch_line_threshold": 600, + "block_line_threshold": 1500, + "largest_file_lines": 719, + "watchlist_count": 0, + "early_watchlist_count": 6, + "hotspot_count": 0, + "blocker_count": 0, + "decision": "pass" + }, + "largest_files": [ + { + "path": "tests/verify_evidence_consistency.py", + "lines": 719, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 703, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_yao_cli.py", + "lines": 700, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "lines": 685, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 676, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "lines": 651, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "tests/verify_output_review_adjudication.py", + "lines": 599, + "kind": "test", + "severity": "pass", + "early_watch": false, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/render_skill_overview.py", + "lines": 588, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "tests/verify_world_class_evidence_ledger.py", + "lines": 587, + "kind": "test", + "severity": "pass", + "early_watch": false, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/build_skill_atlas.py", + "lines": 586, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/optimize_description.py", + "lines": 585, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/trust_check.py", + "lines": 582, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + } + ], + "watchlist": [], + "early_watchlist": [ + { + "path": "tests/verify_evidence_consistency.py", + "lines": 719, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 703, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_yao_cli.py", + "lines": 700, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "lines": 685, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 676, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "lines": 651, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + } + ], + "hotspots": [], + "actions": [], + "artifacts": { + "json": "reports/architecture_maintainability.json", + "markdown": "reports/architecture_maintainability.md" + } +} diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md new file mode 100644 index 00000000..c615e840 --- /dev/null +++ b/reports/architecture_maintainability.md @@ -0,0 +1,66 @@ +# Architecture Maintainability + +Generated at: `2026-06-17` + +## Summary + +- decision: `pass` +- python files: `229` +- scripts: `153` +- tests: `76` +- internal modules: `69` +- CLI scripts: `86` +- Yao CLI command handlers: `68` +- entrypoint command handlers: `18` +- command modules: `6` +- largest file lines: `719` +- early watch threshold lines: `600` +- early watchlist: `6` +- watch threshold lines: `720` +- watchlist: `0` +- hotspots: `0` +- blockers: `0` + +This report keeps maintainability risk visible before the Meta Skill grows more gates, renderers, and CLI commands. + +## Hotspots + +No file-size hotspots found. + +## Watchlist + +No near-threshold files found. + +## Early Watchlist + +| File | Lines | Kind | Recommended next split | +| --- | ---: | --- | --- | +| `tests/verify_evidence_consistency.py` | `719` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `tests/verify_world_class_evidence_intake.py` | `703` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `tests/verify_yao_cli.py` | `700` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `scripts/render_benchmark_reproducibility.py` | `685` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | +| `scripts/render_evidence_consistency.py` | `676` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | +| `scripts/render_world_class_operator_runbook.py` | `651` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | + +## Largest Files + +| File | Lines | Kind | Severity | +| --- | ---: | --- | --- | +| `tests/verify_evidence_consistency.py` | `719` | `test` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `703` | `test` | `pass` | +| `tests/verify_yao_cli.py` | `700` | `test` | `pass` | +| `scripts/render_benchmark_reproducibility.py` | `685` | `cli-script` | `pass` | +| `scripts/render_evidence_consistency.py` | `676` | `cli-script` | `pass` | +| `scripts/render_world_class_operator_runbook.py` | `651` | `cli-script` | `pass` | +| `tests/verify_output_review_adjudication.py` | `599` | `test` | `pass` | +| `scripts/render_skill_overview.py` | `588` | `cli-script` | `pass` | +| `tests/verify_world_class_evidence_ledger.py` | `587` | `test` | `pass` | +| `scripts/build_skill_atlas.py` | `586` | `cli-script` | `pass` | +| `scripts/optimize_description.py` | `585` | `cli-script` | `pass` | +| `scripts/trust_check.py` | `582` | `cli-script` | `pass` | + +## Release Rule + +- `block` hotspots should be split before governed release. +- `warn` hotspots can ship only when Review Studio keeps them visible and a reviewer accepts the modularization plan. +- Do not split a file only for line count; split when a stable responsibility boundary is clear. diff --git a/reports/artifact-design-profile.json b/reports/artifact-design-profile.json index 2e76f3c6..c32e5dc7 100644 --- a/reports/artifact-design-profile.json +++ b/reports/artifact-design-profile.json @@ -2,16 +2,40 @@ "skill_name": "yao-meta-skill", "design_system": "metric editorial", "primary_artifact": { - "key": "code_or_cli", - "label": "Code, CLI, or implementation guide", - "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "key": "dashboard", + "label": "Dashboard or metrics page", + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", "matched_keywords": [ - "code", - "script", - "command" + "metric", + "score", + "table", + "scorecard" ] }, "artifact_families": [ + { + "key": "dashboard", + "label": "Dashboard or metrics page", + "score": 4, + "matched_keywords": [ + "metric", + "score", + "table", + "scorecard" + ], + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." + }, + { + "key": "review_viewer", + "label": "Review viewer", + "score": 3, + "matched_keywords": [ + "review", + "viewer", + "audit" + ], + "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." + }, { "key": "code_or_cli", "label": "Code, CLI, or implementation guide", @@ -33,25 +57,6 @@ ], "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." }, - { - "key": "review_viewer", - "label": "Review viewer", - "score": 2, - "matched_keywords": [ - "review", - "viewer" - ], - "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." - }, - { - "key": "dashboard", - "label": "Dashboard or metrics page", - "score": 1, - "matched_keywords": [ - "table" - ], - "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." - }, { "key": "visual_capture", "label": "Screenshot or visual evidence", @@ -63,14 +68,14 @@ } ], "layout_patterns": [ - "prerequisites", - "commands", - "expected output", - "failure handling", - "rollback or cleanup", - "thesis", - "evidence blocks", - "decision table" + "metric board", + "ranked signals", + "comparison rows", + "interpretation", + "action queue", + "summary", + "variant comparison", + "evidence" ], "design_tokens": { "type": [ @@ -95,14 +100,14 @@ ] }, "quality_gates": [ - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding.", - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities.", "Make differences visible instead of hiding them in prose.", - "Separate author-facing recommendations from reviewer-only evidence." + "Separate author-facing recommendations from reviewer-only evidence.", + "Surface conflicts clearly and keep routine benchmark synthesis quiet.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations." ], "anti_patterns": [ "Do not copy Kami's fixed parchment background as a default.", diff --git a/reports/artifact-design-profile.md b/reports/artifact-design-profile.md index 4b708892..f4fd2531 100644 --- a/reports/artifact-design-profile.md +++ b/reports/artifact-design-profile.md @@ -5,12 +5,22 @@ Design system: `metric editorial` ## Primary Artifact Direction -**Code, CLI, or implementation guide** +**Dashboard or metrics page** -Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit. +Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation. ## Matched Artifact Families +### Dashboard or metrics page +- Matched keywords: metric, score, table, scorecard +- Score: `4` +- Direction: Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation. + +### Review viewer +- Matched keywords: review, viewer, audit +- Score: `3` +- Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix. + ### Code, CLI, or implementation guide - Matched keywords: code, script, command - Score: `3` @@ -21,16 +31,6 @@ Execution-focused technical artifact with environment assumptions, copyable comm - Score: `2` - Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. -### Review viewer -- Matched keywords: review, viewer -- Score: `2` -- Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix. - -### Dashboard or metrics page -- Matched keywords: table -- Score: `1` -- Direction: Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation. - ### Screenshot or visual evidence - Matched keywords: screenshot - Score: `1` @@ -38,14 +38,14 @@ Execution-focused technical artifact with environment assumptions, copyable comm ## Layout Patterns To Prefer -- prerequisites -- commands -- expected output -- failure handling -- rollback or cleanup -- thesis -- evidence blocks -- decision table +- metric board +- ranked signals +- comparison rows +- interpretation +- action queue +- summary +- variant comparison +- evidence ## Design Tokens @@ -71,14 +71,14 @@ Execution-focused technical artifact with environment assumptions, copyable comm ## Quality Gates -- Name the working directory and required inputs for commands. -- Mark destructive, networked, or external side-effect operations. -- Prefer the smallest runnable snippet over broad framework scaffolding. -- Keep the first screen useful without requiring the reader to parse every detail. -- Use tables only for comparisons; move explanations below the table. -- Keep source notes readable without flooding the body with markers. +- Avoid paragraph-heavy table cells. +- Keep charts tied to one analytical question each. +- Preserve stable color meaning across metrics and entities. - Make differences visible instead of hiding them in prose. - Separate author-facing recommendations from reviewer-only evidence. +- Surface conflicts clearly and keep routine benchmark synthesis quiet. +- Name the working directory and required inputs for commands. +- Mark destructive, networked, or external side-effect operations. ## Anti-Patterns diff --git a/reports/baseline-compare.json b/reports/baseline-compare.json index 274a6350..0896a070 100644 --- a/reports/baseline-compare.json +++ b/reports/baseline-compare.json @@ -1,10 +1,10 @@ { "summary": { "target_count": 3, - "baseline_total_errors": 8, - "current_total_errors": 5, - "winner_total_errors": 5, - "winner_vs_baseline_gain": 3, + "baseline_total_errors": 5, + "current_total_errors": 4, + "winner_total_errors": 4, + "winner_vs_baseline_gain": 1, "winner_vs_current_gain": 0 }, "comparisons": [ @@ -20,7 +20,7 @@ "adversarial_errors": 0 }, "current": { - "tokens": 65, + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "blind_errors": 0, @@ -28,7 +28,7 @@ "adversarial_errors": 0 }, "winner": { - "tokens": 65, + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "blind_errors": 0, @@ -79,15 +79,15 @@ "winner_label": "Current", "baseline": { "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2, - "blind_errors": 1, + "dev_errors": 0, + "holdout_errors": 1, + "blind_errors": 0, "judge_blind_errors": 0, "adversarial_errors": 0 }, "current": { "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "blind_errors": 0, "judge_blind_errors": 0, @@ -95,15 +95,15 @@ }, "winner": { "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "blind_errors": 0, "judge_blind_errors": 0, "adversarial_errors": 0 }, "delta": { - "current_vs_baseline": 2, - "winner_vs_baseline": 2, + "current_vs_baseline": 0, + "winner_vs_baseline": 0, "winner_vs_current": 0 } } diff --git a/reports/baseline-compare.md b/reports/baseline-compare.md index ba90da2d..4b8ece0e 100644 --- a/reports/baseline-compare.md +++ b/reports/baseline-compare.md @@ -3,10 +3,10 @@ A lightweight with-skill vs baseline comparison across tracked optimization targets. - Targets: `3` -- Baseline total errors: `8` -- Current total errors: `5` -- Winner total errors: `5` -- Winner vs baseline gain: `3` +- Baseline total errors: `5` +- Current total errors: `4` +- Winner total errors: `4` +- Winner vs baseline gain: `1` - Winner vs current gain: `0` ## Target Breakdown @@ -15,4 +15,4 @@ A lightweight with-skill vs baseline comparison across tracked optimization targ | --- | ---: | ---: | ---: | --- | | root | 1 | 0 | 0 | Current | | team-frontend-review | 3 | 3 | 3 | Current | -| governed-incident-command | 4 | 2 | 2 | Current | +| governed-incident-command | 1 | 1 | 1 | Current | diff --git a/reports/benchmark_methodology.md b/reports/benchmark_methodology.md index 91fa5ff0..7922a2d5 100644 --- a/reports/benchmark_methodology.md +++ b/reports/benchmark_methodology.md @@ -60,6 +60,7 @@ Recommended release evidence: git rev-parse HEAD python3 scripts/run_output_eval.py python3 scripts/export_skill_ir.py . --output-json skill-ir/examples/yao-meta-skill.json +python3 scripts/yao.py benchmark-reproducibility . make ci-test ``` diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json new file mode 100644 index 00000000..74cf5b71 --- /dev/null +++ b/reports/benchmark_reproducibility.json @@ -0,0 +1,488 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "git_status": { + "available": true, + "dirty": true, + "changed_file_count": 29, + "generated_dirty": true, + "generated_changed_file_count": 29, + "source_dirty": false, + "source_changed_file_count": 0, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adaptation_proposals.json", + " M reports/adaptation_proposals.md", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json" + ], + "source_sample": [], + "generated_sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adaptation_proposals.json", + " M reports/adaptation_proposals.md", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json" + ], + "generated_dirty_prefixes": [ + "dist/", + "registry/index.json", + "registry/packages/", + "reports/", + "skill_atlas/", + "skill-ir/examples/" + ], + "scope": "generation-time status before this report is written" + }, + "summary": { + "reproducibility_ready": true, + "release_lock_ready": true, + "methodology_complete": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "evidence_bundle_sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0", + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 23, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 10, + "token_observed_count": 10, + "human_review_complete": false, + "provider_evidence_complete": true, + "world_class_ready": false, + "world_class_open_gap_count": 3, + "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3, + "working_tree_dirty": true, + "changed_file_count": 29, + "source_tree_dirty": false, + "source_changed_file_count": 0, + "generated_tree_dirty": true, + "generated_changed_file_count": 29 + }, + "beta_test_release": { + "ready": true, + "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "blockers": [], + "allowed_deferred_evidence": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "reason": "Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "reason": "Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "reason": "Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "reason": "Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims." + } + ], + "policy": "Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted.", + "required_wording": "Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline." + }, + "public_claim": { + "ready": false, + "scope": "public benchmark or world-class readiness claim", + "blockers": [ + "human blind-review adjudication is incomplete", + "world-class evidence is not accepted yet (3 open gaps, 4 ledger pending)", + "world-class source checks are not all accepted (11/19 pass, 8 blocked)" + ], + "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." + }, + "release_lock": { + "ready": true, + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "status_scope": "generation-time status before this report is written", + "source_changed_file_count": 0, + "generated_changed_file_count": 29, + "reason": "only generated evidence artifacts were dirty at generation time" + }, + "evidence_bundle": { + "algorithm": "sha256(path,label,exists,artifact_sha256)", + "artifact_count": 25, + "existing_count": 25, + "missing_count": 0, + "missing_paths": [], + "sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0" + }, + "methodology": { + "path": "reports/benchmark_methodology.md", + "exists": true, + "sections": [ + { + "heading": "## Benchmark Types", + "exists": true + }, + { + "heading": "## Sample Sources", + "exists": true + }, + { + "heading": "## Evaluation Dimensions", + "exists": true + }, + { + "heading": "## Weighting Rule", + "exists": true + }, + { + "heading": "## Failure Disclosure", + "exists": true + }, + { + "heading": "## Reproduction", + "exists": true + } + ], + "missing_sections": [] + }, + "artifacts_checked": [ + { + "label": "methodology", + "path": "reports/benchmark_methodology.md", + "exists": true, + "bytes": 2715, + "sha256": "57025e0123ce5d10401c5bff376d2eeeac7943c83897ae1ad3fb22cadf790f92" + }, + { + "label": "failure_disclosure", + "path": "evals/failure-cases.md", + "exists": true, + "bytes": 889, + "sha256": "28833c0d4a217d612879d193fb5de199880dd5b3093ab5757e4315600fa4fb08" + }, + { + "label": "output_cases", + "path": "evals/output/cases.jsonl", + "exists": true, + "bytes": 6555, + "sha256": "a6ae9685711620d7203b73ace4412194ae287689f945b5139ec8ad75b9eefe04" + }, + { + "label": "output_schema", + "path": "evals/output/schema.json", + "exists": true, + "bytes": 2193, + "sha256": "8ee340c95064260c5e952be614e19841ac676162c6bf01d21b107e38cb04e0b9" + }, + { + "label": "output_scorecard", + "path": "reports/output_quality_scorecard.json", + "exists": true, + "bytes": 25530, + "sha256": "0806258a8e084b27e112537faff0de64a8519ca90cfdc78b57c0e4c08a514cca" + }, + { + "label": "output_execution", + "path": "reports/output_execution_runs.json", + "exists": true, + "bytes": 8449, + "sha256": "4df66b63d2e717a559a6881b966fb1b92def7329cc89aa48ee0d8596f87402e5" + }, + { + "label": "blind_review", + "path": "reports/output_blind_review_pack.json", + "exists": true, + "bytes": 7804, + "sha256": "bbe2db8ec2776fe289cd7d6bb78d48c2b8baa106ad37f79c15e43669b10c9390" + }, + { + "label": "review_adjudication", + "path": "reports/output_review_adjudication.json", + "exists": true, + "bytes": 14084, + "sha256": "91fd88dd9b0f8876f68027a87893d1852dac59772613345bbbfabba658c9227e" + }, + { + "label": "trigger_scorecard", + "path": "reports/route_scorecard.json", + "exists": true, + "bytes": 16950, + "sha256": "a649547a7d3d5fe02262ca1b188ed4022d8f557770be425f7b7cf762aa76533a" + }, + { + "label": "runtime_conformance", + "path": "reports/conformance_matrix.json", + "exists": true, + "bytes": 10342, + "sha256": "97f9ba949c23a60b00e9ba2ff279ca03ba517845cc7c62aa8a42645d58006c7e" + }, + { + "label": "trust_report", + "path": "reports/security_trust_report.json", + "exists": true, + "bytes": 137787, + "sha256": "bbff1f1bee0ce6f7014efcd4622ba11c70aacfc81f2e69e225411df7004dd616" + }, + { + "label": "python_compatibility", + "path": "reports/python_compatibility.json", + "exists": true, + "bytes": 30127, + "sha256": "d31dc03c74f17242f4163b8cb3f23120fe3d0712edd08fa183cb94f8120d491c" + }, + { + "label": "registry_audit", + "path": "reports/registry_audit.json", + "exists": true, + "bytes": 3137, + "sha256": "1feb80c7cdc833a655cc0e44150f39f9f6d0633465abaa1ae29ebe7cb0649065" + }, + { + "label": "package_verification", + "path": "reports/package_verification.json", + "exists": true, + "bytes": 19338, + "sha256": "7aa6c25e8154d246ace49f7ab9159fa85f96d1d4ebb7e2bb03c7b223260a908b" + }, + { + "label": "install_simulation", + "path": "reports/install_simulation.json", + "exists": true, + "bytes": 8758, + "sha256": "c1f1122264b868c4998482c580ef94333575c6ff7c5c9c297188b225f69a131d" + }, + { + "label": "skill_os2_audit", + "path": "reports/skill_os2_audit.json", + "exists": true, + "bytes": 13824, + "sha256": "6362e79a8cfd3ff54b1a7ca32f37de8ed1cda500b953fdf1074928c6d2d62379" + }, + { + "label": "world_class_evidence_plan", + "path": "reports/world_class_evidence_plan.json", + "exists": true, + "bytes": 23314, + "sha256": "8eb5e1fbaa7580f107fd9920d13a13b239edb72d50b6959fc8f6dd43150f07d7" + }, + { + "label": "world_class_evidence_ledger", + "path": "reports/world_class_evidence_ledger.json", + "exists": true, + "bytes": 26277, + "sha256": "6038b6cd6e0667f25cfd69a5e40afa314d9efc3a93753a2a3a0509ae9b516a15" + }, + { + "label": "world_class_evidence_intake", + "path": "reports/world_class_evidence_intake.json", + "exists": true, + "bytes": 20922, + "sha256": "aa262da0ecee7232974ce061a51c8966c310de0b0be9f91eb82bb28bf1c7c0a8" + }, + { + "label": "world_class_evidence_preflight", + "path": "reports/world_class_evidence_preflight.json", + "exists": true, + "bytes": 94341, + "sha256": "95cb56581224aab06be84d0b338d6ba9aa4aef0b1ebd480801f1e2226b873b0d" + }, + { + "label": "world_class_submission_review", + "path": "reports/world_class_submission_review.json", + "exists": true, + "bytes": 17295, + "sha256": "12fbb84926294c4bd05af526fcbe0983db26e449d2c305006094baf27332a0ba" + }, + { + "label": "world_class_operator_runbook", + "path": "reports/world_class_operator_runbook.json", + "exists": true, + "bytes": 87257, + "sha256": "f321dc118ee931a1e1510559e5e64774168a58b0446ae62d232009eb42e3b384" + }, + { + "label": "world_class_operator_runbook_markdown", + "path": "reports/world_class_operator_runbook.md", + "exists": true, + "bytes": 25549, + "sha256": "858ee11bbacb143f648e9c00735bc248a0b9f3c9303dc9e0ad5c0a65d9d6e07f" + }, + { + "label": "world_class_operator_runbook_html", + "path": "reports/world_class_operator_runbook.html", + "exists": true, + "bytes": 34702, + "sha256": "31b24658c59ee1dee25aec378e72033b042af88dfdf40c325d361d0c9bec2ca7" + }, + { + "label": "world_class_claim_guard", + "path": "reports/world_class_claim_guard.json", + "exists": true, + "bytes": 18596, + "sha256": "abe7f7d60c0025e140373fadaefbed4063f285c140c74b9c7cfb464e354bb526" + } + ], + "missing_artifacts": [], + "reproduction_commands": [ + { + "label": "source commit", + "command": "git rev-parse HEAD", + "evidence": "git commit hash" + }, + { + "label": "trigger eval", + "command": "make eval-suite", + "evidence": "reports/eval_suite.json" + }, + { + "label": "output eval", + "command": "python3 scripts/yao.py output-eval", + "evidence": "reports/output_quality_scorecard.json" + }, + { + "label": "output execution", + "command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'", + "evidence": "reports/output_execution_runs.json" + }, + { + "label": "blind review adjudication", + "command": "python3 scripts/yao.py output-review", + "evidence": "reports/output_review_adjudication.json" + }, + { + "label": "skill ir", + "command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json", + "evidence": "skill-ir/examples/yao-meta-skill.json" + }, + { + "label": "runtime conformance", + "command": "python3 scripts/yao.py conformance .", + "evidence": "reports/conformance_matrix.json" + }, + { + "label": "trust report", + "command": "python3 scripts/yao.py trust .", + "evidence": "reports/security_trust_report.json" + }, + { + "label": "python compatibility", + "command": "python3 scripts/yao.py python-compat .", + "evidence": "reports/python_compatibility.json" + }, + { + "label": "package", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip", + "evidence": "dist/yao-meta-skill.zip" + }, + { + "label": "package verify", + "command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip", + "evidence": "reports/package_verification.json" + }, + { + "label": "install simulate", + "command": "python3 scripts/yao.py install-simulate . --package-dir dist", + "evidence": "reports/install_simulation.json" + }, + { + "label": "registry audit", + "command": "python3 scripts/yao.py registry-audit .", + "evidence": "reports/registry_audit.json" + }, + { + "label": "skill os audit", + "command": "python3 scripts/yao.py skill-os2-audit .", + "evidence": "reports/skill_os2_audit.json" + }, + { + "label": "world-class evidence plan", + "command": "python3 scripts/yao.py world-class-evidence .", + "evidence": "reports/world_class_evidence_plan.json" + }, + { + "label": "world-class evidence ledger", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_ledger.json" + }, + { + "label": "world-class evidence intake", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_intake.json" + }, + { + "label": "world-class evidence preflight", + "command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_preflight.json" + }, + { + "label": "world-class submission review", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_submission_review.json" + }, + { + "label": "world-class operator runbook", + "command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_operator_runbook.json" + }, + { + "label": "world-class claim guard", + "command": "python3 scripts/yao.py world-class-claim-guard .", + "evidence": "reports/world_class_claim_guard.json" + }, + { + "label": "evidence consistency", + "command": "python3 scripts/yao.py evidence-consistency .", + "evidence": "reports/evidence_consistency.json" + }, + { + "label": "full ci", + "command": "make ci-test", + "evidence": "CI target output" + } + ], + "failure_disclosure": { + "path": "evals/failure-cases.md", + "case_count": 3, + "policy": "Keep representative failures visible and tied to regression checks." + }, + "limitations": [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims." + ], + "artifacts": { + "json": "reports/benchmark_reproducibility.json", + "markdown": "reports/benchmark_reproducibility.md" + } +} diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md new file mode 100644 index 00000000..f0a94816 --- /dev/null +++ b/reports/benchmark_reproducibility.md @@ -0,0 +1,181 @@ +# Benchmark Reproducibility + +Generated at: `2026-06-17` +Commit: `e79050ed12d057811d7d13ab7d8bcc5a06af76d4` +Working tree dirty at generation: `true` +Source tree dirty at generation: `false` +Generated evidence dirty at generation: `true` +Evidence bundle SHA256: `55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0` + +## Summary + +- reproducibility ready: `true` +- release lock ready: `true` +- methodology complete: `true` +- required artifacts: `25` +- missing artifacts: `0` +- source contract sha256: `5a99ef3133b0` +- archive sha256: `7db44f059844` +- output cases: `5` +- disclosed failure cases: `3` +- reproduction commands: `23` +- provider evidence complete: `true` +- human review complete: `false` +- world-class ready: `false` +- world-class source checks: `11` pass / `19` total; `8` blocked +- beta test ready: `true` +- beta test blockers: `0` +- beta deferred evidence: `4` +- public claim ready: `false` +- public claim blockers: `3` +- changed files at generation: `29` +- source changed files at generation: `0` +- generated changed files at generation: `29` + +This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. + +## Beta Test Boundary + +- ready: `true` +- scope: beta/public test release without superiority, fully-reviewed, or world-class claims +- policy: Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted. +- required wording: Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline. + +| Blocker | +| --- | +| none | + +| Deferred evidence | Reason | +| --- | --- | +| `provider-holdout` | Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims. | +| `human-adjudication` | Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims. | +| `native-permission-enforcement` | Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims. | +| `native-client-telemetry` | Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims. | + +## Public Claim Boundary + +- ready: `false` +- scope: public benchmark or world-class readiness claim +- policy: Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks. + +| Blocker | +| --- | +| human blind-review adjudication is incomplete | +| world-class evidence is not accepted yet (3 open gaps, 4 ledger pending) | +| world-class source checks are not all accepted (11/19 pass, 8 blocked) | + +## Release Lock + +- ready: `true` +- reason: only generated evidence artifacts were dirty at generation time +- status scope: generation-time status before this report is written + +## Evidence Bundle + +- algorithm: `sha256(path,label,exists,artifact_sha256)` +- artifacts: `25` / `25` +- sha256: `55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0` + +## Methodology Sections + +| Section | Status | +| --- | --- | +| `## Benchmark Types` | present | +| `## Sample Sources` | present | +| `## Evaluation Dimensions` | present | +| `## Weighting Rule` | present | +| `## Failure Disclosure` | present | +| `## Reproduction` | present | + +## Required Artifacts + +| Label | Path | Status | SHA256 | +| --- | --- | --- | --- | +| methodology | `reports/benchmark_methodology.md` | present | `57025e0123ce` | +| failure_disclosure | `evals/failure-cases.md` | present | `28833c0d4a21` | +| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | +| output_schema | `evals/output/schema.json` | present | `8ee340c95064` | +| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | +| output_execution | `reports/output_execution_runs.json` | present | `4df66b63d2e7` | +| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | +| review_adjudication | `reports/output_review_adjudication.json` | present | `91fd88dd9b0f` | +| trigger_scorecard | `reports/route_scorecard.json` | present | `a649547a7d3d` | +| runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | +| trust_report | `reports/security_trust_report.json` | present | `bbff1f1bee0c` | +| python_compatibility | `reports/python_compatibility.json` | present | `d31dc03c74f1` | +| registry_audit | `reports/registry_audit.json` | present | `1feb80c7cdc8` | +| package_verification | `reports/package_verification.json` | present | `7aa6c25e8154` | +| install_simulation | `reports/install_simulation.json` | present | `c1f1122264b8` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `6362e79a8cfd` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `8eb5e1fbaa75` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `6038b6cd6e06` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `aa262da0ecee` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `95cb56581224` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `12fbb8492629` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `f321dc118ee9` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `858ee11bbacb` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `31b24658c59e` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `abe7f7d60c00` | + +## Reproduction Commands + +- `git rev-parse HEAD` + - evidence: `git commit hash` +- `make eval-suite` + - evidence: `reports/eval_suite.json` +- `python3 scripts/yao.py output-eval` + - evidence: `reports/output_quality_scorecard.json` +- `python3 scripts/yao.py output-exec --runner-command '["python3","scripts/local_output_eval_runner.py"]'` + - evidence: `reports/output_execution_runs.json` +- `python3 scripts/yao.py output-review` + - evidence: `reports/output_review_adjudication.json` +- `python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json` + - evidence: `skill-ir/examples/yao-meta-skill.json` +- `python3 scripts/yao.py conformance .` + - evidence: `reports/conformance_matrix.json` +- `python3 scripts/yao.py trust .` + - evidence: `reports/security_trust_report.json` +- `python3 scripts/yao.py python-compat .` + - evidence: `reports/python_compatibility.json` +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip` + - evidence: `dist/yao-meta-skill.zip` +- `python3 scripts/yao.py package-verify . --package-dir dist --require-zip` + - evidence: `reports/package_verification.json` +- `python3 scripts/yao.py install-simulate . --package-dir dist` + - evidence: `reports/install_simulation.json` +- `python3 scripts/yao.py registry-audit .` + - evidence: `reports/registry_audit.json` +- `python3 scripts/yao.py skill-os2-audit .` + - evidence: `reports/skill_os2_audit.json` +- `python3 scripts/yao.py world-class-evidence .` + - evidence: `reports/world_class_evidence_plan.json` +- `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` + - evidence: `reports/world_class_evidence_ledger.json` +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + - evidence: `reports/world_class_evidence_intake.json` +- `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` + - evidence: `reports/world_class_evidence_preflight.json` +- `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` + - evidence: `reports/world_class_submission_review.json` +- `python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions` + - evidence: `reports/world_class_operator_runbook.json` +- `python3 scripts/yao.py world-class-claim-guard .` + - evidence: `reports/world_class_claim_guard.json` +- `python3 scripts/yao.py evidence-consistency .` + - evidence: `reports/evidence_consistency.json` +- `make ci-test` + - evidence: `CI target output` + +## Failure Disclosure + +- path: `evals/failure-cases.md` +- disclosed cases: `3` +- policy: Keep representative failures visible and tied to regression checks. + +## Limits + +- The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately. +- Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet. +- Pending blind-review decisions are visible but do not count as human adjudication. +- World-class readiness remains false until external and human evidence gaps close. +- Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims. diff --git a/reports/candidate_registry.json b/reports/candidate_registry.json index 166fea3c..192a53c8 100644 --- a/reports/candidate_registry.json +++ b/reports/candidate_registry.json @@ -3,9 +3,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -36,44 +36,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -164,7 +164,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.611, + "average_margin": 0.701, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -187,8 +187,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -218,7 +218,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -232,7 +233,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -532,13 +534,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -557,7 +559,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -572,49 +574,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -677,7 +679,8 @@ "causes": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", "gate_status": { @@ -693,7 +696,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.659, + "average_margin": 0.586, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -707,8 +710,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -718,25 +721,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -745,7 +733,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -759,7 +747,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -773,9 +761,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } diff --git a/reports/candidate_registry.md b/reports/candidate_registry.md index 1dd6df26..d6eae6ea 100644 --- a/reports/candidate_registry.md +++ b/reports/candidate_registry.md @@ -3,10 +3,10 @@ | Target | Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors | Reason Tags | | --- | --- | --- | --- | --- | ---: | ---: | ---: | --- | | `yao-meta-skill` | baseline | `Baseline` | reference | reference | 8 | 1 | 0 | - | -| `yao-meta-skill` | current | `Current` | selected_by_dev | kept_current | 65 | 0 | 0 | - | +| `yao-meta-skill` | current | `Current` | selected_by_dev | kept_current | 53 | 0 | 0 | - | | `yao-meta-skill` | candidate | `Minimal` | not_selected | blocked | 41 | 2 | 0 | weaker_dev_fit | -| `yao-meta-skill` | candidate | `Guardrail` | not_selected | blocked | 56 | 2 | 0 | weaker_dev_fit | -| `yao-meta-skill` | candidate | `Balanced` | not_selected | blocked | 60 | 2 | 0 | weaker_dev_fit | +| `yao-meta-skill` | candidate | `Guardrail` | not_selected | blocked | 56 | 2 | 0 | weaker_dev_fit, longer_without_gain | +| `yao-meta-skill` | candidate | `Balanced` | not_selected | blocked | 60 | 2 | 0 | weaker_dev_fit, longer_without_gain | | `yao-meta-skill` | candidate | `Artifact Aware` | not_selected | blocked | 77 | 2 | 0 | weaker_dev_fit, longer_without_gain | | `yao-meta-skill` | candidate | `Boundary` | not_selected | blocked | 83 | 2 | 0 | weaker_dev_fit, longer_without_gain | | `team-frontend-review` | baseline | `Baseline` | reference | reference | 52 | 3 | 0 | - | @@ -15,9 +15,9 @@ | `team-frontend-review` | candidate | `Balanced` | not_selected | blocked | 64 | 3 | 0 | longer_without_gain | | `team-frontend-review` | candidate | `Artifact Aware` | not_selected | blocked | 84 | 3 | 0 | longer_without_gain | | `team-frontend-review` | candidate | `Boundary` | not_selected | blocked | 90 | 3 | 0 | longer_without_gain | -| `governed-incident-command` | baseline | `Baseline` | reference | reference | 93 | 1 | 2 | - | -| `governed-incident-command` | current | `Current` | selected_by_dev | kept_current | 37 | 1 | 1 | - | -| `governed-incident-command` | candidate | `Guardrail` | not_selected | blocked | 51 | 1 | 2 | visible_holdout_regression, longer_without_gain | -| `governed-incident-command` | candidate | `Balanced` | not_selected | blocked | 54 | 1 | 1 | longer_without_gain | -| `governed-incident-command` | candidate | `Boundary` | not_selected | blocked | 78 | 1 | 1 | longer_without_gain | -| `governed-incident-command` | candidate | `Artifact Aware` | not_selected | blocked | 78 | 1 | 1 | longer_without_gain | +| `governed-incident-command` | baseline | `Baseline` | reference | reference | 93 | 0 | 1 | - | +| `governed-incident-command` | current | `Current` | selected_by_dev | kept_current | 37 | 0 | 1 | - | +| `governed-incident-command` | candidate | `Balanced` | not_selected | blocked | 54 | 0 | 1 | longer_without_gain | +| `governed-incident-command` | candidate | `Boundary` | not_selected | blocked | 78 | 0 | 1 | longer_without_gain | +| `governed-incident-command` | candidate | `Artifact Aware` | not_selected | blocked | 78 | 0 | 1 | longer_without_gain | +| `governed-incident-command` | candidate | `Guardrail` | not_selected | blocked | 51 | 1 | 1 | weaker_dev_fit, longer_without_gain | diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index fc077654..f9f14ed0 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,11 +1,11 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { - "target_count": 4, - "pass_count": 4, + "target_count": 5, + "pass_count": 5, "warn_count": 0, "block_count": 0, "failure_count": 0, @@ -30,17 +30,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -76,21 +77,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -112,6 +110,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -139,39 +138,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -186,32 +217,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -251,6 +343,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -259,15 +353,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -304,7 +399,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -338,7 +433,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -355,18 +451,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -375,18 +473,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -401,15 +510,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -418,15 +541,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -441,22 +569,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -476,9 +605,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -490,17 +619,17 @@ "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", "activation": { "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/openai/adapter.json", @@ -509,7 +638,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -580,18 +709,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -600,18 +731,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -626,15 +768,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -643,15 +799,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -666,22 +827,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -699,9 +861,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -713,17 +875,17 @@ "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", "activation": { "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/openai/adapter.json", @@ -732,7 +894,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -819,17 +981,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -865,21 +1028,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -901,6 +1061,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -928,39 +1089,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -975,32 +1168,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -1040,6 +1294,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -1048,15 +1304,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -1093,7 +1350,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -1127,7 +1384,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -1144,18 +1402,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -1164,18 +1424,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1190,15 +1461,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -1207,15 +1492,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -1230,22 +1520,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -1265,9 +1556,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -1279,17 +1570,17 @@ "native_surface": "Claude-compatible neutral source folder with adapter notes", "activation": { "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/claude/adapter.json", @@ -1298,7 +1589,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -1369,18 +1660,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -1389,18 +1682,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1415,15 +1719,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -1432,15 +1750,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -1455,22 +1778,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -1488,9 +1812,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -1502,17 +1826,17 @@ "native_surface": "Claude-compatible neutral source folder with adapter notes", "activation": { "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/claude/adapter.json", @@ -1521,7 +1845,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -1608,17 +1932,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -1654,21 +1979,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -1690,6 +2012,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -1717,39 +2040,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1764,32 +2119,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -1829,6 +2245,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -1837,15 +2255,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -1882,7 +2301,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -1916,7 +2335,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -1933,18 +2353,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -1953,18 +2375,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1979,15 +2412,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -1996,15 +2443,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -2019,22 +2471,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -2054,9 +2507,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -2068,17 +2521,17 @@ "native_surface": "Agent Skills compatible neutral package", "activation": { "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/generic/adapter.json" @@ -2086,7 +2539,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -2151,18 +2604,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -2171,18 +2626,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2197,15 +2663,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -2214,15 +2694,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -2237,22 +2722,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -2270,9 +2756,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -2284,17 +2770,17 @@ "native_surface": "Agent Skills compatible neutral package", "activation": { "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/generic/adapter.json" @@ -2302,7 +2788,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -2381,17 +2867,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -2427,21 +2914,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -2463,6 +2947,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -2490,39 +2975,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2537,32 +3054,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -2602,6 +3180,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -2610,15 +3190,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -2655,7 +3236,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -2689,7 +3270,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -2706,18 +3288,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -2726,18 +3310,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2752,15 +3347,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -2769,15 +3378,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -2792,22 +3406,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -2827,9 +3442,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -2841,17 +3456,17 @@ "native_surface": "Agent Skills standard source tree", "activation": { "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "SKILL.md", @@ -2860,7 +3475,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -2924,18 +3539,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -2944,18 +3561,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -2970,15 +3598,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -2987,15 +3629,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -3010,22 +3657,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -3043,9 +3691,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -3057,17 +3705,17 @@ "native_surface": "Agent Skills standard source tree", "activation": { "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "SKILL.md", @@ -3076,7 +3724,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -3135,6 +3783,951 @@ "unsupported_features": [], "warnings": [], "failures": [] + }, + { + "schema_version": "1.0", + "target": "vscode", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/autonomous-adaptation.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/skillops-decision-policy.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md", + "references/user-memory-policy.md" + ], + "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", + "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", + "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", + "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", + "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", + "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", + "scripts/skill_report_layout.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", + "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/trust_check_scripts.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", + "scripts/yao.py", + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" + ], + "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 27, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 3, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 73, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", + "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", + "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 12, + "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", + "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py", + "scripts/yao_cli_runtime.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 86, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "vscode", + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 3, + "file_write": 73, + "subprocess": 12, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 153, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "target_runtime": { + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "VS Code/Copilot Agent Skills project or user scope" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 3, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 73, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", + "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", + "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 12, + "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", + "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py", + "scripts/yao_cli_runtime.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 86, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 3, + "file_write": 73, + "subprocess": 12, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 153, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "target_transform": { + "target": "vscode", + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "permission_representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "permission_enforcement": "client-or-workspace-trust" + }, + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [], + "failures": [] } ], "failures": [], diff --git a/reports/compiled_targets.md b/reports/compiled_targets.md index ea2ddcb5..f6a11f9b 100644 --- a/reports/compiled_targets.md +++ b/reports/compiled_targets.md @@ -1,8 +1,8 @@ # Compiled Targets - OK: `True` -- Targets: `4` -- Pass: `4` +- Targets: `5` +- Pass: `5` - Warn: `0` - Block: `0` @@ -14,6 +14,7 @@ | `claude` | `pass` | Claude-compatible neutral source folder with adapter notes | `neutral-source-plus-adapter` | `network, file_write, subprocess` | `neutral-source-plus-adapter` | targets/claude/adapter.json, targets/claude/README.md | | `generic` | `pass` | Agent Skills compatible neutral package | `agent-skills-compatible` | `network, file_write, subprocess` | `neutral-source` | targets/generic/adapter.json | | `agent-skills-compatible` | `pass` | Agent Skills standard source tree | `neutral-agent-skills-source` | `network, file_write, subprocess` | `neutral-source` | SKILL.md, agents/interface.yaml | +| `vscode` | `pass` | VS Code/Copilot Agent Skills project or user scope | `vscode-agent-skills-adapter` | `network, file_write, subprocess` | `agent-skills-source-with-vscode-notes` | targets/vscode/adapter.json, targets/vscode/README.md | ## Native Behavior Contracts @@ -53,6 +54,15 @@ - Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False` - Review artifacts: SKILL.md, agents/interface.yaml, reports/review-studio.html +### vscode + +- Native surface: VS Code/Copilot Agent Skills project or user scope +- Activation: Use folder name plus SKILL.md name/description; keep description under platform limits. +- Resources: Install as project or user scoped skill source, preserving relative references and scripts. +- Scripts: Scripts require workspace trust and operator/client approval outside this compiler. +- Permission enforcement: `client-or-workspace-trust`; native enforcement `False` +- Review artifacts: SKILL.md, agents/interface.yaml, reports/review-studio.html + ## Failures diff --git a/reports/conformance_matrix.json b/reports/conformance_matrix.json index 20e9bcea..722a8ef7 100644 --- a/reports/conformance_matrix.json +++ b/reports/conformance_matrix.json @@ -32,6 +32,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -40,8 +41,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [] @@ -76,6 +76,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -84,8 +85,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [] @@ -99,7 +99,7 @@ "frontmatter description exists", "description length <= 1024", "name is runtime-safe", - "directory name matches skill name", + "package identity derives from skill name", "manifest.name exists", "manifest.version exists", "manifest.owner exists", @@ -120,6 +120,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -128,8 +129,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [ @@ -145,14 +145,14 @@ "frontmatter description exists", "description length <= 1024", "name is runtime-safe", - "directory name matches skill name", + "package identity derives from skill name", "manifest.name exists", "manifest.version exists", "manifest.owner exists", "manifest.status exists", "manifest.maturity_tier exists", "manifest.review_cadence exists", - "manifest declares vscode via agent-skills-compatible", + "manifest declares vscode", "interface.display_name exists", "interface.short_description exists", "interface.default_prompt exists", @@ -166,6 +166,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -174,8 +175,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [ @@ -212,6 +212,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -220,8 +221,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [] diff --git a/reports/context_budget.json b/reports/context_budget.json index 49e96a06..f7f9638e 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -5,13 +5,70 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 811, - "other_text_tokens": 869504, - "estimated_initial_load_tokens": 987, - "estimated_total_text_tokens": 870315, - "relevant_file_count": 327, + "skill_body_tokens": 805, + "other_text_tokens": 1156781, + "estimated_initial_load_tokens": 998, + "estimated_total_text_tokens": 1157586, + "deferred_resource_tokens": 525864, + "deferred_resource_warn_threshold": 120000, + "deferred_resource_dirs": [ + { + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153 + }, + { + "path": "references", + "estimated_tokens": 28292, + "file_count": 35 + }, + { + "path": "evals", + "estimated_tokens": 21835, + "file_count": 27 + }, + { + "path": "assets", + "estimated_tokens": 11176, + "file_count": 4 + } + ], + "large_deferred_resource_dirs": [ + { + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153 + } + ], + "deferred_resource_governance": { + "status": "governed", + "large_dir_count": 1, + "governed_large_dir_count": 1, + "directories": [ + { + "status": "governed", + "evidence": [ + "reports/security_trust_report.json", + "reports/architecture_maintainability.json", + "reports/python_compatibility.json" + ], + "reasons": [ + "trust report covers scripts", + "architecture report has no script hotspots or blockers", + "Python compatibility report has no issues" + ], + "missing": [], + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153, + "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." + } + ], + "summary": "Large deferred resources are indexed and backed by evidence." + }, + "relevant_file_count": 712, "unused_resource_dirs": [], - "quality_signal_points": 130, - "quality_density": 131.7 + "quality_signal_points": 140, + "quality_density": 140.3 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index 9d678e58..46bab4f9 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -1,9 +1,9 @@ # Context Budget Summary -| Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status | -| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | -| root | `.` | `production` | 1000 | 987 | 811 | 131.7 | - | ok | -| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok | -| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok | +| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | +| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | +| root | `.` | `production` | 1000 | 998 | 805 | 525864 | `governed` | scripts:464561 | 140.3 | - | ok | +| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | +| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | Per-target JSON reports are written beside each target and under `reports/`. diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index ac17bd28..3a08c3b6 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,9 +6,43 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 987, - "skill_body_tokens": 811, - "quality_density": 131.7, + "initial_tokens": 998, + "skill_body_tokens": 805, + "deferred_resource_tokens": 525864, + "large_deferred_resource_dirs": [ + { + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153 + } + ], + "deferred_resource_governance": { + "status": "governed", + "large_dir_count": 1, + "governed_large_dir_count": 1, + "directories": [ + { + "status": "governed", + "evidence": [ + "reports/security_trust_report.json", + "reports/architecture_maintainability.json", + "reports/python_compatibility.json" + ], + "reasons": [ + "trust report covers scripts", + "architecture report has no script hotspots or blockers", + "Python compatibility report has no issues" + ], + "missing": [], + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153, + "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." + } + ], + "summary": "Large deferred resources are indexed and backed by evidence." + }, + "quality_density": 140.3, "unused_resource_dirs": [], "ok": true, "warnings": [] @@ -20,6 +54,15 @@ "budget_limit": 1000, "initial_tokens": 790, "skill_body_tokens": 718, + "deferred_resource_tokens": 1657, + "large_deferred_resource_dirs": [], + "deferred_resource_governance": { + "status": "not-required", + "large_dir_count": 0, + "governed_large_dir_count": 0, + "directories": [], + "summary": "No large deferred resource directory exceeds the per-dir threshold." + }, "quality_density": 164.6, "unused_resource_dirs": [], "ok": true, @@ -32,6 +75,15 @@ "budget_limit": 1000, "initial_tokens": 760, "skill_body_tokens": 658, + "deferred_resource_tokens": 1030, + "large_deferred_resource_dirs": [], + "deferred_resource_governance": { + "status": "not-required", + "large_dir_count": 0, + "governed_large_dir_count": 0, + "directories": [], + "summary": "No large deferred resource directory exceeds the per-dir threshold." + }, "quality_density": 171.1, "unused_resource_dirs": [], "ok": true, diff --git a/reports/description_optimization.json b/reports/description_optimization.json index dc4d7662..08f61621 100644 --- a/reports/description_optimization.json +++ b/reports/description_optimization.json @@ -1,11 +1,11 @@ { - "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "current_candidate": { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -21,7 +21,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -2148,9 +2148,9 @@ "winner": { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -2166,7 +2166,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -2321,18 +2321,16 @@ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", - "score": 0.415, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -2342,9 +2340,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -2364,33 +2362,32 @@ { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", - "score": 0.394, + "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.327, - "support_score": 0, - "lexical_support": 0.417, + "semantic_coverage": 0.333, + "support_score": 0.16, + "lexical_support": 0.25, "negative_penalty": 0, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ @@ -2415,32 +2412,31 @@ { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", - "score": 0.843, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -2471,18 +2467,16 @@ { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", - "score": 0.422, + "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -2494,7 +2488,7 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, "lexical_support": 0.4, "negative_penalty": 0, @@ -2520,31 +2514,30 @@ { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", - "score": 0.557, + "score": 0.742, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.16, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, @@ -2571,18 +2564,16 @@ { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", - "score": 0.474, + "score": 0.763, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -2593,9 +2584,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0, - "lexical_support": 0.571, + "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -2619,32 +2610,31 @@ { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", - "score": 0.868, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0, + "semantic_coverage": 1.0, + "support_score": 0.34, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, @@ -2677,35 +2667,33 @@ { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", - "score": 0.889, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0.06, - "lexical_support": 0.346, + "semantic_coverage": 1.0, + "support_score": 0.4, + "lexical_support": 0.231, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -2752,11 +2740,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -2769,9 +2755,9 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { @@ -2798,11 +2784,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -2839,11 +2823,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -2860,9 +2842,9 @@ "brainstorm_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.2, + "lexical_support": 0.133, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { @@ -2896,11 +2878,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -2913,7 +2893,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.167, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { @@ -2937,11 +2917,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -2954,9 +2932,9 @@ "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.111, + "lexical_support": 0.222, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { @@ -2983,11 +2961,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3002,9 +2978,9 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.174, + "lexical_support": 0.087, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { @@ -3029,37 +3005,36 @@ { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", - "score": 0.256, + "score": 0.306, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.333, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.222, "negative_penalty": 0.3, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -3094,11 +3069,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3114,7 +3087,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, @@ -3149,11 +3122,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3166,9 +3137,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.143, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { @@ -3195,18 +3166,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -3214,11 +3184,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.3, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -3249,18 +3219,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "iterate_existing_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -3269,11 +3238,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.327, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.16, "lexical_support": 0.267, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "iterate_existing_skill": [ @@ -3304,18 +3273,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -3323,11 +3291,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.286, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -3359,18 +3327,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "future_outline", @@ -3380,11 +3347,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.238, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.19, "negative_penalty": 0.82, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -3423,18 +3390,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -3443,11 +3409,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.34, "lexical_support": 0.263, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -3588,18 +3554,16 @@ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", - "score": 0.468, + "score": 0.767, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3612,9 +3576,9 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0.06, - "lexical_support": 0.316, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3641,33 +3605,32 @@ { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", - "score": 0.727, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.691, - "support_score": 0, - "lexical_support": 0.389, + "semantic_coverage": 1.0, + "support_score": 0.16, + "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3698,18 +3661,16 @@ { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", - "score": 0.568, + "score": 0.936, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3720,9 +3681,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, + "semantic_coverage": 0.933, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3745,34 +3706,32 @@ { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", - "score": 0.858, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0.06, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.4, + "lexical_support": 0.312, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3803,32 +3762,31 @@ { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", - "score": 0.744, + "score": 0.839, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.709, - "support_score": 0, - "lexical_support": 0.4, + "semantic_coverage": 0.733, + "support_score": 0.34, + "lexical_support": 0.24, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3865,11 +3823,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3885,9 +3841,9 @@ "no_build_directive", "summarize_only" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { @@ -3921,11 +3877,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3941,7 +3895,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, @@ -3977,11 +3931,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3995,7 +3947,7 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, @@ -4029,19 +3981,18 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "no_build_directive", "translate_only" @@ -4050,11 +4001,11 @@ "no_build_directive", "translate_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.25, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.188, "negative_penalty": 0.77, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -4092,17 +4043,15 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "matched_desired_concepts": [ + "matched_desired_concepts": [], + "extra_positive_concepts": [ "build_skill" ], - "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" @@ -4111,9 +4060,9 @@ "brainstorm_only", "future_outline" ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, + "semantic_coverage": 0.0, + "support_score": 0.34, + "lexical_support": 0.077, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { @@ -4143,11 +4092,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -4162,7 +4109,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.214, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -4190,18 +4137,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -4209,11 +4155,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.615, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.308, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -4239,26 +4185,25 @@ { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", - "score": 0.0, + "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -4266,11 +4211,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.673, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.34, "lexical_support": 0.438, "negative_penalty": 0.6, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -4443,18 +4388,16 @@ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", - "score": 0.415, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4464,9 +4407,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -4486,33 +4429,32 @@ { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", - "score": 0.394, + "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.327, - "support_score": 0, - "lexical_support": 0.417, + "semantic_coverage": 0.333, + "support_score": 0.16, + "lexical_support": 0.25, "negative_penalty": 0, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ @@ -4537,32 +4479,31 @@ { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", - "score": 0.843, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4593,18 +4534,16 @@ { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", - "score": 0.422, + "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4616,7 +4555,7 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, "lexical_support": 0.4, "negative_penalty": 0, @@ -4642,31 +4581,30 @@ { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", - "score": 0.557, + "score": 0.742, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.16, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, @@ -4693,18 +4631,16 @@ { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", - "score": 0.474, + "score": 0.763, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4715,9 +4651,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0, - "lexical_support": 0.571, + "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4741,32 +4677,31 @@ { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", - "score": 0.868, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0, + "semantic_coverage": 1.0, + "support_score": 0.34, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, @@ -4799,35 +4734,33 @@ { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", - "score": 0.889, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0.06, - "lexical_support": 0.346, + "semantic_coverage": 1.0, + "support_score": 0.4, + "lexical_support": 0.231, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4874,11 +4807,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4891,9 +4822,9 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { @@ -4920,11 +4851,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -4961,11 +4890,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4982,9 +4909,9 @@ "brainstorm_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.2, + "lexical_support": 0.133, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { @@ -5018,11 +4945,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -5035,7 +4960,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.167, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { @@ -5059,11 +4984,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5076,9 +4999,9 @@ "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.111, + "lexical_support": 0.222, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { @@ -5105,11 +5028,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5124,9 +5045,9 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.174, + "lexical_support": 0.087, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { @@ -5151,37 +5072,36 @@ { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", - "score": 0.256, + "score": 0.306, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.333, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.222, "negative_penalty": 0.3, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -5216,11 +5136,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5236,7 +5154,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, @@ -5271,11 +5189,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5288,9 +5204,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.143, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { @@ -5317,18 +5233,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -5336,11 +5251,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.3, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5371,18 +5286,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "iterate_existing_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -5391,11 +5305,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.327, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.16, "lexical_support": 0.267, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "iterate_existing_skill": [ @@ -5426,18 +5340,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -5445,11 +5358,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.286, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5481,18 +5394,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "future_outline", @@ -5502,11 +5414,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.238, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.19, "negative_penalty": 0.82, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5545,18 +5457,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -5565,11 +5476,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.34, "lexical_support": 0.263, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5710,18 +5621,16 @@ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", - "score": 0.468, + "score": 0.767, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5734,9 +5643,9 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0.06, - "lexical_support": 0.316, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5763,33 +5672,32 @@ { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", - "score": 0.727, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.691, - "support_score": 0, - "lexical_support": 0.389, + "semantic_coverage": 1.0, + "support_score": 0.16, + "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5820,18 +5728,16 @@ { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", - "score": 0.568, + "score": 0.936, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5842,9 +5748,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, + "semantic_coverage": 0.933, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5867,34 +5773,32 @@ { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", - "score": 0.858, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0.06, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.4, + "lexical_support": 0.312, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5925,32 +5829,31 @@ { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", - "score": 0.744, + "score": 0.839, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.709, - "support_score": 0, - "lexical_support": 0.4, + "semantic_coverage": 0.733, + "support_score": 0.34, + "lexical_support": 0.24, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5987,11 +5890,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6007,9 +5908,9 @@ "no_build_directive", "summarize_only" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { @@ -6043,11 +5944,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6063,7 +5962,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, @@ -6099,11 +5998,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6117,7 +6014,7 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, @@ -6151,19 +6048,18 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "no_build_directive", "translate_only" @@ -6172,11 +6068,11 @@ "no_build_directive", "translate_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.25, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.188, "negative_penalty": 0.77, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -6214,17 +6110,15 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "matched_desired_concepts": [ + "matched_desired_concepts": [], + "extra_positive_concepts": [ "build_skill" ], - "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" @@ -6233,9 +6127,9 @@ "brainstorm_only", "future_outline" ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, + "semantic_coverage": 0.0, + "support_score": 0.34, + "lexical_support": 0.077, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { @@ -6265,11 +6159,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -6284,7 +6176,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.214, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -6312,18 +6204,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -6331,11 +6222,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.615, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.308, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6361,26 +6252,25 @@ { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", - "score": 0.0, + "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -6388,11 +6278,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.673, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.34, "lexical_support": 0.438, "negative_penalty": 0.6, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -6495,18 +6385,16 @@ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", - "score": 0.392, + "score": 0.665, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6516,9 +6404,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.214, + "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -6533,24 +6421,21 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", - "score": 0.405, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6560,9 +6445,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.375, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -6578,26 +6463,23 @@ }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", - "score": 0.028, + "score": 0.16, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6610,9 +6492,9 @@ "document_only" ], "exclusive_negative_concepts": [], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.318, + "lexical_support": 0.227, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { @@ -6642,11 +6524,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -6684,11 +6564,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -6703,7 +6581,7 @@ ], "semantic_coverage": 0.0, "support_score": 0.06, - "lexical_support": 0.308, + "lexical_support": 0.231, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { @@ -6723,18 +6601,16 @@ { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", - "score": 0.026, + "score": 0.305, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6748,7 +6624,7 @@ "exclusive_negative_concepts": [ "future_outline" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, @@ -6768,7 +6644,8 @@ ] } } - } + }, + "boundary_case": true } ] } @@ -6847,18 +6724,16 @@ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", - "score": 0.392, + "score": 0.665, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6868,9 +6743,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.214, + "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -6885,24 +6760,21 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", - "score": 0.405, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6912,9 +6784,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.375, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -6930,26 +6802,23 @@ }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", - "score": 0.028, + "score": 0.16, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6962,9 +6831,9 @@ "document_only" ], "exclusive_negative_concepts": [], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.318, + "lexical_support": 0.227, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { @@ -6994,11 +6863,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7036,11 +6903,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7055,7 +6920,7 @@ ], "semantic_coverage": 0.0, "support_score": 0.06, - "lexical_support": 0.308, + "lexical_support": 0.231, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { @@ -7075,18 +6940,16 @@ { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", - "score": 0.026, + "score": 0.305, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7100,7 +6963,7 @@ "exclusive_negative_concepts": [ "future_outline" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, @@ -7120,7 +6983,8 @@ ] } } - } + }, + "boundary_case": true } ] } @@ -8586,32 +8450,31 @@ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", - "score": 0.834, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.263, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8638,32 +8501,31 @@ { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", - "score": 0.842, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.368, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -8698,11 +8560,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8719,9 +8579,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, - "lexical_support": 0.429, + "lexical_support": 0.357, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { @@ -8757,11 +8617,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8779,9 +8637,9 @@ "summarize_only", "translate_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.188, + "lexical_support": 0.125, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { @@ -8820,11 +8678,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -8839,7 +8695,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.2, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -8868,11 +8724,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8889,9 +8743,9 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.182, + "lexical_support": 0.273, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { @@ -8991,32 +8845,31 @@ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", - "score": 0.834, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.263, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -9043,32 +8896,31 @@ { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", - "score": 0.842, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.368, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -9103,11 +8955,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9124,9 +8974,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, - "lexical_support": 0.429, + "lexical_support": 0.357, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { @@ -9162,11 +9012,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9184,9 +9032,9 @@ "summarize_only", "translate_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.188, + "lexical_support": 0.125, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { @@ -9225,11 +9073,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -9244,7 +9090,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.2, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -9273,11 +9119,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9294,9 +9138,9 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.182, + "lexical_support": 0.273, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { @@ -9725,9 +9569,9 @@ { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -9743,7 +9587,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -10077,30 +9921,30 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" @@ -10178,33 +10022,33 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "baseline_calibration": { "threshold": 0.33, @@ -10353,14 +10197,14 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -10368,14 +10212,14 @@ }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -10430,8 +10274,8 @@ }, "summary": { "winner_label": "Current", - "winner_tokens": 65, - "current_tokens": 65, + "winner_tokens": 53, + "current_tokens": 53, "winner_dev_total_errors": 0, "current_dev_total_errors": 0, "winner_holdout_total_errors": 0, @@ -10444,7 +10288,7 @@ "current_adversarial_holdout_total_errors": 0, "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", - "winner_adversarial_score_gap": 0.834, + "winner_adversarial_score_gap": 1.0, "candidate_count": 6, "baseline_tokens": 8, "baseline_dev_total_errors": 1, diff --git a/reports/description_optimization.md b/reports/description_optimization.md index 3081aa26..3f86caf9 100644 --- a/reports/description_optimization.md +++ b/reports/description_optimization.md @@ -2,19 +2,19 @@ Winner: `Current` -- current tokens: `65` -- winner tokens: `65` +- current tokens: `53` +- winner tokens: `53` - baseline tokens: `8` ## Winner -Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. ## Candidate Ranking | Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| `Current` | 65 | 0 | 0 | 1.0 | 0 | 0 | +| `Current` | 53 | 0 | 0 | 1.0 | 0 | 0 | | `Minimal` | 41 | 1 | 1 | 1.0 | 0 | 0 | | `Guardrail` | 56 | 1 | 1 | 1.0 | 0 | 0 | | `Balanced` | 60 | 1 | 1 | 1.0 | 0 | 0 | @@ -34,9 +34,9 @@ Create, refactor, evaluate, and package agent skills from workflows, prompts, tr | Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap | | --- | ---: | --- | ---: | ---: | ---: | -| Holdout | 0.468 | healthy | 0.0 | 0.468 | 0.416 | -| Blind Holdout | 0.364 | watch | 0.333 | 0.364 | 0.368 | -| Adversarial Holdout | 0.834 | healthy | 0.0 | 0.834 | 0.845 | +| Holdout | 0.744 | healthy | 0.0 | 0.744 | 0.416 | +| Blind Holdout | 0.36 | tight | 0.167 | 0.36 | 0.368 | +| Adversarial Holdout | 1.0 | healthy | 0.0 | 1.0 | 0.845 | ## Judge Blind Summary diff --git a/reports/description_optimization_suite.json b/reports/description_optimization_suite.json index b80cda28..46800ec1 100644 --- a/reports/description_optimization_suite.json +++ b/reports/description_optimization_suite.json @@ -3,9 +3,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -36,44 +36,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -295,13 +295,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -320,7 +320,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -335,49 +335,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, diff --git a/reports/description_optimization_suite.md b/reports/description_optimization_suite.md index a93a7ecb..6344d2fa 100644 --- a/reports/description_optimization_suite.md +++ b/reports/description_optimization_suite.md @@ -2,9 +2,9 @@ | Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Judge Blind Errors | Adv FP | Adv FN | Adv Gap | Adv Risk | Status | | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | -| `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.834 | healthy | ok | +| `yao-meta-skill` | `Current` | 53 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1.0 | healthy | ok | | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.509 | healthy | ok | -| `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0.598 | healthy | ok | +| `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0.538 | watch | ok | ## Family Coverage diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json new file mode 100644 index 00000000..ed224071 --- /dev/null +++ b/reports/evidence_consistency.json @@ -0,0 +1,3611 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "check_count": 39, + "pass_count": 39, + "warn_count": 0, + "fail_count": 0, + "decision": "consistent" + }, + "status_counts": { + "pass": 39, + "warn": 0, + "fail": 0 + }, + "checks": [ + { + "key": "required-report-artifacts", + "label": "Required report artifacts are readable", + "status": "pass", + "expected": "all required JSON and Markdown reports exist and parse", + "actual": "all readable", + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json", + "reports/skill-interpretation.json", + "reports/adoption_drift_report.json", + "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_plan.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_preflight.json", + "reports/world_class_submission_review.json", + "reports/world_class_operator_runbook.json", + "reports/skill_os2_coverage.json", + "reports/review-studio.json", + "reports/package_verification.json", + "reports/install_simulation.json", + "reports/security_trust_report.json", + "reports/context_budget.json", + "reports/world_class_claim_guard.json", + "reports/skill-os-2-review.md" + ], + "detail": "The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable." + }, + { + "key": "release-evidence-flow-covers-first-class-reports", + "label": "Release evidence flow covers first-class reports", + "status": "pass", + "expected": { + "AGENTS.md": true, + "source_refresh_section": true, + "clean_lock_section": true, + "source_refresh_commands": { + "python3 scripts/run_output_execution.py --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'": true, + "python3 scripts/compile_skill.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip": true, + "python3 scripts/simulate_install.py . --package-dir dist --install-root dist/install-simulation --output-json reports/install_simulation.json --output-md reports/install_simulation.md --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/trust_check.py . --output-json reports/security_trust_report.json --output-md reports/security_trust_report.md": true, + "python3 scripts/registry_audit.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_adoption_drift_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_architecture_maintainability.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/python_compat_check.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/probe_runtime_permissions.py . --package-dir dist": true, + "python3 scripts/render_review_waivers.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_review_annotations.py .": true, + "python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json --today \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_evidence_plan.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_evidence_ledger.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_evidence_intake.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_submission_review.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_operator_runbook.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_claim_guard.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_daily_skillops_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_weekly_curator_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_os2_audit.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_os2_coverage.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_context_reports.py --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_benchmark_reproducibility.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_overview.py .": true, + "python3 scripts/render_skill_interpretation.py .": true, + "python3 scripts/render_review_viewer.py .": true, + "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json": true, + "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"": true + }, + "clean_lock_commands": { + "python3 scripts/render_context_reports.py --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_benchmark_reproducibility.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_daily_skillops_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_weekly_curator_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_overview.py .": true, + "python3 scripts/render_skill_interpretation.py .": true, + "python3 scripts/render_review_viewer.py .": true, + "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json": true, + "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"": true + } + }, + "actual": { + "AGENTS.md": true, + "source_refresh_section": true, + "clean_lock_section": true, + "source_refresh_commands": { + "python3 scripts/run_output_execution.py --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'": true, + "python3 scripts/compile_skill.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip": true, + "python3 scripts/simulate_install.py . --package-dir dist --install-root dist/install-simulation --output-json reports/install_simulation.json --output-md reports/install_simulation.md --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/trust_check.py . --output-json reports/security_trust_report.json --output-md reports/security_trust_report.md": true, + "python3 scripts/registry_audit.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_adoption_drift_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_architecture_maintainability.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/python_compat_check.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/probe_runtime_permissions.py . --package-dir dist": true, + "python3 scripts/render_review_waivers.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_review_annotations.py .": true, + "python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json --today \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_evidence_plan.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_evidence_ledger.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_evidence_intake.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_submission_review.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_operator_runbook.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_world_class_claim_guard.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_daily_skillops_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_weekly_curator_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_os2_audit.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_os2_coverage.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_context_reports.py --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_benchmark_reproducibility.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_overview.py .": true, + "python3 scripts/render_skill_interpretation.py .": true, + "python3 scripts/render_review_viewer.py .": true, + "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json": true, + "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"": true + }, + "clean_lock_commands": { + "python3 scripts/render_context_reports.py --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_benchmark_reproducibility.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_daily_skillops_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_weekly_curator_report.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_skill_overview.py .": true, + "python3 scripts/render_skill_interpretation.py .": true, + "python3 scripts/render_review_viewer.py .": true, + "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"": true, + "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json": true, + "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"": true + } + }, + "paths": [ + "AGENTS.md", + "reports/output_execution_runs.json", + "reports/install_simulation.json", + "reports/security_trust_report.json", + "reports/registry_audit.json", + "reports/package_verification.json", + "reports/upgrade_check.json", + "reports/adoption_drift_report.json", + "reports/architecture_maintainability.json", + "reports/python_compatibility.json", + "reports/runtime_permission_probes.json", + "reports/review_waivers.json", + "reports/review_annotations.json", + "reports/skill_atlas.json", + "reports/skill_os2_audit.json", + "reports/skill_os2_coverage.json", + "reports/context_budget.json", + "reports/context_budget_summary.json", + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json", + "reports/skill-interpretation.json", + "reports/review-viewer.json", + "reports/world_class_evidence_preflight.json", + "reports/skillops/daily", + "reports/skillops/weekly", + "reports/review-studio.json", + "reports/evidence_consistency.json" + ], + "detail": "Release refresh and clean-lock instructions must regenerate every first-class report before evidence consistency can be trusted." + }, + { + "key": "review-studio-gate-action-mirror", + "label": "Review Studio gates mirror review actions", + "status": "pass", + "expected": { + "non_pass_gate_keys": [ + "output-lab", + "review-waivers", + "world-class-evidence" + ], + "action_gate_keys": [ + "output-lab", + "review-waivers", + "world-class-evidence" + ], + "pass_gate_action_ids_empty": true, + "gate_action_mirrors": { + "output-lab": { + "review_action_id": "review-action-output-lab", + "review_action_status": "warn", + "title_present": true, + "next_step_present": true, + "reason_present": true, + "source_ref_present": true, + "verification_command_present": true, + "top_level_action_id": "review-action-output-lab", + "top_level_action_status": "warn", + "top_level_title_present": true, + "top_level_next_step_present": true, + "top_level_reason_present": true, + "top_level_source_ref_present": true, + "top_level_verification_command_present": true + }, + "review-waivers": { + "review_action_id": "review-action-review-waivers", + "review_action_status": "warn", + "title_present": true, + "next_step_present": true, + "reason_present": true, + "source_ref_present": true, + "verification_command_present": true, + "top_level_action_id": "review-action-review-waivers", + "top_level_action_status": "warn", + "top_level_title_present": true, + "top_level_next_step_present": true, + "top_level_reason_present": true, + "top_level_source_ref_present": true, + "top_level_verification_command_present": true + }, + "world-class-evidence": { + "review_action_id": "review-action-world-class-evidence", + "review_action_status": "warn", + "title_present": true, + "next_step_present": true, + "reason_present": true, + "source_ref_present": true, + "verification_command_present": true, + "top_level_action_id": "review-action-world-class-evidence", + "top_level_action_status": "warn", + "top_level_title_present": true, + "top_level_next_step_present": true, + "top_level_reason_present": true, + "top_level_source_ref_present": true, + "top_level_verification_command_present": true + } + } + }, + "actual": { + "non_pass_gate_keys": [ + "output-lab", + "review-waivers", + "world-class-evidence" + ], + "action_gate_keys": [ + "output-lab", + "review-waivers", + "world-class-evidence" + ], + "pass_gate_action_ids_empty": true, + "gate_action_mirrors": { + "output-lab": { + "review_action_id": "review-action-output-lab", + "review_action_status": "warn", + "title_present": true, + "next_step_present": true, + "reason_present": true, + "source_ref_present": true, + "verification_command_present": true, + "top_level_action_id": "review-action-output-lab", + "top_level_action_status": "warn", + "top_level_title_present": true, + "top_level_next_step_present": true, + "top_level_reason_present": true, + "top_level_source_ref_present": true, + "top_level_verification_command_present": true + }, + "review-waivers": { + "review_action_id": "review-action-review-waivers", + "review_action_status": "warn", + "title_present": true, + "next_step_present": true, + "reason_present": true, + "source_ref_present": true, + "verification_command_present": true, + "top_level_action_id": "review-action-review-waivers", + "top_level_action_status": "warn", + "top_level_title_present": true, + "top_level_next_step_present": true, + "top_level_reason_present": true, + "top_level_source_ref_present": true, + "top_level_verification_command_present": true + }, + "world-class-evidence": { + "review_action_id": "review-action-world-class-evidence", + "review_action_status": "warn", + "title_present": true, + "next_step_present": true, + "reason_present": true, + "source_ref_present": true, + "verification_command_present": true, + "top_level_action_id": "review-action-world-class-evidence", + "top_level_action_status": "warn", + "top_level_title_present": true, + "top_level_next_step_present": true, + "top_level_reason_present": true, + "top_level_source_ref_present": true, + "top_level_verification_command_present": true + } + } + }, + "paths": [ + "reports/review-studio.json" + ], + "detail": "Every non-pass Review Studio gate must have exactly one top-level review action and a gate-local action summary with source refs and a verification command." + }, + { + "key": "review-studio-context-budget-mirror", + "label": "Review Studio mirrors context budget governance", + "status": "pass", + "expected": { + "status": "pass", + "detail": "initial load 998/1000; deferred 525864/120000; top deferred scripts 464561; resource governance governed; quality density 140.3", + "evidence": "reports/context_budget.json" + }, + "actual": { + "status": "pass", + "detail": "initial load 998/1000; deferred 525864/120000; top deferred scripts 464561; resource governance governed; quality density 140.3", + "evidence": "reports/context_budget.json" + }, + "paths": [ + "reports/context_budget.json", + "reports/review-studio.json" + ], + "detail": "Review Studio must not keep stale context warnings after context reports prove large deferred resources are governed." + }, + { + "key": "benchmark-release-lock-self-consistency", + "label": "Benchmark release lock matches source dirty state", + "status": "pass", + "expected": true, + "actual": true, + "paths": [ + "reports/benchmark_reproducibility.json" + ], + "detail": "The benchmark release lock must be blocked by source changes, while generated evidence artifacts are tracked as generation context." + }, + { + "key": "benchmark-clean-worktree-release-lock", + "label": "Clean worktree keeps a clean benchmark release lock", + "status": "pass", + "expected": "checked only when git is available and the current worktree is clean", + "actual": { + "available": true, + "clean": false, + "changed_file_count": 27 + }, + "paths": [ + "reports/benchmark_reproducibility.json" + ], + "detail": "Dirty or non-git worktrees cannot prove final release-lock freshness, so this check is advisory until the final clean-lock pass." + }, + { + "key": "skill-ir-evidence-path-contract", + "label": "Human-facing reports expose the canonical Skill IR artifact", + "status": "pass", + "expected": { + "overview_source_path": "skill-ir/examples/yao-meta-skill.json", + "interpretation_source_path": "skill-ir/examples/yao-meta-skill.json", + "review_studio_evidence_path": "skill-ir/examples/yao-meta-skill.json", + "overview_deliverable": true, + "interpretation_deliverable": true, + "exists": true, + "schema_version": "2.0.0" + }, + "actual": { + "overview_source_path": "skill-ir/examples/yao-meta-skill.json", + "interpretation_source_path": "skill-ir/examples/yao-meta-skill.json", + "review_studio_evidence_path": "skill-ir/examples/yao-meta-skill.json", + "overview_deliverable": true, + "interpretation_deliverable": true, + "exists": true, + "schema_version": "2.0.0" + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json", + "reports/review-studio.json", + "skill-ir/examples/yao-meta-skill.json" + ], + "detail": "Skill IR is the 2.0 platform-neutral semantic source, so user-facing reports must link to the artifact that actually exists." + }, + { + "key": "overview-benchmark-commit", + "label": "overview embeds the benchmark commit", + "status": "pass", + "expected": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "actual": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json" + ], + "detail": "Human-facing reports must point to the same benchmark release-lock commit." + }, + { + "key": "overview-benchmark-summary", + "label": "overview embeds benchmark summary fields", + "status": "pass", + "expected": { + "release_lock_ready": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3 + }, + "actual": { + "release_lock_ready": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3 + }, + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "overview-adoption-summary", + "label": "overview embeds adoption drift summary fields", + "status": "pass", + "expected": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "actual": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "paths": [ + "reports/adoption_drift_report.json", + "reports/skill-overview.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "overview-world-class-ledger-summary", + "label": "overview embeds world-class ledger summary fields", + "status": "pass", + "expected": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "actual": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-overview.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "overview-world-class-readiness", + "label": "overview derives readiness from the ledger", + "status": "pass", + "expected": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 19, + "source_pass_count": 11 + }, + "actual": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 19, + "source_pass_count": 11 + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-overview.json" + ], + "detail": "Readiness summaries must be derived from the evidence ledger, not hand-maintained copy." + }, + { + "key": "interpretation-benchmark-commit", + "label": "interpretation embeds the benchmark commit", + "status": "pass", + "expected": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "actual": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-interpretation.json" + ], + "detail": "Human-facing reports must point to the same benchmark release-lock commit." + }, + { + "key": "interpretation-benchmark-summary", + "label": "interpretation embeds benchmark summary fields", + "status": "pass", + "expected": { + "release_lock_ready": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3 + }, + "actual": { + "release_lock_ready": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3 + }, + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/skill-interpretation.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "interpretation-adoption-summary", + "label": "interpretation embeds adoption drift summary fields", + "status": "pass", + "expected": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "actual": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "adoption_rate": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "source_types": { + "manual": 1 + } + }, + "paths": [ + "reports/adoption_drift_report.json", + "reports/skill-interpretation.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "interpretation-world-class-ledger-summary", + "label": "interpretation embeds world-class ledger summary fields", + "status": "pass", + "expected": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "actual": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-interpretation.json" + ], + "detail": "Selected summary fields must match exactly across generated reports." + }, + { + "key": "interpretation-world-class-readiness", + "label": "interpretation derives readiness from the ledger", + "status": "pass", + "expected": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 19, + "source_pass_count": 11 + }, + "actual": { + "ready": false, + "decision": "evidence-pending", + "pending_count": 4, + "accepted_count": 0, + "source_check_count": 19, + "source_pass_count": 11 + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill-interpretation.json" + ], + "detail": "Readiness summaries must be derived from the evidence ledger, not hand-maintained copy." + }, + { + "key": "overview-interpretation-lockstep-scorecard", + "label": "Overview and interpretation share scorecard", + "status": "pass", + "expected": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 100, + "reasons": [ + "已生成 20 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 339 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 339 个词/字，references 约 16934 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "actual": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 100, + "reasons": [ + "已生成 20 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 339 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 339 个词/字，references 约 16934 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-capability-profile", + "label": "Overview and interpretation share capability_profile", + "status": "pass", + "expected": { + "archetype": "governed", + "task_family": "Execution operation", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "actual": { + "archetype": "governed", + "task_family": "Execution operation", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-principle-model", + "label": "Overview and interpretation share principle_model", + "status": "pass", + "expected": { + "nodes": [ + { + "title": "意图澄清", + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "actual": { + "nodes": [ + { + "title": "意图澄清", + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-contract-boundary", + "label": "Overview and interpretation share contract_boundary", + "status": "pass", + "expected": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "outputs": [ + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "actual": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "outputs": [ + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-quality-review", + "label": "Overview and interpretation share quality_review", + "status": "pass", + "expected": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" + ], + "gaps": [ + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." + ] + }, + "prompt_quality": { + "overall_quality_score": 89.0, + "highlights": [ + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." + ] + }, + "system_model": { + "stability": { + "score": 100, + "band": "system-ready" + }, + "highlights": [ + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." + ] + } + }, + "actual": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" + ], + "gaps": [ + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." + ] + }, + "prompt_quality": { + "overall_quality_score": 89.0, + "highlights": [ + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." + ] + }, + "system_model": { + "stability": { + "score": 100, + "band": "system-ready" + }, + "highlights": [ + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." + ] + } + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-risk-governance", + "label": "Overview and interpretation share risk_governance", + "status": "pass", + "expected": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 339 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "actual": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 339 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-world-class-readiness", + "label": "Overview and interpretation share world_class_readiness", + "status": "pass", + "expected": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 19, + "source_pass_count": 11, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete", + "Reviewer metadata" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, + "actual": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 19, + "source_pass_count": 11, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete", + "Reviewer metadata" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-package-assets", + "label": "Overview and interpretation share package_assets", + "status": "pass", + "expected": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 35 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 153 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 29 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 235 + } + ], + "file_count": 456, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 35 + }, + { + "label": "scripts", + "value": 153 + }, + { + "label": "evals", + "value": 29 + }, + { + "label": "reports", + "value": 235 + } + ] + }, + "actual": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 35 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 153 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 29 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 235 + } + ], + "file_count": 456, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 35 + }, + { + "label": "scripts", + "value": 153 + }, + { + "label": "evals", + "value": 29 + }, + { + "label": "reports", + "value": 235 + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-interpretation-lockstep-iteration-roadmap", + "label": "Overview and interpretation share iteration_roadmap", + "status": "pass", + "expected": { + "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + } + ] + }, + "actual": { + "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + } + ] + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-interpretation.json" + ], + "detail": "The first-class interpretation report must stay in lockstep with the canonical overview model." + }, + { + "key": "overview-html-contract", + "label": "overview has a stable HTML contract", + "status": "pass", + "expected": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-overview.html", + "html_exists": true + }, + "actual": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-overview.html", + "html_exists": true + }, + "paths": [ + "reports/skill-overview.json", + "reports/skill-overview.html" + ], + "detail": "Report output paths and language defaults are part of the user-facing contract." + }, + { + "key": "interpretation-html-contract", + "label": "interpretation has a stable HTML contract", + "status": "pass", + "expected": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-interpretation.html", + "html_exists": true + }, + "actual": { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": "reports/skill-interpretation.html", + "html_exists": true + }, + "paths": [ + "reports/skill-interpretation.json", + "reports/skill-interpretation.html" + ], + "detail": "Report output paths and language defaults are part of the user-facing contract." + }, + { + "key": "coverage-world-class-boundary", + "label": "Coverage report mirrors world-class evidence boundary", + "status": "pass", + "expected": { + "world_class_evidence_pending_count": 4, + "public_world_class_ready": false + }, + "actual": { + "world_class_evidence_pending_count": 4, + "public_world_class_ready": false + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/skill_os2_coverage.json" + ], + "detail": "Blueprint coverage can be locally complete while public world-class evidence remains pending." + }, + { + "key": "benchmark-world-class-boundary", + "label": "Benchmark report mirrors world-class evidence boundary", + "status": "pass", + "expected": { + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "public_claim_ready": false + }, + "actual": { + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "public_claim_ready": false + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/benchmark_reproducibility.json" + ], + "detail": "Benchmark reproducibility must not overstate public claim readiness." + }, + { + "key": "benchmark-beta-public-claim-split", + "label": "Benchmark separates beta testing from public claims", + "status": "pass", + "expected": { + "beta_test_ready": true, + "public_claim_ready": false, + "human_review_complete": false, + "beta_release_ready": true, + "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "deferred_evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "deferred_human_review": true + }, + "actual": { + "beta_test_ready": true, + "public_claim_ready": false, + "human_review_complete": false, + "beta_release_ready": true, + "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "deferred_evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "deferred_human_review": true + }, + "paths": [ + "reports/benchmark_reproducibility.json", + "reports/world_class_evidence_ledger.json" + ], + "detail": "Beta/public testing may proceed with human blind review deferred, but public world-class or superiority claims must remain blocked." + }, + { + "key": "preflight-world-class-boundary", + "label": "Preflight mirrors ledger without accepting evidence", + "status": "pass", + "expected": { + "pending_count": 4, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "preflight_counts_as_evidence": false, + "credential_value_exposed": false + }, + "actual": { + "pending_count": 4, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "preflight_counts_as_evidence": false, + "credential_value_exposed": false + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_preflight.json" + ], + "detail": "Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness." + }, + { + "key": "preflight-submission-kit-handoff", + "label": "Preflight exposes a safe submission-kit handoff", + "status": "pass", + "expected": { + "directory": "evidence/world_class/submissions", + "drafts_count_as_evidence": false, + "preflight_counts_submission_as_completion": false, + "html_report": "reports/world_class_evidence_preflight.html", + "html_exists": true, + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + "artifact_prefill_counts_as_evidence": false, + "artifact_role_source": "world-class-submission-kit", + "artifact_role_counts_as_evidence": false, + "artifact_role_prefill_counts_as_evidence": false, + "submission_ref_role_present": true, + "supporting_evidence_role_present": true, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false, + "submission_ref_total_present": true, + "supporting_evidence_total_present": true + }, + "actual": { + "directory": "evidence/world_class/submissions", + "drafts_count_as_evidence": false, + "preflight_counts_submission_as_completion": false, + "html_report": "reports/world_class_evidence_preflight.html", + "html_exists": true, + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + "artifact_prefill_counts_as_evidence": false, + "artifact_role_source": "world-class-submission-kit", + "artifact_role_counts_as_evidence": false, + "artifact_role_prefill_counts_as_evidence": false, + "submission_ref_role_present": true, + "supporting_evidence_role_present": true, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false, + "submission_ref_total_present": true, + "supporting_evidence_total_present": true + }, + "paths": [ + "reports/world_class_evidence_preflight.json", + "reports/world_class_evidence_preflight.html" + ], + "detail": "Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence." + }, + { + "key": "review-studio-preflight-artifact-role-handoff", + "label": "Review Studio mirrors preflight artifact roles", + "status": "pass", + "expected": { + "provider-holdout": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "human-adjudication": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-permission-enforcement": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-client-telemetry": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + } + }, + "actual": { + "provider-holdout": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "human-adjudication": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-permission-enforcement": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-client-telemetry": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + } + }, + "paths": [ + "reports/world_class_evidence_preflight.json", + "reports/review-studio.json" + ], + "detail": "The Review Studio world-class action card must carry the same submission-ref versus supporting-evidence contract as the preflight handoff." + }, + { + "key": "world-class-phase-queue-consistency", + "label": "World-class phase queues mirror repair rows", + "status": "pass", + "expected": { + "summary": { + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false + }, + "top_level_phase_queue": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "item_phase_queues": { + "provider-holdout": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "human-adjudication": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-permission-enforcement": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-client-telemetry": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ] + }, + "phase_queue_counts_as_completion": false, + "operator_runbook_summary": { + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false + }, + "operator_runbook_top_level_phase_queue": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "operator_runbook_phase_queues": { + "provider-holdout": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "human-adjudication": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-permission-enforcement": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-client-telemetry": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ] + }, + "review_studio_phase_queues": { + "provider-holdout": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "human-adjudication": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-permission-enforcement": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-client-telemetry": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ] + } + }, + "actual": { + "summary": { + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false + }, + "top_level_phase_queue": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "operator_runbook_summary": { + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false + }, + "operator_runbook_top_level_phase_queue": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "item_phase_queues": { + "provider-holdout": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "human-adjudication": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-permission-enforcement": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-client-telemetry": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ] + }, + "operator_runbook_phase_queues": { + "provider-holdout": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "human-adjudication": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-permission-enforcement": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-client-telemetry": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ] + }, + "review_studio_phase_queues": { + "provider-holdout": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "human-adjudication": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-permission-enforcement": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "native-client-telemetry": [ + { + "phase": "unblock-access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ] + }, + "phase_queue_counts_as_completion": false + }, + "paths": [ + "reports/world_class_evidence_preflight.json", + "reports/world_class_operator_runbook.json", + "reports/review-studio.json" + ], + "detail": "Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook and Review Studio without counting queue guidance as completion evidence." + }, + { + "key": "review-studio-no-overclaim", + "label": "Review Studio does not overclaim pending world-class evidence", + "status": "pass", + "expected": false, + "actual": false, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/review-studio.json" + ], + "detail": "When world-class evidence is pending, Review Studio must stay in a review or warning posture." + }, + { + "key": "claim-guard-package-runtime-surface", + "label": "Claim guard covers package and runtime claim surfaces", + "status": "pass", + "expected": { + "overclaim_guard_active": true, + "violation_count": 0, + "ledger_ready_to_claim_world_class": false, + "ledger_pending_count": 4, + "metadata_covers_json": true, + "package_surface_minimum": true, + "claim_surface_covers_package": true, + "required_surfaces": { + "README.md": true, + "SKILL.md": true, + "manifest.json": true, + "agents/interface.yaml": true, + "evidence/world_class/README.md": true, + "security/permission_policy.json": true, + "reports/world_class_evidence_ledger.json": true, + "dist/manifest.json": true, + "dist/targets/openai/adapter.json": true + }, + "prohibited_surfaces": [] + }, + "actual": { + "overclaim_guard_active": true, + "violation_count": 0, + "ledger_ready_to_claim_world_class": false, + "ledger_pending_count": 4, + "metadata_covers_json": true, + "package_surface_minimum": true, + "claim_surface_covers_package": true, + "required_surfaces": { + "README.md": true, + "SKILL.md": true, + "manifest.json": true, + "agents/interface.yaml": true, + "evidence/world_class/README.md": true, + "security/permission_policy.json": true, + "reports/world_class_evidence_ledger.json": true, + "dist/manifest.json": true, + "dist/targets/openai/adapter.json": true + }, + "prohibited_surfaces": [] + }, + "paths": [ + "reports/world_class_claim_guard.json", + "manifest.json", + "agents/interface.yaml", + "dist/manifest.json", + "dist/targets/openai/adapter.json", + "evidence/world_class/README.md", + "security/permission_policy.json", + "reports/world_class_evidence_ledger.json" + ], + "detail": "The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted." + }, + { + "key": "world-class-evidence-workflow-coverage", + "label": "World-class evidence workflows cover every pending ledger entry", + "status": "pass", + "expected": { + "keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "plan_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "submission_review_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_runbook_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_coordination_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_coordination_counts_as_completion": false, + "operator_release_gate_ready": false, + "operator_release_gate_counts_as_completion": false, + "review_studio_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_ready_to_claim_world_class": false, + "submission_review_ready_to_claim_world_class": false, + "submission_review_counts_as_completion": false, + "operator_runbook_ready_to_claim_world_class": false, + "operator_runbook_counts_as_completion": false, + "next_actions_present": { + "human-adjudication": true, + "native-client-telemetry": true, + "native-permission-enforcement": true, + "provider-holdout": true + }, + "commands": { + "human-adjudication": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-client-telemetry": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-permission-enforcement": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "provider-holdout": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + } + } + }, + "actual": { + "keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "plan_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "submission_review_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_runbook_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_coordination_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "operator_coordination_counts_as_completion": false, + "operator_release_gate_ready": false, + "operator_release_gate_counts_as_completion": false, + "review_studio_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "intake_ready_to_claim_world_class": false, + "submission_review_ready_to_claim_world_class": false, + "submission_review_counts_as_completion": false, + "operator_runbook_ready_to_claim_world_class": false, + "operator_runbook_counts_as_completion": false, + "next_actions_present": { + "human-adjudication": true, + "native-client-telemetry": true, + "native-permission-enforcement": true, + "provider-holdout": true + }, + "commands": { + "human-adjudication": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-client-telemetry": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "native-permission-enforcement": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + }, + "provider-holdout": { + "intake": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "operator_runbook": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + }, + "review_studio": { + "prepare_submission": true, + "validate_intake": true, + "submission_review": true, + "refresh_ledger": true, + "guard_claim": true + } + } + } + }, + "paths": [ + "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_plan.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_submission_review.json", + "reports/world_class_operator_runbook.json", + "reports/review-studio.json" + ], + "detail": "Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion." + }, + { + "key": "skill-os-2-review-current-evidence", + "label": "Skill OS 2.0 review summary mirrors current evidence", + "status": "pass", + "expected": [ + "score `91`", + "`16` gates", + "`3` warnings", + "`67` declared internal modules", + "`86 / 86` CLI help smoke checks passing across `153` scripts", + "`689` zip entries", + "archive with `689` entries", + "`12` installer permission checks enforced", + "`0` permission failures", + "`25` required artifacts", + "`23` reproduction commands", + "initial load `998/1000`", + "target count is `82`" + ], + "actual": "all present", + "paths": [ + "reports/skill-os-2-review.md", + "reports/review-studio.json", + "reports/package_verification.json", + "reports/install_simulation.json", + "reports/security_trust_report.json", + "reports/context_budget.json", + "reports/benchmark_reproducibility.json", + "scripts/ci_test.py" + ], + "detail": "Manual 2.0 review summaries must not drift from generated gate, package, trust, context, benchmark, or CI evidence." + } + ], + "artifacts": { + "json": "reports/evidence_consistency.json", + "markdown": "reports/evidence_consistency.md" + } +} diff --git a/reports/evidence_consistency.md b/reports/evidence_consistency.md new file mode 100644 index 00000000..013fa854 --- /dev/null +++ b/reports/evidence_consistency.md @@ -0,0 +1,57 @@ +# Evidence Consistency + +Generated at: `2026-06-17` + +## Summary + +- decision: `consistent` +- checks: `39` +- pass: `39` +- warn: `0` +- fail: `0` + +This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist. + +## Checks + +| Check | Status | Detail | Paths | +| --- | --- | --- | --- | +| Required report artifacts are readable | `pass` | The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/adoption_drift_report.json`, `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_evidence_preflight.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/skill_os2_coverage.json`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/world_class_claim_guard.json`, `reports/skill-os-2-review.md` | +| Release evidence flow covers first-class reports | `pass` | Release refresh and clean-lock instructions must regenerate every first-class report before evidence consistency can be trusted. | `AGENTS.md`, `reports/output_execution_runs.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/registry_audit.json`, `reports/package_verification.json`, `reports/upgrade_check.json`, `reports/adoption_drift_report.json`, `reports/architecture_maintainability.json`, `reports/python_compatibility.json`, `reports/runtime_permission_probes.json`, `reports/review_waivers.json`, `reports/review_annotations.json`, `reports/skill_atlas.json`, `reports/skill_os2_audit.json`, `reports/skill_os2_coverage.json`, `reports/context_budget.json`, `reports/context_budget_summary.json`, `reports/benchmark_reproducibility.json`, `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/review-viewer.json`, `reports/world_class_evidence_preflight.json`, `reports/skillops/daily`, `reports/skillops/weekly`, `reports/review-studio.json`, `reports/evidence_consistency.json` | +| Review Studio gates mirror review actions | `pass` | Every non-pass Review Studio gate must have exactly one top-level review action and a gate-local action summary with source refs and a verification command. | `reports/review-studio.json` | +| Review Studio mirrors context budget governance | `pass` | Review Studio must not keep stale context warnings after context reports prove large deferred resources are governed. | `reports/context_budget.json`, `reports/review-studio.json` | +| Benchmark release lock matches source dirty state | `pass` | The benchmark release lock must be blocked by source changes, while generated evidence artifacts are tracked as generation context. | `reports/benchmark_reproducibility.json` | +| Clean worktree keeps a clean benchmark release lock | `pass` | Dirty or non-git worktrees cannot prove final release-lock freshness, so this check is advisory until the final clean-lock pass. | `reports/benchmark_reproducibility.json` | +| Human-facing reports expose the canonical Skill IR artifact | `pass` | Skill IR is the 2.0 platform-neutral semantic source, so user-facing reports must link to the artifact that actually exists. | `reports/skill-overview.json`, `reports/skill-interpretation.json`, `reports/review-studio.json`, `skill-ir/examples/yao-meta-skill.json` | +| overview embeds the benchmark commit | `pass` | Human-facing reports must point to the same benchmark release-lock commit. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` | +| overview embeds benchmark summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/benchmark_reproducibility.json`, `reports/skill-overview.json` | +| overview embeds adoption drift summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/adoption_drift_report.json`, `reports/skill-overview.json` | +| overview embeds world-class ledger summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/world_class_evidence_ledger.json`, `reports/skill-overview.json` | +| overview derives readiness from the ledger | `pass` | Readiness summaries must be derived from the evidence ledger, not hand-maintained copy. | `reports/world_class_evidence_ledger.json`, `reports/skill-overview.json` | +| interpretation embeds the benchmark commit | `pass` | Human-facing reports must point to the same benchmark release-lock commit. | `reports/benchmark_reproducibility.json`, `reports/skill-interpretation.json` | +| interpretation embeds benchmark summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/benchmark_reproducibility.json`, `reports/skill-interpretation.json` | +| interpretation embeds adoption drift summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/adoption_drift_report.json`, `reports/skill-interpretation.json` | +| interpretation embeds world-class ledger summary fields | `pass` | Selected summary fields must match exactly across generated reports. | `reports/world_class_evidence_ledger.json`, `reports/skill-interpretation.json` | +| interpretation derives readiness from the ledger | `pass` | Readiness summaries must be derived from the evidence ledger, not hand-maintained copy. | `reports/world_class_evidence_ledger.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share scorecard | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share capability_profile | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share principle_model | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share contract_boundary | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share quality_review | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share risk_governance | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share world_class_readiness | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share package_assets | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| Overview and interpretation share iteration_roadmap | `pass` | The first-class interpretation report must stay in lockstep with the canonical overview model. | `reports/skill-overview.json`, `reports/skill-interpretation.json` | +| overview has a stable HTML contract | `pass` | Report output paths and language defaults are part of the user-facing contract. | `reports/skill-overview.json`, `reports/skill-overview.html` | +| interpretation has a stable HTML contract | `pass` | Report output paths and language defaults are part of the user-facing contract. | `reports/skill-interpretation.json`, `reports/skill-interpretation.html` | +| Coverage report mirrors world-class evidence boundary | `pass` | Blueprint coverage can be locally complete while public world-class evidence remains pending. | `reports/world_class_evidence_ledger.json`, `reports/skill_os2_coverage.json` | +| Benchmark report mirrors world-class evidence boundary | `pass` | Benchmark reproducibility must not overstate public claim readiness. | `reports/world_class_evidence_ledger.json`, `reports/benchmark_reproducibility.json` | +| Benchmark separates beta testing from public claims | `pass` | Beta/public testing may proceed with human blind review deferred, but public world-class or superiority claims must remain blocked. | `reports/benchmark_reproducibility.json`, `reports/world_class_evidence_ledger.json` | +| Preflight mirrors ledger without accepting evidence | `pass` | Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_preflight.json` | +| Preflight exposes a safe submission-kit handoff | `pass` | Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence. | `reports/world_class_evidence_preflight.json`, `reports/world_class_evidence_preflight.html` | +| Review Studio mirrors preflight artifact roles | `pass` | The Review Studio world-class action card must carry the same submission-ref versus supporting-evidence contract as the preflight handoff. | `reports/world_class_evidence_preflight.json`, `reports/review-studio.json` | +| World-class phase queues mirror repair rows | `pass` | Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook and Review Studio without counting queue guidance as completion evidence. | `reports/world_class_evidence_preflight.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` | +| Review Studio does not overclaim pending world-class evidence | `pass` | When world-class evidence is pending, Review Studio must stay in a review or warning posture. | `reports/world_class_evidence_ledger.json`, `reports/review-studio.json` | +| Claim guard covers package and runtime claim surfaces | `pass` | The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted. | `reports/world_class_claim_guard.json`, `manifest.json`, `agents/interface.yaml`, `dist/manifest.json`, `dist/targets/openai/adapter.json`, `evidence/world_class/README.md`, `security/permission_policy.json`, `reports/world_class_evidence_ledger.json` | +| World-class evidence workflows cover every pending ledger entry | `pass` | Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` | +| Skill OS 2.0 review summary mirrors current evidence | `pass` | Manual 2.0 review summaries must not drift from generated gate, package, trust, context, benchmark, or CI evidence. | `reports/skill-os-2-review.md`, `reports/review-studio.json`, `reports/package_verification.json`, `reports/install_simulation.json`, `reports/security_trust_report.json`, `reports/context_budget.json`, `reports/benchmark_reproducibility.json`, `scripts/ci_test.py` | diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 1d67799e..07155151 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -3,17 +3,21 @@ "schema_version": "2.0", "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 488, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, "interface_loaded": true, - "adapter_count": 3, + "adapter_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, "install_root_is_temp": false, "failure_count": 0, "warning_count": 0 @@ -22,7 +26,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -108,14 +112,169 @@ "id": "adapter-openai-name", "status": "pass", "detail": "openai adapter name matches package manifest" + }, + { + "id": "adapter-vscode", + "status": "pass", + "detail": "vscode adapter is readable after package install simulation" + }, + { + "id": "adapter-vscode-name", + "status": "pass", + "detail": "vscode adapter name matches package manifest" + }, + { + "id": "permission-policy-load", + "status": "pass", + "detail": "Installed permission policy is readable" + }, + { + "id": "permission-claude-contract", + "status": "pass", + "detail": "claude adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-claude-file_write-approved", + "status": "pass", + "detail": "claude capability file_write has active reviewer approval" + }, + { + "id": "permission-claude-file_write-target-enforcement", + "status": "pass", + "detail": "claude capability file_write has target enforcement note" + }, + { + "id": "permission-claude-network-approved", + "status": "pass", + "detail": "claude capability network has active reviewer approval" + }, + { + "id": "permission-claude-network-target-enforcement", + "status": "pass", + "detail": "claude capability network has target enforcement note" + }, + { + "id": "permission-claude-subprocess-approved", + "status": "pass", + "detail": "claude capability subprocess has active reviewer approval" + }, + { + "id": "permission-claude-subprocess-target-enforcement", + "status": "pass", + "detail": "claude capability subprocess has target enforcement note" + }, + { + "id": "permission-generic-contract", + "status": "pass", + "detail": "generic adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-generic-file_write-approved", + "status": "pass", + "detail": "generic capability file_write has active reviewer approval" + }, + { + "id": "permission-generic-file_write-target-enforcement", + "status": "pass", + "detail": "generic capability file_write has target enforcement note" + }, + { + "id": "permission-generic-network-approved", + "status": "pass", + "detail": "generic capability network has active reviewer approval" + }, + { + "id": "permission-generic-network-target-enforcement", + "status": "pass", + "detail": "generic capability network has target enforcement note" + }, + { + "id": "permission-generic-subprocess-approved", + "status": "pass", + "detail": "generic capability subprocess has active reviewer approval" + }, + { + "id": "permission-generic-subprocess-target-enforcement", + "status": "pass", + "detail": "generic capability subprocess has target enforcement note" + }, + { + "id": "permission-openai-contract", + "status": "pass", + "detail": "openai adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-openai-file_write-approved", + "status": "pass", + "detail": "openai capability file_write has active reviewer approval" + }, + { + "id": "permission-openai-file_write-target-enforcement", + "status": "pass", + "detail": "openai capability file_write has target enforcement note" + }, + { + "id": "permission-openai-network-approved", + "status": "pass", + "detail": "openai capability network has active reviewer approval" + }, + { + "id": "permission-openai-network-target-enforcement", + "status": "pass", + "detail": "openai capability network has target enforcement note" + }, + { + "id": "permission-openai-subprocess-approved", + "status": "pass", + "detail": "openai capability subprocess has active reviewer approval" + }, + { + "id": "permission-openai-subprocess-target-enforcement", + "status": "pass", + "detail": "openai capability subprocess has target enforcement note" + }, + { + "id": "permission-vscode-contract", + "status": "pass", + "detail": "vscode adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-vscode-file_write-approved", + "status": "pass", + "detail": "vscode capability file_write has active reviewer approval" + }, + { + "id": "permission-vscode-file_write-target-enforcement", + "status": "pass", + "detail": "vscode capability file_write has target enforcement note" + }, + { + "id": "permission-vscode-network-approved", + "status": "pass", + "detail": "vscode capability network has active reviewer approval" + }, + { + "id": "permission-vscode-network-target-enforcement", + "status": "pass", + "detail": "vscode capability network has target enforcement note" + }, + { + "id": "permission-vscode-subprocess-approved", + "status": "pass", + "detail": "vscode capability subprocess has active reviewer approval" + }, + { + "id": "permission-vscode-subprocess-target-enforcement", + "status": "pass", + "detail": "vscode capability subprocess has target enforcement note" } ], "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } } diff --git a/reports/install_simulation.md b/reports/install_simulation.md index 7a06607a..b0d16dae 100644 --- a/reports/install_simulation.md +++ b/reports/install_simulation.md @@ -1,12 +1,14 @@ # Install Simulation - OK: `True` -- Package directory: `dist` +- Package directory: `tests/tmp_review_studio/dist` - Archive extracted: `True` - Entrypoint loaded: `True` - Manifest loaded: `True` - Interface loaded: `True` -- Adapters readable: `3` +- Adapters readable: `4` +- Installer permissions enforced: `12` +- Installer permission failures: `0` - Failures: `0` - Warnings: `0` @@ -14,7 +16,7 @@ | Check | Status | Detail | | --- | --- | --- | -| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip | +| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip | | `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | | `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill | | `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable | @@ -32,6 +34,37 @@ | `adapter-generic-name` | `pass` | generic adapter name matches package manifest | | `adapter-openai` | `pass` | openai adapter is readable after package install simulation | | `adapter-openai-name` | `pass` | openai adapter name matches package manifest | +| `adapter-vscode` | `pass` | vscode adapter is readable after package install simulation | +| `adapter-vscode-name` | `pass` | vscode adapter name matches package manifest | +| `permission-policy-load` | `pass` | Installed permission policy is readable | +| `permission-claude-contract` | `pass` | claude adapter exposes target permission contract for installer enforcement | +| `permission-claude-file_write-approved` | `pass` | claude capability file_write has active reviewer approval | +| `permission-claude-file_write-target-enforcement` | `pass` | claude capability file_write has target enforcement note | +| `permission-claude-network-approved` | `pass` | claude capability network has active reviewer approval | +| `permission-claude-network-target-enforcement` | `pass` | claude capability network has target enforcement note | +| `permission-claude-subprocess-approved` | `pass` | claude capability subprocess has active reviewer approval | +| `permission-claude-subprocess-target-enforcement` | `pass` | claude capability subprocess has target enforcement note | +| `permission-generic-contract` | `pass` | generic adapter exposes target permission contract for installer enforcement | +| `permission-generic-file_write-approved` | `pass` | generic capability file_write has active reviewer approval | +| `permission-generic-file_write-target-enforcement` | `pass` | generic capability file_write has target enforcement note | +| `permission-generic-network-approved` | `pass` | generic capability network has active reviewer approval | +| `permission-generic-network-target-enforcement` | `pass` | generic capability network has target enforcement note | +| `permission-generic-subprocess-approved` | `pass` | generic capability subprocess has active reviewer approval | +| `permission-generic-subprocess-target-enforcement` | `pass` | generic capability subprocess has target enforcement note | +| `permission-openai-contract` | `pass` | openai adapter exposes target permission contract for installer enforcement | +| `permission-openai-file_write-approved` | `pass` | openai capability file_write has active reviewer approval | +| `permission-openai-file_write-target-enforcement` | `pass` | openai capability file_write has target enforcement note | +| `permission-openai-network-approved` | `pass` | openai capability network has active reviewer approval | +| `permission-openai-network-target-enforcement` | `pass` | openai capability network has target enforcement note | +| `permission-openai-subprocess-approved` | `pass` | openai capability subprocess has active reviewer approval | +| `permission-openai-subprocess-target-enforcement` | `pass` | openai capability subprocess has target enforcement note | +| `permission-vscode-contract` | `pass` | vscode adapter exposes target permission contract for installer enforcement | +| `permission-vscode-file_write-approved` | `pass` | vscode capability file_write has active reviewer approval | +| `permission-vscode-file_write-target-enforcement` | `pass` | vscode capability file_write has target enforcement note | +| `permission-vscode-network-approved` | `pass` | vscode capability network has active reviewer approval | +| `permission-vscode-network-target-enforcement` | `pass` | vscode capability network has target enforcement note | +| `permission-vscode-subprocess-approved` | `pass` | vscode capability subprocess has active reviewer approval | +| `permission-vscode-subprocess-target-enforcement` | `pass` | vscode capability subprocess has target enforcement note | ## Failures diff --git a/reports/iteration_bundles/governed-incident-command/bundle.json b/reports/iteration_bundles/governed-incident-command/bundle.json index cab8ebee..98501ad8 100644 --- a/reports/iteration_bundles/governed-incident-command/bundle.json +++ b/reports/iteration_bundles/governed-incident-command/bundle.json @@ -7,7 +7,8 @@ "cause_tags": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "gate_status": { "visible_holdout_non_regression": true, @@ -28,8 +29,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -39,25 +40,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -66,7 +52,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -80,7 +66,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -94,9 +80,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } diff --git a/reports/iteration_bundles/governed-incident-command/bundle.md b/reports/iteration_bundles/governed-incident-command/bundle.md index bcb085aa..4ea7fd31 100644 --- a/reports/iteration_bundles/governed-incident-command/bundle.md +++ b/reports/iteration_bundles/governed-incident-command/bundle.md @@ -10,6 +10,7 @@ - `no_candidate_outperformed_current` - `current_holdout_gap_present` - `current_holdout_risk` +- `adversarial_watch_risk` ## Gate Status @@ -28,12 +29,12 @@ | Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors | | --- | --- | --- | --- | ---: | ---: | ---: | -| baseline | `Baseline` | reference | reference | 93 | 1 | 2 | -| current | `Current` | selected_by_dev | kept_current | 37 | 1 | 1 | -| candidate | `Guardrail` | not_selected | blocked | 51 | 1 | 2 | -| candidate | `Balanced` | not_selected | blocked | 54 | 1 | 1 | -| candidate | `Boundary` | not_selected | blocked | 78 | 1 | 1 | -| candidate | `Artifact Aware` | not_selected | blocked | 78 | 1 | 1 | +| baseline | `Baseline` | reference | reference | 93 | 0 | 1 | +| current | `Current` | selected_by_dev | kept_current | 37 | 0 | 1 | +| candidate | `Balanced` | not_selected | blocked | 54 | 0 | 1 | +| candidate | `Boundary` | not_selected | blocked | 78 | 0 | 1 | +| candidate | `Artifact Aware` | not_selected | blocked | 78 | 0 | 1 | +| candidate | `Guardrail` | not_selected | blocked | 51 | 1 | 1 | ## Human Review Stub diff --git a/reports/iteration_bundles/yao-meta-skill/bundle.json b/reports/iteration_bundles/yao-meta-skill/bundle.json index 47d9cd6a..c071b070 100644 --- a/reports/iteration_bundles/yao-meta-skill/bundle.json +++ b/reports/iteration_bundles/yao-meta-skill/bundle.json @@ -35,8 +35,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -66,7 +66,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -80,7 +81,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -116,8 +118,8 @@ ], "review_template": { "target": "yao-meta-skill", - "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "candidate_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "candidate_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "focus": [ "no_candidate_outperformed_current" ] diff --git a/reports/iteration_bundles/yao-meta-skill/bundle.md b/reports/iteration_bundles/yao-meta-skill/bundle.md index 0abafe75..b207ee61 100644 --- a/reports/iteration_bundles/yao-meta-skill/bundle.md +++ b/reports/iteration_bundles/yao-meta-skill/bundle.md @@ -27,7 +27,7 @@ | Role | Label | Ranking State | Promotion State | Tokens | Dev Errors | Holdout Errors | | --- | --- | --- | --- | ---: | ---: | ---: | | baseline | `Baseline` | reference | reference | 8 | 1 | 0 | -| current | `Current` | selected_by_dev | kept_current | 65 | 0 | 0 | +| current | `Current` | selected_by_dev | kept_current | 53 | 0 | 0 | | candidate | `Minimal` | not_selected | blocked | 41 | 2 | 0 | | candidate | `Guardrail` | not_selected | blocked | 56 | 2 | 0 | | candidate | `Balanced` | not_selected | blocked | 60 | 2 | 0 | @@ -37,8 +37,8 @@ ## Human Review Stub - target: yao-meta-skill -- current description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. -- candidate description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +- current description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. +- candidate description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. - review focus: no_candidate_outperformed_current ## Artifact Paths diff --git a/reports/iteration_bundles/yao-meta-skill/review.md b/reports/iteration_bundles/yao-meta-skill/review.md index 7559bd45..69c18c9d 100644 --- a/reports/iteration_bundles/yao-meta-skill/review.md +++ b/reports/iteration_bundles/yao-meta-skill/review.md @@ -1,8 +1,8 @@ # Human Review - target: yao-meta-skill -- current description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. -- candidate description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +- current description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. +- candidate description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. - suggested focus: no_candidate_outperformed_current Use the shared template in `references/human-review-template.md` to complete the final decision. diff --git a/reports/iteration_ledger.md b/reports/iteration_ledger.md index 0d875ca4..acf95708 100644 --- a/reports/iteration_ledger.md +++ b/reports/iteration_ledger.md @@ -33,7 +33,7 @@ | --- | --- | --- | --- | --- | | `yao-meta-skill` | `keep_current` | `Current` | no_candidate_outperformed_current | Keep the current description and open a new candidate only when fresh route evidence appears. | | `team-frontend-review` | `keep_current` | `Current` | no_candidate_outperformed_current, current_holdout_risk | Keep the current description and open a new candidate only when fresh route evidence appears. | -| `governed-incident-command` | `keep_current` | `Current` | no_candidate_outperformed_current, current_holdout_gap_present, current_holdout_risk | Keep the current description and open a new candidate only when fresh route evidence appears. | +| `governed-incident-command` | `keep_current` | `Current` | no_candidate_outperformed_current, current_holdout_gap_present, current_holdout_risk, adversarial_watch_risk | Keep the current description and open a new candidate only when fresh route evidence appears. | ## Current Route Scorecard @@ -43,7 +43,7 @@ | Route | Precision | Recall | Avg Margin | | --- | ---: | ---: | ---: | -| `yao-meta-skill` | 1.0 | 1.0 | 0.611 | +| `yao-meta-skill` | 1.0 | 1.0 | 0.701 | | `team-frontend-review` | 1.0 | 1.0 | 0.805 | -| `governed-incident-command` | 1.0 | 1.0 | 0.659 | +| `governed-incident-command` | 1.0 | 1.0 | 0.586 | | `no_route` | 1.0 | 1.0 | - | diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 6cb2055f..4ce4e7cf 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -6,22 +6,32 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "summary": { "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -31,17 +41,17 @@ "case_id": "skill-package-contract", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.11, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 5980.01, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 16, - "output_tokens": 17, - "total_tokens": 33, - "estimated": true + "input_tokens": 100, + "output_tokens": 384, + "total_tokens": 484, + "estimated": false }, "score": 0.0, "passed_count": 0, @@ -52,47 +62,49 @@ "has-report", "has-resource-boundary" ], - "output_sha256": "ece26a6edbbbea86a93faaf9f0eadf88efbae12f1c3ad81da9376cf2a889d101", + "output_sha256": "1aa12d65d55863581eef67aaa173b3a9962524a91533693044cb340a342b760b", "failure": "" }, { "case_id": "skill-package-contract", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.56, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 11217.19, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 16, - "output_tokens": 57, - "total_tokens": 73, - "estimated": true + "input_tokens": 850, + "output_tokens": 998, + "total_tokens": 1848, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "8bee0131e17fdbe87e48457130ac6aeaa55ea30c919a45e77109852609f33eba", + "score": 75.0, + "passed_count": 3, + "failed_count": 1, + "failed_assertions": [ + "has-report" + ], + "output_sha256": "ac2cb822520b3099150ea1df07de59b9c6df382b7251792e0662631a60989d89", "failure": "" }, { "case_id": "output-eval-expectation", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 24.17, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 3178.95, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 24, - "output_tokens": 12, - "total_tokens": 36, - "estimated": true + "input_tokens": 107, + "output_tokens": 129, + "total_tokens": 236, + "estimated": false }, "score": 0.0, "passed_count": 0, @@ -103,183 +115,192 @@ "has-scorecard", "has-failure-taxonomy" ], - "output_sha256": "98fa332fb493b5dc14656a9d65a6c0a9c25ff61a14b71b647aad89cbdbd3ab35", + "output_sha256": "a128ff4dbd7870e287ac651b6cab6b7887e347d6477e5bd8c6f40cd9479539f0", "failure": "" }, { "case_id": "output-eval-expectation", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.84, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4611.78, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 24, - "output_tokens": 56, - "total_tokens": 80, - "estimated": true + "input_tokens": 857, + "output_tokens": 264, + "total_tokens": 1121, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "8efdbfefa6a36603682b7063b1753f9f939b3810c082d29686e297906d9f51c8", + "score": 25.0, + "passed_count": 1, + "failed_count": 3, + "failed_assertions": [ + "has-baseline-comparison", + "has-assertion-grading", + "has-failure-taxonomy" + ], + "output_sha256": "4df98d20e2199e56dae2618b6a64e59d5c96484a4f71ae0ebc674b755cf533c3", "failure": "" }, { "case_id": "ir-before-packaging", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 22.51, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 6772.59, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 18, - "output_tokens": 15, - "total_tokens": 33, - "estimated": true + "input_tokens": 104, + "output_tokens": 661, + "total_tokens": 765, + "estimated": false }, - "score": 0.0, - "passed_count": 0, - "failed_count": 4, + "score": 25.0, + "passed_count": 1, + "failed_count": 3, "failed_assertions": [ "has-ir-first", "has-core-ir-fields", - "has-targets", "has-semantic-preservation" ], - "output_sha256": "07a4aa9ee9e48341d4b84d5cb9b6414901aa2c8841340d67447e187240c6c4c2", + "output_sha256": "1696075b4f884ff8b478786b5565aff73ad73235936eb39c2ed615c0746861d2", "failure": "" }, { "case_id": "ir-before-packaging", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 26.03, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 17471.19, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 18, - "output_tokens": 62, - "total_tokens": 80, - "estimated": true + "input_tokens": 854, + "output_tokens": 2132, + "total_tokens": 2986, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "707fde419b3d61e921561b1c4a8523e1f7b53b1acd5758a47ce21d9ed711fde0", - "failure": "" - }, - { - "case_id": "near-neighbor-boundary", - "variant": "baseline", - "status": "pass", - "execution_mode": "command", - "model_executed": false, - "command_executed": true, - "duration_ms": 28.48, - "provider": "local-output-eval-runner", - "model": "", - "usage": { - "input_tokens": 17, - "output_tokens": 19, - "total_tokens": 36, - "estimated": true - }, - "score": 0.0, - "passed_count": 0, + "score": 25.0, + "passed_count": 1, "failed_count": 3, + "failed_assertions": [ + "has-ir-first", + "has-core-ir-fields", + "has-semantic-preservation" + ], + "output_sha256": "4f87633e53f360948d6403126d49ad3dabcb1b47aff596918650719b7840abcf", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "baseline", + "status": "pass", + "execution_mode": "model", + "model_executed": true, + "command_executed": true, + "duration_ms": 3091.33, + "provider": "deepseek", + "model": "deepseek-v4-flash", + "usage": { + "input_tokens": 105, + "output_tokens": 93, + "total_tokens": 198, + "estimated": false + }, + "score": 33.33, + "passed_count": 1, + "failed_count": 2, "failed_assertions": [ "declines-skill", - "names-near-neighbor", "requires-reuse-signal" ], - "output_sha256": "648ebaa6e09486d05f33fa6639dc973b029d115158588fd192efbe15ccaaeac4", + "output_sha256": "56c8f48f3b2b9963adad3afdb1b6a2bed6b162067506edee1e0f1acd32668a91", "failure": "" }, { "case_id": "near-neighbor-boundary", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.17, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4152.85, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 48, - "total_tokens": 65, - "estimated": true + "input_tokens": 855, + "output_tokens": 160, + "total_tokens": 1015, + "estimated": false }, - "score": 100.0, - "passed_count": 3, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "17a3a5286ad573dea86e51f82c04b596babaecb92cde8879f09e7c519205e2e1", + "score": 33.33, + "passed_count": 1, + "failed_count": 2, + "failed_assertions": [ + "declines-skill", + "requires-reuse-signal" + ], + "output_sha256": "d2d85a2aae5db5c4c7e6e0db5bdd865157a90018afe302b0a703940cc84ba68c", "failure": "" }, { "case_id": "file-backed-governed-package", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.94, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4750.01, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 20, - "total_tokens": 37, - "estimated": true + "input_tokens": 200, + "output_tokens": 302, + "total_tokens": 502, + "estimated": false }, - "score": 0.0, - "passed_count": 0, - "failed_count": 5, + "score": 60.0, + "passed_count": 3, + "failed_count": 2, "failed_assertions": [ "uses-file-backed-evidence", - "has-governance", - "has-output-contract", - "has-trust-and-scorecard", - "does-not-invent-evidence" + "has-trust-and-scorecard" ], - "output_sha256": "6e5827219459dee34143e4f11663415db80c3111291f75fb6c2115c7bda4cf30", + "output_sha256": "d41880b0c2f7e60ee6acb2cb1f4acb9377fd63de72b8e026e6259a962b89af0b", "failure": "" }, { "case_id": "file-backed-governed-package", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 22.69, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 9406.25, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 81, - "total_tokens": 98, - "estimated": true + "input_tokens": 950, + "output_tokens": 660, + "total_tokens": 1610, + "estimated": false }, - "score": 100.0, - "passed_count": 5, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "bcfdf036666d406abbc945fc42b93eb74d7074118893e540253c7d8715baad73", + "score": 60.0, + "passed_count": 3, + "failed_count": 2, + "failed_assertions": [ + "has-governance", + "has-output-contract" + ], + "output_sha256": "c670ae2ef81a819a9a0f1b0c046e82ba4c649f56dcbe96a45a6938c7f214fba4", "failure": "" } ], diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index d2e3be45..576e77c2 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -5,37 +5,33 @@ This report records how output-eval variants were produced and whether timing or - Cases: `5` - Variant runs: `10` - Command executed: `10` -- Model executed: `0` +- Model executed: `10` - Recorded fixtures: `0` - Timing observed: `10` -- Token observed: `0` -- Token estimated: `10` -- Delta: `100.0` +- Token observed: `10` +- Token estimated: `0` +- Delta: `20.0` - Gate pass: `True` -No model-executed runs are recorded yet. - -Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence. - Command runner evidence is present. This proves the eval harness executed an external command, but it is not provider-backed model evidence unless the runner reports model metadata. ## Runs | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 23.11 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 23.56 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 24.17 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 23.84 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 22.51 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.03 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.48 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 23.17 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 23.94 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 22.69 | 98 | 100.0 | pass | +| skill-package-contract | baseline | model | deepseek-v4-flash | 5980.01 | 484 | 0.0 | pass | +| skill-package-contract | with_skill | model | deepseek-v4-flash | 11217.19 | 1848 | 75.0 | pass | +| output-eval-expectation | baseline | model | deepseek-v4-flash | 3178.95 | 236 | 0.0 | pass | +| output-eval-expectation | with_skill | model | deepseek-v4-flash | 4611.78 | 1121 | 25.0 | pass | +| ir-before-packaging | baseline | model | deepseek-v4-flash | 6772.59 | 765 | 25.0 | pass | +| ir-before-packaging | with_skill | model | deepseek-v4-flash | 17471.19 | 2986 | 25.0 | pass | +| near-neighbor-boundary | baseline | model | deepseek-v4-flash | 3091.33 | 198 | 33.33 | pass | +| near-neighbor-boundary | with_skill | model | deepseek-v4-flash | 4152.85 | 1015 | 33.33 | pass | +| file-backed-governed-package | baseline | model | deepseek-v4-flash | 4750.01 | 502 | 60.0 | pass | +| file-backed-governed-package | with_skill | model | deepseek-v4-flash | 9406.25 | 1610 | 60.0 | pass | ## Next Fixes - Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence. -- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior. +- Use `scripts/provider_output_eval_runner.py` for provider-backed holdout cases when release confidence depends on real generation behavior. - Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse. diff --git a/reports/output_review_adjudication.json b/reports/output_review_adjudication.json index 0d58d328..c8ebdbe5 100644 --- a/reports/output_review_adjudication.json +++ b/reports/output_review_adjudication.json @@ -8,12 +8,50 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, - "failure_count": 0 + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "failure_count": 0, + "reviewer_checklist_count": 5, + "reviewer_checklist_pending_count": 5, + "reviewer_checklist_invalid_count": 0, + "reviewer_checklist_ready_count": 0 }, "reviewer": "", "reviewed_at": "", + "review_integrity": { + "blind_pack_sha256": "788604c67fa31178c9bba06a52731adcbee9c1e72c00767cf7950643812a6e5c", + "case_count": 5, + "case_ids": [ + "skill-package-contract", + "output-eval-expectation", + "ir-before-packaging", + "near-neighbor-boundary", + "file-backed-governed-package" + ], + "prompt_sha256_by_case": { + "skill-package-contract": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5", + "output-eval-expectation": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4", + "ir-before-packaging": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80", + "near-neighbor-boundary": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0", + "file-backed-governed-package": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d" + } + }, + "reviewer_attestation": { + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded": true, + "reviewer_reason_required": true + }, "artifacts": { "blind_pack": "reports/output_blind_review_pack.json", "answer_key": "reports/output_blind_answer_key.json", @@ -26,47 +64,214 @@ { "case_id": "skill-package-contract", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Turn this repeated workflow into a reusable team skill package." + "prompt_sha256": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5" }, { "case_id": "output-eval-expectation", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline." + "prompt_sha256": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4" }, { "case_id": "ir-before-packaging", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets." + "prompt_sha256": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80" }, { "case_id": "near-neighbor-boundary", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "I only need a one-off summary of these notes, not a reusable process." + "prompt_sha256": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0" }, { "case_id": "file-backed-governed-package", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Turn the attached release brief source into a governed skill package." + "prompt_sha256": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d" + } + ], + "reviewer_checklist": [ + { + "case_id": "skill-package-contract", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "output-eval-expectation", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "ir-before-packaging", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "near-neighbor-boundary", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "file-backed-governed-package", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] } ], "failures": [] diff --git a/reports/output_review_adjudication.md b/reports/output_review_adjudication.md index 6c12de42..9b757c8d 100644 --- a/reports/output_review_adjudication.md +++ b/reports/output_review_adjudication.md @@ -7,20 +7,198 @@ This report adjudicates reviewer choices from the blind A/B output review pack a - Pending: `5` - Agreement rate: `n/a` - Invalid decisions: `0` +- Answer keys revealed: `0` +- Pending/invalid answers hidden: `5` +- Reviewer checklist: `0` ready / `5` total +- Reviewer metadata present: `false` +- Blind review attested: `false` +- Raw content excluded: `true` +- Ready for human evidence: `false` No reviewer decisions recorded yet. Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication. +Expected winners stay hidden until a valid reviewer decision is recorded. ## Case Adjudication | Case | Reviewer | Expected | Status | Confidence | Reason | | --- | --- | --- | --- | ---: | --- | -| skill-package-contract | pending | A | pending | | | -| output-eval-expectation | pending | A | pending | | | -| ir-before-packaging | pending | B | pending | | | -| near-neighbor-boundary | pending | B | pending | | | -| file-backed-governed-package | pending | B | pending | | | +| skill-package-contract | pending | hidden | pending | | | +| output-eval-expectation | pending | hidden | pending | | | +| ir-before-packaging | pending | hidden | pending | | | +| near-neighbor-boundary | pending | hidden | pending | | | +| file-backed-governed-package | pending | hidden | pending | | | + +## Reviewer Checklist + +| Case | Readiness | Answer key | Decision file | +| --- | --- | --- | --- | +| `skill-package-contract` | `awaiting-decision` | `hidden` | `reports/output_review_decisions.json` | +| `output-eval-expectation` | `awaiting-decision` | `hidden` | `reports/output_review_decisions.json` | +| `ir-before-packaging` | `awaiting-decision` | `hidden` | `reports/output_review_decisions.json` | +| `near-neighbor-boundary` | `awaiting-decision` | `hidden` | `reports/output_review_decisions.json` | +| `file-backed-governed-package` | `awaiting-decision` | `hidden` | `reports/output_review_decisions.json` | + +### skill-package-contract + +- readiness: `awaiting-decision` +- blocking reason: Reviewer has not selected A or B yet; answer key remains hidden. +- answer key visible: `false` +- blind pack: `reports/output_blind_review_pack.json` +- decisions: `reports/output_review_decisions.json` + +#### Commands + +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` +- write_template: `python3 scripts/adjudicate_output_review.py --write-template` +- import_decisions: `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- adjudicate: `python3 scripts/yao.py output-review` +- refresh_review_studio: `python3 scripts/yao.py review-studio .` + +#### Required Fields + +- winner_variant: A or B after reading only the blind review pack. +- confidence: Optional number from 0 to 1. +- reason: Required rationale; do not reveal baseline or with-skill labels before adjudication. +- reviewer: Human reviewer name or review group at the decision-file top level. +- reviewed_at: Review date or timestamp at the decision-file top level. +- reviewer_attestation.blind_review_completed_before_answer_key: True only after the reviewer has completed choices before opening the answer key. +- reviewer_attestation.answer_key_not_opened_before_decisions: True only when the answer key was not opened before decisions were recorded. + +#### Privacy Contract + +- Do not paste raw private user data into the decision reason. +- Do not open the answer key before reviewer choices are recorded. +- Leave winner_variant blank when the reviewer is not ready to decide. + +### output-eval-expectation + +- readiness: `awaiting-decision` +- blocking reason: Reviewer has not selected A or B yet; answer key remains hidden. +- answer key visible: `false` +- blind pack: `reports/output_blind_review_pack.json` +- decisions: `reports/output_review_decisions.json` + +#### Commands + +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` +- write_template: `python3 scripts/adjudicate_output_review.py --write-template` +- import_decisions: `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- adjudicate: `python3 scripts/yao.py output-review` +- refresh_review_studio: `python3 scripts/yao.py review-studio .` + +#### Required Fields + +- winner_variant: A or B after reading only the blind review pack. +- confidence: Optional number from 0 to 1. +- reason: Required rationale; do not reveal baseline or with-skill labels before adjudication. +- reviewer: Human reviewer name or review group at the decision-file top level. +- reviewed_at: Review date or timestamp at the decision-file top level. +- reviewer_attestation.blind_review_completed_before_answer_key: True only after the reviewer has completed choices before opening the answer key. +- reviewer_attestation.answer_key_not_opened_before_decisions: True only when the answer key was not opened before decisions were recorded. + +#### Privacy Contract + +- Do not paste raw private user data into the decision reason. +- Do not open the answer key before reviewer choices are recorded. +- Leave winner_variant blank when the reviewer is not ready to decide. + +### ir-before-packaging + +- readiness: `awaiting-decision` +- blocking reason: Reviewer has not selected A or B yet; answer key remains hidden. +- answer key visible: `false` +- blind pack: `reports/output_blind_review_pack.json` +- decisions: `reports/output_review_decisions.json` + +#### Commands + +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` +- write_template: `python3 scripts/adjudicate_output_review.py --write-template` +- import_decisions: `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- adjudicate: `python3 scripts/yao.py output-review` +- refresh_review_studio: `python3 scripts/yao.py review-studio .` + +#### Required Fields + +- winner_variant: A or B after reading only the blind review pack. +- confidence: Optional number from 0 to 1. +- reason: Required rationale; do not reveal baseline or with-skill labels before adjudication. +- reviewer: Human reviewer name or review group at the decision-file top level. +- reviewed_at: Review date or timestamp at the decision-file top level. +- reviewer_attestation.blind_review_completed_before_answer_key: True only after the reviewer has completed choices before opening the answer key. +- reviewer_attestation.answer_key_not_opened_before_decisions: True only when the answer key was not opened before decisions were recorded. + +#### Privacy Contract + +- Do not paste raw private user data into the decision reason. +- Do not open the answer key before reviewer choices are recorded. +- Leave winner_variant blank when the reviewer is not ready to decide. + +### near-neighbor-boundary + +- readiness: `awaiting-decision` +- blocking reason: Reviewer has not selected A or B yet; answer key remains hidden. +- answer key visible: `false` +- blind pack: `reports/output_blind_review_pack.json` +- decisions: `reports/output_review_decisions.json` + +#### Commands + +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` +- write_template: `python3 scripts/adjudicate_output_review.py --write-template` +- import_decisions: `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- adjudicate: `python3 scripts/yao.py output-review` +- refresh_review_studio: `python3 scripts/yao.py review-studio .` + +#### Required Fields + +- winner_variant: A or B after reading only the blind review pack. +- confidence: Optional number from 0 to 1. +- reason: Required rationale; do not reveal baseline or with-skill labels before adjudication. +- reviewer: Human reviewer name or review group at the decision-file top level. +- reviewed_at: Review date or timestamp at the decision-file top level. +- reviewer_attestation.blind_review_completed_before_answer_key: True only after the reviewer has completed choices before opening the answer key. +- reviewer_attestation.answer_key_not_opened_before_decisions: True only when the answer key was not opened before decisions were recorded. + +#### Privacy Contract + +- Do not paste raw private user data into the decision reason. +- Do not open the answer key before reviewer choices are recorded. +- Leave winner_variant blank when the reviewer is not ready to decide. + +### file-backed-governed-package + +- readiness: `awaiting-decision` +- blocking reason: Reviewer has not selected A or B yet; answer key remains hidden. +- answer key visible: `false` +- blind pack: `reports/output_blind_review_pack.json` +- decisions: `reports/output_review_decisions.json` + +#### Commands + +- prepare_review_kit: `python3 scripts/yao.py output-review-kit` +- write_template: `python3 scripts/adjudicate_output_review.py --write-template` +- import_decisions: `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- adjudicate: `python3 scripts/yao.py output-review` +- refresh_review_studio: `python3 scripts/yao.py review-studio .` + +#### Required Fields + +- winner_variant: A or B after reading only the blind review pack. +- confidence: Optional number from 0 to 1. +- reason: Required rationale; do not reveal baseline or with-skill labels before adjudication. +- reviewer: Human reviewer name or review group at the decision-file top level. +- reviewed_at: Review date or timestamp at the decision-file top level. +- reviewer_attestation.blind_review_completed_before_answer_key: True only after the reviewer has completed choices before opening the answer key. +- reviewer_attestation.answer_key_not_opened_before_decisions: True only when the answer key was not opened before decisions were recorded. + +#### Privacy Contract + +- Do not paste raw private user data into the decision reason. +- Do not open the answer key before reviewer choices are recorded. +- Leave winner_variant blank when the reviewer is not ready to decide. ## Next Fixes diff --git a/reports/output_review_decisions.json b/reports/output_review_decisions.json new file mode 100644 index 00000000..64a178be --- /dev/null +++ b/reports/output_review_decisions.json @@ -0,0 +1,67 @@ +{ + "schema_version": "1.0", + "reviewer": "", + "reviewed_at": "", + "review_integrity": { + "blind_pack_sha256": "788604c67fa31178c9bba06a52731adcbee9c1e72c00767cf7950643812a6e5c", + "case_count": 5, + "case_ids": [ + "skill-package-contract", + "output-eval-expectation", + "ir-before-packaging", + "near-neighbor-boundary", + "file-backed-governed-package" + ], + "prompt_sha256_by_case": { + "skill-package-contract": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5", + "output-eval-expectation": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4", + "ir-before-packaging": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80", + "near-neighbor-boundary": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0", + "file-backed-governed-package": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d" + } + }, + "reviewer_attestation": { + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded": true, + "reviewer_reason_required": true + }, + "decision_contract": { + "winner_variant": "Use A or B after reading the blind review pack. Leave blank when pending.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.", + "reviewer_attestation": "Set blind_review_completed_before_answer_key and answer_key_not_opened_before_decisions to true only after a real blind review." + }, + "decisions": [ + { + "case_id": "skill-package-contract", + "winner_variant": "", + "confidence": null, + "reason": "" + }, + { + "case_id": "output-eval-expectation", + "winner_variant": "", + "confidence": null, + "reason": "" + }, + { + "case_id": "ir-before-packaging", + "winner_variant": "", + "confidence": null, + "reason": "" + }, + { + "case_id": "near-neighbor-boundary", + "winner_variant": "", + "confidence": null, + "reason": "" + }, + { + "case_id": "file-backed-governed-package", + "winner_variant": "", + "confidence": null, + "reason": "" + } + ] +} diff --git a/reports/output_review_kit.html b/reports/output_review_kit.html new file mode 100644 index 00000000..ae9348f3 --- /dev/null +++ b/reports/output_review_kit.html @@ -0,0 +1,220 @@ + + + + + + Output Review Kit + + + + +
+
+ Blind A/B Human Review +

Reviewer cockpit for output quality decisions

+

Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.

+
Cases5
Ready0
Pending5
Invalid0
+
+
+

Review Flow

  1. Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.
  2. Record choices in reports/output_review_decisions.json without opening the answer key.
  3. Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.
  4. Set reviewer_attestation truthfully after choices are recorded and before adjudication.
  5. Run python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication after choices are recorded.
  6. Refresh python3 scripts/yao.py review-studio . before asking for release approval.
+ +
+
+
+
+
+ Case 01 +

skill-package-contract

+
+ Awaiting +
+

Turn this repeated workflow into a reusable team skill package.

+

Rubric

  • has-entrypoint

    Output names the routeable Skill entrypoint.

    1.0
  • has-interface

    Output includes neutral interface metadata.

    1.0
  • has-report

    Output points reviewer to generated overview report.

    1.0
  • has-resource-boundary

    Output preserves entrypoint/resource separation.

    1.0
+
+

Variant A

Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.
+

Variant B

I can write a prompt for that workflow and include a short checklist.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 02 +

output-eval-expectation

+
+ Awaiting +
+

Upgrade this production skill so we know whether the generated output is better than baseline.

+

Rubric

  • has-baseline-comparison

    Output explicitly compares with-skill and baseline outputs.

    1.0
  • has-assertion-grading

    Output includes objective assertion grading.

    1.0
  • has-scorecard

    Output produces a scorecard report path.

    1.0
  • has-failure-taxonomy

    Output records failure taxonomy.

    1.0
+
+

Variant A

Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.
+

Variant B

Add more examples and run the trigger eval again.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 03 +

ir-before-packaging

+
+ Awaiting +
+

Package this skill for OpenAI, Claude, Agent Skills, and generic targets.

+

Rubric

  • has-ir-first

    Output requires Skill IR before packaging.

    1.0
  • has-core-ir-fields

    Output lists core IR contract fields.

    1.0
  • has-targets

    Output names requested runtime targets.

    1.0
  • has-semantic-preservation

    Output says targets must preserve the capability contract.

    1.0
+
+

Variant A

Export folders for each platform and adjust files as needed.
+

Variant B

Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 04 +

near-neighbor-boundary

+
+ Awaiting +
+

I only need a one-off summary of these notes, not a reusable process.

+

Rubric

  • declines-skill

    Output refuses unnecessary skill creation for one-off work.

    1.0
  • names-near-neighbor

    Output labels the request as near-neighbor instead of owned work.

    1.0
  • requires-reuse-signal

    Output asks for repeat-use evidence before packaging.

    1.0
+
+

Variant A

Create a SKILL.md and a reusable workflow anyway so future notes can use it.
+

Variant B

Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 05 +

file-backed-governed-package

+
+ Awaiting +
+

Turn the attached release brief source into a governed skill package.

+

Rubric

  • uses-file-backed-evidence

    Output names file-backed source evidence.

    1.0
  • has-governance

    Output preserves governed ownership metadata.

    1.0
  • has-output-contract

    Output preserves output and rollback boundaries.

    1.0
  • has-trust-and-scorecard

    Output requires trust and output scorecard artifacts.

    1.0
  • does-not-invent-evidence

    Output forbids invented launch evidence.

    1.0
+
+

Variant A

Draft a release announcement with the changelog, support notes, and owner name.
+

Variant B

Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+

Decision Template

Use this shape in reports/output_review_decisions.json; leave a case blank when the reviewer is not ready.

{
+  "schema_version": "1.0",
+  "reviewer": "",
+  "reviewed_at": "",
+  "decisions": [
+    {
+      "case_id": "skill-package-contract",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "output-eval-expectation",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "ir-before-packaging",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "near-neighbor-boundary",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "file-backed-governed-package",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    }
+  ]
+}
+
+ + diff --git a/reports/output_review_kit.json b/reports/output_review_kit.json new file mode 100644 index 00000000..d8bf98a0 --- /dev/null +++ b/reports/output_review_kit.json @@ -0,0 +1,271 @@ +{ + "schema_version": "1.0", + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "template_written": false, + "review_contract": { + "reviewer_steps": [ + "Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.", + "Record choices in reports/output_review_decisions.json without opening the answer key.", + "Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.", + "Set reviewer_attestation truthfully after choices are recorded and before adjudication.", + "Run python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication after choices are recorded.", + "Refresh python3 scripts/yao.py review-studio . before asking for release approval." + ], + "required_fields": { + "reviewer": "Human reviewer name or review group.", + "reviewed_at": "Review date or timestamp.", + "winner_variant": "A or B for every case.", + "confidence": "Optional numeric confidence from 0 to 1.", + "reason": "Required rationale based on the rubric, not on hidden labels.", + "reviewer_attestation": "Blind-review and answer-key-order attestation after decisions are complete.", + "review_integrity": "Blind pack and prompt fingerprints generated by the template." + }, + "privacy_contract": [ + "The answer key is intentionally withheld from this kit.", + "Do not inspect hidden labels or expected winners before decisions are recorded.", + "Do not paste private user data into decision reasons.", + "Pending decisions must stay pending instead of being counted as human agreement." + ] + }, + "cases": [ + { + "case_id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0 + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0 + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0 + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "skill-package-contract:A", + "output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/." + }, + "variant_b": { + "blind_id": "skill-package-contract:B", + "output": "I can write a prompt for that workflow and include a short checklist." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0 + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0 + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0 + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "output-eval-expectation:A", + "output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md." + }, + "variant_b": { + "blind_id": "output-eval-expectation:B", + "output": "Add more examples and run the trigger eval again." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0 + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0 + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0 + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "ir-before-packaging:A", + "output": "Export folders for each platform and adjust files as needed." + }, + "variant_b": { + "blind_id": "ir-before-packaging:B", + "output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0 + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0 + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "near-neighbor-boundary:A", + "output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it." + }, + "variant_b": { + "blind_id": "near-neighbor-boundary:B", + "output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0 + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0 + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0 + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0 + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "file-backed-governed-package:A", + "output": "Draft a release announcement with the changelog, support notes, and owner name." + }, + "variant_b": { + "blind_id": "file-backed-governed-package:B", + "output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + } + ], + "failures": [] +} diff --git a/reports/output_review_kit.md b/reports/output_review_kit.md new file mode 100644 index 00000000..54a0a289 --- /dev/null +++ b/reports/output_review_kit.md @@ -0,0 +1,141 @@ +# Output Review Kit + +This reviewer-facing packet contains the blind A/B cases, decision fields, and command flow. It intentionally does not expose the answer key. + +## Summary + +- cases: `5` +- ready for adjudication: `0` +- pending decisions: `5` +- invalid decisions: `0` +- reviewer metadata present: `false` +- answer key hidden: `true` +- answer key path exposed: `false` + +## Review Flow + +1. Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case. +2. Record choices in reports/output_review_decisions.json without opening the answer key. +3. Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case. +4. Set reviewer_attestation truthfully after choices are recorded and before adjudication. +5. Run python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication after choices are recorded. +6. Refresh python3 scripts/yao.py review-studio . before asking for release approval. + +## Required Fields + +- `reviewer`: Human reviewer name or review group. +- `reviewed_at`: Review date or timestamp. +- `winner_variant`: A or B for every case. +- `confidence`: Optional numeric confidence from 0 to 1. +- `reason`: Required rationale based on the rubric, not on hidden labels. +- `reviewer_attestation`: Blind-review and answer-key-order attestation after decisions are complete. +- `review_integrity`: Blind pack and prompt fingerprints generated by the template. + +## Privacy Contract + +- The answer key is intentionally withheld from this kit. +- Do not inspect hidden labels or expected winners before decisions are recorded. +- Do not paste private user data into decision reasons. +- Pending decisions must stay pending instead of being counted as human agreement. + +## Decision States + +| Case | State | Winner | Confidence | Reason | Blocking Reason | +| --- | --- | --- | --- | --- | --- | +| `skill-package-contract` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `output-eval-expectation` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `ir-before-packaging` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `near-neighbor-boundary` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | +| `file-backed-governed-package` | `awaiting-decision` | `false` | `false` | `false` | Decision template exists but this row is still blank. | + +## Blind Cases + +### skill-package-contract + +Prompt: Turn this repeated workflow into a reusable team skill package. + +Rubric: +- `has-entrypoint` (1.0): Output names the routeable Skill entrypoint. +- `has-interface` (1.0): Output includes neutral interface metadata. +- `has-report` (1.0): Output points reviewer to generated overview report. +- `has-resource-boundary` (1.0): Output preserves entrypoint/resource separation. + +#### Variant A + +Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/. + +#### Variant B + +I can write a prompt for that workflow and include a short checklist. + +### output-eval-expectation + +Prompt: Upgrade this production skill so we know whether the generated output is better than baseline. + +Rubric: +- `has-baseline-comparison` (1.0): Output explicitly compares with-skill and baseline outputs. +- `has-assertion-grading` (1.0): Output includes objective assertion grading. +- `has-scorecard` (1.0): Output produces a scorecard report path. +- `has-failure-taxonomy` (1.0): Output records failure taxonomy. + +#### Variant A + +Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md. + +#### Variant B + +Add more examples and run the trigger eval again. + +### ir-before-packaging + +Prompt: Package this skill for OpenAI, Claude, Agent Skills, and generic targets. + +Rubric: +- `has-ir-first` (1.0): Output requires Skill IR before packaging. +- `has-core-ir-fields` (1.0): Output lists core IR contract fields. +- `has-targets` (1.0): Output names requested runtime targets. +- `has-semantic-preservation` (1.0): Output says targets must preserve the capability contract. + +#### Variant A + +Export folders for each platform and adjust files as needed. + +#### Variant B + +Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract. + +### near-neighbor-boundary + +Prompt: I only need a one-off summary of these notes, not a reusable process. + +Rubric: +- `declines-skill` (1.0): Output refuses unnecessary skill creation for one-off work. +- `names-near-neighbor` (1.0): Output labels the request as near-neighbor instead of owned work. +- `requires-reuse-signal` (1.0): Output asks for repeat-use evidence before packaging. + +#### Variant A + +Create a SKILL.md and a reusable workflow anyway so future notes can use it. + +#### Variant B + +Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract. + +### file-backed-governed-package + +Prompt: Turn the attached release brief source into a governed skill package. + +Rubric: +- `uses-file-backed-evidence` (1.0): Output names file-backed source evidence. +- `has-governance` (1.0): Output preserves governed ownership metadata. +- `has-output-contract` (1.0): Output preserves output and rollback boundaries. +- `has-trust-and-scorecard` (1.0): Output requires trust and output scorecard artifacts. +- `does-not-invent-evidence` (1.0): Output forbids invented launch evidence. + +#### Variant A + +Draft a release announcement with the changelog, support notes, and owner name. + +#### Variant B + +Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them. diff --git a/reports/package_verification.json b/reports/package_verification.json index 8c66a4eb..c372b4b4 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -1,15 +1,15 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "package_dir": "dist", "summary": { - "target_count": 3, - "adapter_count": 3, + "target_count": 4, + "adapter_count": 4, "archive_present": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", - "archive_entry_count": 488, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "archive_entry_count": 689, "failure_count": 0, "warning_count": 0 }, @@ -424,6 +424,141 @@ "status": "pass", "detail": "generic adapter includes field: portability_profile" }, + { + "id": "vscode-adapter", + "status": "pass", + "detail": "Adapter exists for target: vscode" + }, + { + "id": "vscode-field-name", + "status": "pass", + "detail": "vscode adapter includes field: name" + }, + { + "id": "vscode-field-description", + "status": "pass", + "detail": "vscode adapter includes field: description" + }, + { + "id": "vscode-field-version", + "status": "pass", + "detail": "vscode adapter includes field: version" + }, + { + "id": "vscode-field-display_name", + "status": "pass", + "detail": "vscode adapter includes field: display_name" + }, + { + "id": "vscode-field-short_description", + "status": "pass", + "detail": "vscode adapter includes field: short_description" + }, + { + "id": "vscode-field-default_prompt", + "status": "pass", + "detail": "vscode adapter includes field: default_prompt" + }, + { + "id": "vscode-field-job_to_be_done", + "status": "pass", + "detail": "vscode adapter includes field: job_to_be_done" + }, + { + "id": "vscode-field-ir_source", + "status": "pass", + "detail": "vscode adapter includes field: ir_source" + }, + { + "id": "vscode-field-ir_schema_version", + "status": "pass", + "detail": "vscode adapter includes field: ir_schema_version" + }, + { + "id": "vscode-field-semantic_contract", + "status": "pass", + "detail": "vscode adapter includes field: semantic_contract" + }, + { + "id": "vscode-field-semantic_parity", + "status": "pass", + "detail": "vscode adapter includes field: semantic_parity" + }, + { + "id": "vscode-field-compiler", + "status": "pass", + "detail": "vscode adapter includes field: compiler" + }, + { + "id": "vscode-field-compiled_contract", + "status": "pass", + "detail": "vscode adapter includes field: compiled_contract" + }, + { + "id": "vscode-field-permission_contract", + "status": "pass", + "detail": "vscode adapter includes field: permission_contract" + }, + { + "id": "vscode-field-target_permission_contract", + "status": "pass", + "detail": "vscode adapter includes field: target_permission_contract" + }, + { + "id": "vscode-field-target_native_contract", + "status": "pass", + "detail": "vscode adapter includes field: target_native_contract" + }, + { + "id": "vscode-field-target_transform", + "status": "pass", + "detail": "vscode adapter includes field: target_transform" + }, + { + "id": "vscode-field-canonical_metadata", + "status": "pass", + "detail": "vscode adapter includes field: canonical_metadata" + }, + { + "id": "vscode-field-canonical_format", + "status": "pass", + "detail": "vscode adapter includes field: canonical_format" + }, + { + "id": "vscode-field-activation_mode", + "status": "pass", + "detail": "vscode adapter includes field: activation_mode" + }, + { + "id": "vscode-field-execution_context", + "status": "pass", + "detail": "vscode adapter includes field: execution_context" + }, + { + "id": "vscode-field-shell", + "status": "pass", + "detail": "vscode adapter includes field: shell" + }, + { + "id": "vscode-field-trust_level", + "status": "pass", + "detail": "vscode adapter includes field: trust_level" + }, + { + "id": "vscode-field-remote_inline_execution", + "status": "pass", + "detail": "vscode adapter includes field: remote_inline_execution" + }, + { + "id": "vscode-field-degradation_strategy", + "status": "pass", + "detail": "vscode adapter includes field: degradation_strategy" + }, + { + "id": "vscode-field-portability_profile", + "status": "pass", + "detail": "vscode adapter includes field: portability_profile" + }, { "id": "openai-file-targets/openai/adapter.json", "status": "pass", @@ -449,6 +584,16 @@ "status": "pass", "detail": "Package contains targets/generic/adapter.json" }, + { + "id": "vscode-file-targets/vscode/adapter.json", + "status": "pass", + "detail": "Package contains targets/vscode/adapter.json" + }, + { + "id": "vscode-file-targets/vscode/README.md", + "status": "pass", + "detail": "Package contains targets/vscode/README.md" + }, { "id": "archive-safe-paths", "status": "pass", @@ -472,7 +617,7 @@ { "id": "archive-excludes-generated", "status": "pass", - "detail": "Archive excludes generated dist/ and tests/tmp* contents" + "detail": "Archive excludes generated dist/, .previews/, and tests/tmp* contents" }, { "id": "registry-ok", @@ -503,6 +648,11 @@ "id": "registry-compat-generic", "status": "pass", "detail": "Registry compatibility is reviewable for target: generic" + }, + { + "id": "registry-compat-vscode", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: vscode" } ], "failures": [], diff --git a/reports/package_verification.md b/reports/package_verification.md index 71cfac0b..96668ccf 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -2,9 +2,9 @@ - OK: `True` - Package directory: `dist` -- Targets: `3 / 3` adapters present +- Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9` +- Archive SHA256: `7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2` - Failures: `0` - Warnings: `0` @@ -94,22 +94,52 @@ | `generic-field-remote_inline_execution` | `pass` | generic adapter includes field: remote_inline_execution | | `generic-field-degradation_strategy` | `pass` | generic adapter includes field: degradation_strategy | | `generic-field-portability_profile` | `pass` | generic adapter includes field: portability_profile | +| `vscode-adapter` | `pass` | Adapter exists for target: vscode | +| `vscode-field-name` | `pass` | vscode adapter includes field: name | +| `vscode-field-description` | `pass` | vscode adapter includes field: description | +| `vscode-field-version` | `pass` | vscode adapter includes field: version | +| `vscode-field-display_name` | `pass` | vscode adapter includes field: display_name | +| `vscode-field-short_description` | `pass` | vscode adapter includes field: short_description | +| `vscode-field-default_prompt` | `pass` | vscode adapter includes field: default_prompt | +| `vscode-field-job_to_be_done` | `pass` | vscode adapter includes field: job_to_be_done | +| `vscode-field-ir_source` | `pass` | vscode adapter includes field: ir_source | +| `vscode-field-ir_schema_version` | `pass` | vscode adapter includes field: ir_schema_version | +| `vscode-field-semantic_contract` | `pass` | vscode adapter includes field: semantic_contract | +| `vscode-field-semantic_parity` | `pass` | vscode adapter includes field: semantic_parity | +| `vscode-field-compiler` | `pass` | vscode adapter includes field: compiler | +| `vscode-field-compiled_contract` | `pass` | vscode adapter includes field: compiled_contract | +| `vscode-field-permission_contract` | `pass` | vscode adapter includes field: permission_contract | +| `vscode-field-target_permission_contract` | `pass` | vscode adapter includes field: target_permission_contract | +| `vscode-field-target_native_contract` | `pass` | vscode adapter includes field: target_native_contract | +| `vscode-field-target_transform` | `pass` | vscode adapter includes field: target_transform | +| `vscode-field-canonical_metadata` | `pass` | vscode adapter includes field: canonical_metadata | +| `vscode-field-canonical_format` | `pass` | vscode adapter includes field: canonical_format | +| `vscode-field-activation_mode` | `pass` | vscode adapter includes field: activation_mode | +| `vscode-field-execution_context` | `pass` | vscode adapter includes field: execution_context | +| `vscode-field-shell` | `pass` | vscode adapter includes field: shell | +| `vscode-field-trust_level` | `pass` | vscode adapter includes field: trust_level | +| `vscode-field-remote_inline_execution` | `pass` | vscode adapter includes field: remote_inline_execution | +| `vscode-field-degradation_strategy` | `pass` | vscode adapter includes field: degradation_strategy | +| `vscode-field-portability_profile` | `pass` | vscode adapter includes field: portability_profile | | `openai-file-targets/openai/adapter.json` | `pass` | Package contains targets/openai/adapter.json | | `openai-file-targets/openai/agents/openai.yaml` | `pass` | Package contains targets/openai/agents/openai.yaml | | `claude-file-targets/claude/adapter.json` | `pass` | Package contains targets/claude/adapter.json | | `claude-file-targets/claude/README.md` | `pass` | Package contains targets/claude/README.md | | `generic-file-targets/generic/adapter.json` | `pass` | Package contains targets/generic/adapter.json | +| `vscode-file-targets/vscode/adapter.json` | `pass` | Package contains targets/vscode/adapter.json | +| `vscode-file-targets/vscode/README.md` | `pass` | Package contains targets/vscode/README.md | | `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | | `archive-entry-yao-meta-skill/SKILL.md` | `pass` | Archive contains yao-meta-skill/SKILL.md | | `archive-entry-yao-meta-skill/manifest.json` | `pass` | Archive contains yao-meta-skill/manifest.json | | `archive-entry-yao-meta-skill/agents/interface.yaml` | `pass` | Archive contains yao-meta-skill/agents/interface.yaml | -| `archive-excludes-generated` | `pass` | Archive excludes generated dist/ and tests/tmp* contents | +| `archive-excludes-generated` | `pass` | Archive excludes generated dist/, .previews/, and tests/tmp* contents | | `registry-ok` | `pass` | Registry audit is OK | | `registry-name-match` | `pass` | Registry package name matches package manifest | | `registry-version-match` | `pass` | Registry package version matches package manifest | | `registry-compat-openai` | `pass` | Registry compatibility is reviewable for target: openai | | `registry-compat-claude` | `pass` | Registry compatibility is reviewable for target: claude | | `registry-compat-generic` | `pass` | Registry compatibility is reviewable for target: generic | +| `registry-compat-vscode` | `pass` | Registry compatibility is reviewable for target: vscode | ## Failures diff --git a/reports/portability_score.json b/reports/portability_score.json index d8b36971..a6c25b6e 100644 --- a/reports/portability_score.json +++ b/reports/portability_score.json @@ -3,14 +3,14 @@ "score": 100, "band": "world-class", "summary": { - "adapter_target_count": 3, + "adapter_target_count": 4, "activation_mode": "manual", "execution_context": "inline", "shell": "bash", "trust_level": "local", "remote_inline_execution": "forbid", - "degradation_coverage": 3, - "snapshot_count": 3, + "degradation_coverage": 4, + "snapshot_count": 4, "ir_source": "skill-ir/examples/yao-meta-skill.json", "ir_schema_version": "2.0.0" }, diff --git a/reports/portability_score.md b/reports/portability_score.md index 067e3b9b..8ed9caba 100644 --- a/reports/portability_score.md +++ b/reports/portability_score.md @@ -2,7 +2,7 @@ - score: `100/100` - band: `world-class` -- adapter targets: `3` +- adapter targets: `4` - activation mode: `manual` - execution context: `inline` - shell: `bash` diff --git a/reports/promotion_decisions.json b/reports/promotion_decisions.json index f4d8406b..e84063f2 100644 --- a/reports/promotion_decisions.json +++ b/reports/promotion_decisions.json @@ -9,9 +9,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -42,44 +42,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -170,7 +170,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.611, + "average_margin": 0.701, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -193,8 +193,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -224,7 +224,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -238,7 +239,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -538,13 +540,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -563,7 +565,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -578,49 +580,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -683,7 +685,8 @@ "causes": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", "gate_status": { @@ -699,7 +702,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.659, + "average_margin": 0.586, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -713,8 +716,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -724,25 +727,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -751,7 +739,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -765,7 +753,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -779,9 +767,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } diff --git a/reports/promotion_decisions.md b/reports/promotion_decisions.md index 3ebd47f8..9be35c29 100644 --- a/reports/promotion_decisions.md +++ b/reports/promotion_decisions.md @@ -4,4 +4,4 @@ | --- | --- | --- | --- | --- | --- | --- | --- | | `yao-meta-skill` | `keep_current` | `Current` | True | 1.0 | healthy | no_candidate_outperformed_current | Keep the current description and open a new candidate only when fresh route evidence appears. | | `team-frontend-review` | `keep_current` | `Current` | True | 1.0 | healthy | no_candidate_outperformed_current, current_holdout_risk | Keep the current description and open a new candidate only when fresh route evidence appears. | -| `governed-incident-command` | `keep_current` | `Current` | True | 1.0 | healthy | no_candidate_outperformed_current, current_holdout_gap_present, current_holdout_risk | Keep the current description and open a new candidate only when fresh route evidence appears. | +| `governed-incident-command` | `keep_current` | `Current` | True | 1.0 | watch | no_candidate_outperformed_current, current_holdout_gap_present, current_holdout_risk, adversarial_watch_risk | Keep the current description and open a new candidate only when fresh route evidence appears. | diff --git a/reports/prompt-quality-profile.json b/reports/prompt-quality-profile.json index 8af0544b..cacfa510 100644 --- a/reports/prompt-quality-profile.json +++ b/reports/prompt-quality-profile.json @@ -1,6 +1,6 @@ { "skill_name": "yao-meta-skill", - "relevance": "prompt-heavy", + "relevance": "prompt-aware", "primary_task_family": { "key": "execution_operation", "label": "Execution operation", @@ -22,18 +22,6 @@ "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." }, - { - "key": "dialogue_interaction", - "label": "Dialogue interaction", - "score": 2, - "matched_keywords": [ - "dialogue", - "conversation" - ], - "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", - "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", - "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." - }, { "key": "creative_generation", "label": "Creative generation", @@ -46,15 +34,26 @@ "format_guidance": "Return options with rationale, selection criteria, and refinement paths." }, { - "key": "prompt_engineering", - "label": "Prompt engineering", + "key": "analytical_reasoning", + "label": "Analytical reasoning", "score": 1, "matched_keywords": [ - "prompt" + "decision" ], - "role_guidance": "Use a prompt engineer role only when role design materially improves execution.", - "task_guidance": "Map Role, Task, and Format into skill behavior rather than copying a large prompt template.", - "format_guidance": "Return a compact prompt contract plus tests, quality matrix, and usage notes." + "role_guidance": "Use an analyst role that separates evidence, inference, uncertainty, and recommendation.", + "task_guidance": "State assumptions, compare alternatives, and make the decision path inspectable.", + "format_guidance": "Return findings, evidence, tradeoffs, recommendation, and residual risks." + }, + { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "score": 1, + "matched_keywords": [ + "dialogue" + ], + "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." } ], "complexity": { @@ -80,6 +79,7 @@ "label": "Completeness", "score": 100, "matched_signals": [ + "input", "output", "constraint", "standard" diff --git a/reports/prompt-quality-profile.md b/reports/prompt-quality-profile.md index a0611304..500be5d6 100644 --- a/reports/prompt-quality-profile.md +++ b/reports/prompt-quality-profile.md @@ -1,7 +1,7 @@ # Prompt Quality Profile Skill: `yao-meta-skill` -Relevance: `prompt-heavy` +Relevance: `prompt-aware` Overall quality score: `89.0/100` ## Primary Task Family @@ -32,7 +32,7 @@ Overall quality score: `89.0/100` ## Quality Matrix ### Completeness — 100/100 -- Matched signals: output, constraint, standard +- Matched signals: input, output, constraint, standard - Repair: Name missing inputs, outputs, constraints, or success standards before deepening the package. ### Clarity — 85/100 @@ -60,13 +60,6 @@ Overall quality score: `89.0/100` - Task: Convert the job into ordered steps with validation checks and stop conditions. - Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant. -### Dialogue interaction -- Score: `2` -- Keywords: dialogue, conversation -- Role: Use a conversational role that asks only high-leverage questions and remembers the user's goal. -- Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list. -- Format: Return concise prompts, decision points, and reviewer-visible assumptions. - ### Creative generation - Score: `1` - Keywords: content @@ -74,12 +67,19 @@ Overall quality score: `89.0/100` - Task: Generate variants, explain selection logic, and preserve the user's distinctive constraints. - Format: Return options with rationale, selection criteria, and refinement paths. -### Prompt engineering +### Analytical reasoning - Score: `1` -- Keywords: prompt -- Role: Use a prompt engineer role only when role design materially improves execution. -- Task: Map Role, Task, and Format into skill behavior rather than copying a large prompt template. -- Format: Return a compact prompt contract plus tests, quality matrix, and usage notes. +- Keywords: decision +- Role: Use an analyst role that separates evidence, inference, uncertainty, and recommendation. +- Task: State assumptions, compare alternatives, and make the decision path inspectable. +- Format: Return findings, evidence, tradeoffs, recommendation, and residual risks. + +### Dialogue interaction +- Score: `1` +- Keywords: dialogue +- Role: Use a conversational role that asks only high-leverage questions and remembers the user's goal. +- Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list. +- Format: Return concise prompts, decision points, and reviewer-visible assumptions. ## Self-Repair Checks diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json new file mode 100644 index 00000000..5c81f5f1 --- /dev/null +++ b/reports/python_compatibility.json @@ -0,0 +1,1423 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "root": ".", + "summary": { + "target_python": "3.11", + "file_count": 232, + "issue_count": 0, + "syntax_error_count": 0, + "fstring_311_violation_count": 0, + "decision": "pass" + }, + "rules": [ + { + "key": "current-python-syntax", + "reason": "Every scanned Python source file must compile under the running interpreter." + }, + { + "key": "fstring-expression-backslash", + "reason": "Python 3.11 rejects backslashes inside f-string expressions; keep escaping outside the expression." + } + ], + "files": [ + { + "path": "examples/complex-release-orchestrator/generated-skill/scripts/build_release_packet.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "examples/governed-incident-command/generated-skill/scripts/build_incident_packet.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "geo-ranking-article-generator/scripts/build_html_index.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/adaptation_patch_safety.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/adaptation_report_contracts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/adjudicate_output_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/apply_adaptation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/benchmark_release_lock.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_confusion_matrix.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_skill_atlas.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_skill_atlas_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_skill_atlas_opportunities.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/check_update.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/ci_test.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/collect_feedback.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/compile_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/compile_skill_targets.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/context_sizer.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/create_iteration_snapshot.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/cross_packager.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/cross_packager_contracts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/description_optimizer_reporting.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/diff_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/emit_telemetry_event.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_core.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_phase_queue.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_release.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_skill_os2_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/export_skill_ir.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/github_benchmark_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/governance_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/import_output_review_decisions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/import_telemetry_events.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/init_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/judge_blind_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/lint_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/local_output_eval_runner.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/optimize_description.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/output_review_privacy.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/prepare_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/prepare_world_class_submission_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/probe_runtime_permissions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/promotion_checker.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/propose_adaptation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/provider_output_eval_runner.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/python_compat_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/reference_synthesis_markdown.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/registry_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_adoption_drift_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_architecture_maintainability.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_artifact_design_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_baseline_compare.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_context_reports.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_daily_skillops_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_description_drift_history.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_eval_dashboard.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_evidence_consistency.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_intent_confidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_intent_dialogue.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_iteration_directions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_iteration_ledger.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_output_risk_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_portability_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_prompt_quality_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_reference_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_reference_synthesis.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_regression_history.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_annotations.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_studio.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_viewer.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_waivers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_interpretation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_os2_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_os2_coverage.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_overview.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_social_preview.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_system_model.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_telemetry_hook_recipes.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_weekly_curator_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_claim_guard.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_evidence_intake.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_evidence_plan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_preflight.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/resource_boundary_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_action_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_actions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_data.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_formatting.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_gate_contract.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_gate_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_gates.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_output_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_panels.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_skillops.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_waivers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_viewer_data.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_conformance_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_description_optimization_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_eval_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_output_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_output_execution.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/simulate_install.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_ir_paths.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_os2_coverage_markdown.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_charts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_i18n.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_metrics.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_model.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_sections.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_sources.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skillops_opportunity.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/summarize_user_signals.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/sync_local_install.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/telemetry_native_host.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/trigger_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/trust_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/trust_check_scripts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/upgrade_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/validate_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/verify_package.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_evidence_contract.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_human_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_operator_runbook_coordination.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_phase_queue.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_preflight_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_preflight_markdown.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_provider_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_repair_checklist.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_source_checks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_submission_kit_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_submission_matrix.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_adaptation_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_config.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_create_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_distribution_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_output_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_parser.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_parser_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_parser_operations.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_report_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_runtime.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_telemetry.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/review_studio_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/review_studio_world_class_assertions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_adaptation_safety.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_adapter_snapshots.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_adoption_drift.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_architecture_maintainability.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_artifact_design_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_baseline_compare.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_benchmark_reproducibility.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_compile_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_conformance_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_daily_skillops.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_description_optimization.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_evidence_consistency.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_failure_regressions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_feedback.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_github_benchmark_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_install_simulation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_intent_confidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_iteration_directions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_local_install_sync.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_eval_lab.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_execution_runs.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_review_adjudication.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_risk_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_package_verification.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_packager_failures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_portability_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_promotion_checker.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_prompt_quality_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_python_compat_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_quality_checks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_reference_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_reference_synthesis.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_registry_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_annotations.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_studio.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_viewer.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_waivers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_route_confusion.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_runtime_permission_probes.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_security_boundaries.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_atlas.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_interpretation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_ir.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_ir_paths.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_os2_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_os2_coverage.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_overview.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_report_charts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_report_metrics.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skillops_opportunity.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_system_model.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_emit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_hooks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_import.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_native_host.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_trust_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_upgrade_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_weekly_curator.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_claim_guard.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_evidence_ledger.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_evidence_plan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_preflight.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_submission_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_yao_cli.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_yao_cli_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/world_class_human_fixtures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/world_class_native_telemetry_fixtures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/yao_cli_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/yao_cli_report_refresh.py", + "ok": true, + "issue_count": 0, + "issues": [] + } + ], + "issues": [], + "artifacts": { + "json": "reports/python_compatibility.json", + "markdown": "reports/python_compatibility.md" + } +} diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md new file mode 100644 index 00000000..15ecf61f --- /dev/null +++ b/reports/python_compatibility.md @@ -0,0 +1,20 @@ +# Python Compatibility + +Generated at: `2026-06-17` + +## Summary + +- decision: `pass` +- target python: `3.11` +- files scanned: `232` +- issues: `0` +- syntax errors: `0` +- f-string 3.11 violations: `0` + +This report catches Python syntax and compatibility hazards that can pass on a newer local interpreter but fail in the supported CI/runtime interpreter. + +## Issues + +| Path | Line | Rule | Message | +| --- | ---: | --- | --- | +| `none` | 0 | `none` | none | diff --git a/reports/reference-synthesis.json b/reports/reference-synthesis.json index a6141d27..b27e846a 100644 --- a/reports/reference-synthesis.json +++ b/reports/reference-synthesis.json @@ -1,6 +1,6 @@ { "skill_name": "yao-meta-skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "intent_confidence": { "score": 100, "band": "high", diff --git a/reports/reference-synthesis.md b/reports/reference-synthesis.md index 77adb184..82c5a69e 100644 --- a/reports/reference-synthesis.md +++ b/reports/reference-synthesis.md @@ -1,7 +1,7 @@ # Reference Synthesis Skill: `yao-meta-skill` -- Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +- Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution. - Intent confidence: `100/100` (`high`) ## Live GitHub Benchmarks diff --git a/reports/registry_audit.json b/reports/registry_audit.json index c8736dd8..a5d62d41 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -7,12 +7,13 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "maturity": "governed", "owner": "Yao Team", @@ -20,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f", - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { "openai": "pass", @@ -52,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -73,10 +74,11 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index eee4c665..e26d5cbc 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f` -- Archive SHA256: `7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9` +- Package SHA256: `5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257` +- Archive SHA256: `7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 5382095a..e5d9c2b2 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -158,6 +158,217 @@ } .action-card.warn { border-left: 4px solid var(--warn); } .action-card.block { border-left: 4px solid var(--block); } + .action-card.with-evidence { + grid-column: 1 / -1; + } + .action-evidence-panel { + display: grid; + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + margin-top: 2px; + } + .action-evidence-panel h4 { + margin: 0; + color: var(--ink); + font-size: 16px; + font-weight: 500; + } + .action-evidence-panel p { + color: var(--muted); + font-size: 14px; + } + .action-evidence-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 14px; + } + .action-evidence-item { + border: 1px solid var(--line); + border-radius: 8px; + padding: 14px; + display: grid; + gap: 10px; + min-width: 0; + background: var(--soft); + } + .action-evidence-item.pending { border-left: 4px solid var(--warn); } + .action-evidence-item.accepted { border-left: 4px solid var(--pass); } + .action-evidence-item h4, + .action-evidence-item h5 { + margin: 0; + color: var(--ink); + font-weight: 500; + } + .action-evidence-item h4 { font-size: 16px; } + .action-evidence-item h5 { font-size: 14px; } + .action-evidence-item section { + border-top: 1px solid var(--line); + padding-top: 10px; + display: grid; + gap: 8px; + } + .action-command-details, + .action-phase-details, + .action-runbook-details, + .action-collection-details { + border-top: 1px solid var(--line); + padding-top: 10px; + display: grid; + gap: 8px; + } + .action-command-details summary, + .action-phase-details summary, + .action-runbook-details summary, + .action-collection-details summary { + cursor: pointer; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .action-command-details[open] summary, + .action-phase-details[open] summary, + .action-runbook-details[open] summary, + .action-collection-details[open] summary { + margin-bottom: 8px; + } + .action-collection-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; + } + .action-collection-grid section { + border-top: 1px solid var(--line); + padding-top: 8px; + display: grid; + gap: 6px; + } + .action-collection-grid ul, + .action-collection-grid ol { + margin: 0; + padding-left: 18px; + display: grid; + gap: 5px; + } + .action-collection-grid li { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .action-artifact-roles { + padding: 0; + margin: 0; + list-style: none; + display: grid; + gap: 8px; + } + .action-artifact-roles li { + border: 1px solid var(--line); + border-radius: 8px; + background: #fff; + padding: 10px; + display: grid; + gap: 4px; + min-width: 0; + } + .action-artifact-roles strong { + color: var(--ink); + font-weight: 500; + overflow-wrap: anywhere; + } + .action-artifact-roles span, + .action-artifact-roles code, + .action-artifact-roles small { + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; + white-space: normal; + } + .action-evidence-item dl { + display: grid; + grid-template-columns: 76px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .action-evidence-item dt { + color: var(--ink); + font-size: 14px; + } + .action-evidence-item dd { + margin: 0; + min-width: 0; + } + .action-evidence-checks, + .action-phase-list, + .action-command-list, + .action-repair-list, + .action-runbook-list { + margin: 0; + display: grid; + gap: 8px; + } + .action-evidence-checks, + .action-phase-list, + .action-repair-list, + .action-command-list { + padding: 0; + list-style: none; + } + .action-runbook-list { + padding-left: 20px; + } + .action-evidence-check, + .action-phase-row, + .action-repair-row, + .action-command-list li { + display: grid; + gap: 3px; + min-width: 0; + border-top: 1px solid var(--line); + padding-top: 8px; + } + .action-evidence-check.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .action-evidence-check.pass { border-left: 3px solid var(--pass); padding-left: 8px; } + .action-phase-row.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .action-phase-row.ready { border-left: 3px solid var(--pass); padding-left: 8px; } + .action-repair-row.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .action-repair-row.ready { border-left: 3px solid var(--pass); padding-left: 8px; } + .action-evidence-check span, + .action-phase-row span, + .action-repair-row span, + .action-command-list span { + color: var(--ink); + font-size: 13px; + } + .action-phase-row strong, + .action-repair-row strong { + color: var(--ink); + font-size: 14px; + overflow-wrap: anywhere; + } + .action-phase-row em, + .action-repair-row em { + color: var(--muted); + font-size: 12px; + font-style: normal; + overflow-wrap: anywhere; + } + .action-evidence-check code, + .action-evidence-check small, + .action-phase-row code, + .action-phase-row small, + .action-repair-row code, + .action-repair-row small, + .action-command-list code, + .action-runbook-list li { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .action-command-list code { + display: block; + white-space: normal; + } .annotation-card { border: 1px solid var(--line); border-radius: 8px; @@ -170,6 +381,316 @@ .annotation-card.warning { border-left: 4px solid var(--warn); } .annotation-card.blocker { border-left: 4px solid var(--block); } .annotation-card.resolved { opacity: .72; } + .world-evidence-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + margin-top: 16px; + } + .world-evidence-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .world-evidence-card.pending { border-left: 4px solid var(--warn); } + .world-evidence-card.accepted { border-left: 4px solid var(--pass); } + .world-evidence-card span, + .world-evidence-card p, + .world-evidence-card dd, + .world-evidence-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .world-evidence-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .world-evidence-card dt { color: var(--ink); font-size: 14px; } + .world-evidence-card dd { margin: 0; } + .world-evidence-columns { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-evidence-columns h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-evidence-columns ul { + margin: 0; + padding-left: 18px; + } + .world-source-panel { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-runbook-panel { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-runbook-panel h4, + .world-source-panel h4 { + margin: 0 0 8px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-runbook-panel ul { + margin: 0; + padding-left: 18px; + } + .world-source-checks { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 8px; + } + .world-source-check { + display: grid; + grid-template-columns: minmax(0, .7fr) minmax(0, .6fr) minmax(0, 1fr); + gap: 10px; + align-items: start; + min-width: 0; + padding: 9px 0 0; + border-top: 1px solid var(--line); + } + .world-source-check span { + color: var(--ink); + font-size: 14px; + } + .world-source-check code, + .world-source-check small { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .world-source-check.pass { border-left: 3px solid var(--pass); padding-left: 8px; } + .world-source-check.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } + .output-review-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + margin-top: 16px; + } + .output-review-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .output-review-card.awaiting-decision, + .output-review-card.fix-decision { border-left: 4px solid var(--warn); } + .output-review-card.adjudicated { border-left: 4px solid var(--pass); } + .output-review-card span, + .output-review-card p, + .output-review-card dd, + .output-review-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .output-review-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .output-review-card dt { color: var(--ink); font-size: 14px; } + .output-review-card dd { margin: 0; min-width: 0; } + .output-review-steps { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .output-review-steps h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .output-review-steps ul { + margin: 0; + padding-left: 18px; + } + .output-review-commands, + .output-review-fields { + list-style: none; + padding: 0; + display: grid; + gap: 8px; + } + .output-review-commands li, + .output-review-fields li { + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + } + .output-review-commands code { + overflow-wrap: anywhere; + white-space: normal; + display: block; + } + .world-intake-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + margin-top: 16px; + } + .world-intake-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .world-intake-card.awaiting-submission, + .world-intake-card.fix-submission, + .world-intake-card.fix-template { border-left: 4px solid var(--warn); } + .world-intake-card.ready-for-ledger-review, + .world-intake-card.accepted { border-left: 4px solid var(--pass); } + .world-intake-card span, + .world-intake-card p, + .world-intake-card dd, + .world-intake-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .world-intake-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .world-intake-card dt { color: var(--ink); font-size: 14px; } + .world-intake-card dd { margin: 0; min-width: 0; } + .world-intake-steps { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-intake-steps h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-intake-steps ul { + margin: 0; + padding-left: 18px; + } + .world-intake-commands { + list-style: none; + padding: 0; + display: grid; + gap: 8px; + } + .world-intake-commands li { + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + } + .world-intake-commands code { + overflow-wrap: anywhere; + white-space: normal; + display: block; + } + .section-subtitle { + margin-top: 22px; + margin-bottom: 12px; + color: var(--ink); + } + .waiver-summary { margin-top: 16px; } + .waiver-candidate-grid { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 16px; + } + .waiver-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 12px; + min-width: 0; + } + .waiver-card.allowed { border-left: 4px solid var(--warn); } + .waiver-card.blocked { border-left: 4px solid var(--block); } + .waiver-card span, + .waiver-card p, + .waiver-card dd, + .waiver-card li { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .waiver-card dl { + display: grid; + grid-template-columns: 86px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .waiver-card dt { + color: var(--ink); + font-size: 14px; + } + .waiver-card dd { margin: 0; min-width: 0; } + .waiver-review { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .waiver-review h4 { + margin: 0 0 6px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .waiver-review ul { + margin: 0; + padding-left: 18px; + } + .waiver-command { + display: grid; + gap: 4px; + border-top: 1px solid var(--line); + padding-top: 12px; + } + .waiver-command span { + color: var(--ink); + font-size: 14px; + } + .waiver-command code { + display: block; + max-width: 100%; + overflow-wrap: anywhere; + white-space: normal; + } .action-card span, .annotation-card span, .action-card small, @@ -214,12 +735,21 @@ font-size: 12px; color: var(--muted); } + .source-ref-list blockquote { + margin: 2px 0 0; + padding: 7px 9px; + border-left: 3px solid var(--line); + background: var(--soft); + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; + } code { font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 13px; } @media (max-width: 980px) { - .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .kv-grid { grid-template-columns: 1fr; } + .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .action-evidence-grid, .action-evidence-item dl, .action-collection-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-source-check, .world-intake-grid, .world-intake-steps, .waiver-candidate-grid, .waiver-card dl, .kv-grid { grid-template-columns: 1fr; } main { padding: 32px 18px 60px; } nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; } nav a { flex: 0 0 auto; } @@ -227,37 +757,42 @@ - +
Review Studio 2.0

Yao Meta Skill

-

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

审查结论 - ready - Score 100/100 + review + Score 91/100

核心指标

-
Skill IR2.0.0

4 targets in platform-neutral contract

Compiler4/4

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe3/3

0 native enforcement targets

Trust0

66 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

4 targets; MIT license

Archivepass

488 zip entries; package verification

Installpass

3 adapters readable; local install simulation

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 10; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

3 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

153 scripts scanned; secrets found

Py Compat0

232 files scanned for Python 3.11

Arch Debt0

719 largest lines; 0 watchlist; 6 early; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock false

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

689 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
通过

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 987/1000; quality density 131.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 66 scripts; 2 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

3/3 targets probed; native 0; metadata fallback 3; residual risks 3

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
通过

人工批准

0 active waivers cover current warnings

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 998/1000; deferred 525864/120000; top deferred scripts 464561; resource governance governed; quality density 140.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 153 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 232 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

229 Python files; 0 hotspots; 0 watchlist files; 6 early watch files; 0 blockers; largest 719 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
+ +
+

闸门契约

+
Ok
是
期望 Gate
16
实际 Gate
16
缺失 Gate
无
未知 Gate
无
重复 Gate
无
未加权 Gate
无

阻断事项

无。

-

关注事项

无。

+

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

修复动作

-

当前没有 blocker 或 warning。保持现有证据链即可。

+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 10; token-observed 10

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
0 blocked / 3 pass
修复
1 repair rows; counts as completion: false
阶段
1 blocked phases; counts as completion: false
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查

暂无阻断检查。

阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: operator with provider credentialsevidence: provider-holdoutSet one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckprovider-api-keyoperator with provider credentialsRequired env_any precheck is missing.Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  2. For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  3. For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
5 blocked / 5 pass
修复
6 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: human reviewerevidence: human-adjudicationAssign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source5 / 5 blockedowners: human reviewerevidence: human-adjudicationSet reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckhuman-reviewerhuman reviewerRequired human precheck is human-required.Assign a real reviewer identity before claiming human adjudication.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkblind_review_attestedhuman reviewerCurrent value False does not satisfy 'true'.Set reviewer_attestation only after choices are completed before opening the answer key.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkjudgment_counthuman reviewerCurrent value 0 does not satisfy '==pair_count'.Every pair needs one valid human judgment.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkpending_counthuman reviewerCurrent value 5 does not satisfy '==0'.Record a reviewer choice and reason for every pair.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkready_for_human_evidencehuman reviewerCurrent value False does not satisfy 'true'.Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkreviewer_metadata_presenthuman reviewerCurrent value False does not satisfy 'true'.Record reviewer and reviewed_at before adjudication can count.python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
修复
2 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementAttach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source1 / 1 blockedowners: target client or installer integratorevidence: native-permission-enforcementCollect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · prechecknative-guardtarget client or installer integratorRequired external precheck is external-required.Attach a real target-client or external installer runtime guard; metadata fallback is not enough.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checknative_enforcement_counttarget client or installer integratorCurrent value 0 does not satisfy '>0'.Collect real target-client or external runtime guard proof.python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
修复
3 repair rows; counts as completion: false
阶段
2 blocked phases; counts as completion: false
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
阶段队列
  • #20 · unblock-accessUnblock access1 / 1 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryInstall a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-sourceCollect source2 / 2 blockedowners: Browser/Chrome/IDE/provider client integratorevidence: native-client-telemetryTelemetry must include adoption outcome evidence.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
修复清单
  • #20 · unblock-access · precheckexternal-clientBrowser/Chrome/IDE/provider client integratorRequired external precheck is external-required.Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkadoption_sample_countBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Telemetry must include adoption outcome evidence.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
  • #40 · collect-source · source-checkexternal_source_eventsBrowser/Chrome/IDE/provider client integratorCurrent value 0 does not satisfy '>0'.Import at least one metadata-only event from a real client.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -273,83 +808,144 @@

证据路径

-
  • skill_overviewreports/skill-overview.html
  • review_viewerreports/review-viewer.html
  • output_evalreports/output_quality_scorecard.md
  • output_executionreports/output_execution_runs.md
  • output_blind_reviewreports/output_blind_review_pack.md
  • output_review_adjudicationreports/output_review_adjudication.md
  • runtime_conformancereports/conformance_matrix.md
  • trust_reportreports/security_trust_report.md
  • permission_policysecurity/permission_policy.md
  • runtime_permissionsreports/runtime_permission_probes.md
  • skill_atlasreports/skill_atlas.html
  • compiled_targetsreports/compiled_targets.md
  • adoption_driftreports/adoption_drift_report.md
  • review_waiversreports/review_waivers.md
  • review_annotationsreports/review_annotations.md
  • registry_auditreports/registry_audit.md
  • package_verificationreports/package_verification.md
  • install_simulationreports/install_simulation.md
  • upgrade_checkreports/upgrade_check.md
  • migrationdocs/migration-v2.md
+
  • skill_overviewreports/skill-overview.html
  • review_viewerreports/review-viewer.html
  • output_evalreports/output_quality_scorecard.md
  • output_executionreports/output_execution_runs.md
  • output_blind_reviewreports/output_blind_review_pack.md
  • output_review_kitreports/output_review_kit.md
  • output_review_kit_htmlreports/output_review_kit.html
  • output_review_decisionsreports/output_review_decisions.json
  • output_review_adjudicationreports/output_review_adjudication.md
  • benchmark_reproducibilityreports/benchmark_reproducibility.md
  • skill_os2_coveragereports/skill_os2_coverage.md
  • runtime_conformancereports/conformance_matrix.md
  • trust_reportreports/security_trust_report.md
  • python_compatibilityreports/python_compatibility.md
  • architecture_maintainabilityreports/architecture_maintainability.md
  • permission_policysecurity/permission_policy.md
  • runtime_permissionsreports/runtime_permission_probes.md
  • skill_atlasreports/skill_atlas.html
  • compiled_targetsreports/compiled_targets.md
  • adoption_driftreports/adoption_drift_report.md
  • review_waiversreports/review_waivers.md
  • review_annotationsreports/review_annotations.md
  • adaptation_proposalsreports/adaptation_proposals.md
  • adaptation_approval_ledgerreports/adaptation_approval_ledger.json
  • adaptation_regressionreports/adaptation_regression_report.md
  • world_class_evidence_planreports/world_class_evidence_plan.md
  • world_class_evidence_ledgerreports/world_class_evidence_ledger.md
  • world_class_evidence_intakereports/world_class_evidence_intake.md
  • world_class_evidence_preflightreports/world_class_evidence_preflight.md
  • world_class_evidence_preflight_htmlreports/world_class_evidence_preflight.html
  • world_class_submission_reviewreports/world_class_submission_review.md
  • world_class_operator_runbookreports/world_class_operator_runbook.md
  • world_class_operator_runbook_htmlreports/world_class_operator_runbook.html
  • world_class_claim_guardreports/world_class_claim_guard.md
  • registry_auditreports/registry_audit.md
  • package_verificationreports/package_verification.md
  • install_simulationreports/install_simulation.md
  • upgrade_checkreports/upgrade_check.md
  • migrationdocs/migration-v2.md
  • skill_irskill-ir/examples/yao-meta-skill.json
  • daily_skillopsreports/skillops/daily/2026-06-17.md
  • weekly_curatorreports/skillops/weekly/2026-W25.md

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

-

组合治理

Skill 数
12
纳入审查
1
待处理冲突
0
待处理 owner
0
待处理过期
0
非行动项
24
+

组合治理

Skill 数
12
纳入审查
1
待处理冲突
0
待处理 owner
0
待处理过期
0
Telemetry Report Count
1
Actionable Drift Signal Count
0
Drift Signal Count
0
非行动项
24

输出实验

案例数
5
With Skill
100
Baseline
0
增益
100
Gate
是
-

执行证据

运行数
10
命令执行
10
模型执行
0
记录样本
0
计时样本
10
估算 Token
10
+

执行证据

运行数
10
命令执行
10
模型执行
10
记录样本
0
计时样本
10
估算 Token
0

盲评包

Pair Count
5
Answer Key Separate
是
With Skill Hidden Count
5
-

审定报告

Pair Count
5
Judgment Count
0
待审
5
Agreement Count
0
Disagreement Count
0
Invalid Decision Count
0
+

审定报告

Pair Count
5
Judgment Count
0
待审
5
Agreement Count
0
Disagreement Count
0
Invalid Decision Count
0
答案揭示
0
答案隐藏
5
Reviewer Checklist Count
5
Reviewer Checklist Pending Count
5
+

评审清单

先打开 reports/output_review_kit.md；每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

awaiting-decision · 答案隐藏

skill-package-contract

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示哈希
95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • import_decisionspython3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonRequired rationale; do not reveal baseline or with-skill labels before adjudication.
  • reviewerHuman reviewer name or review group at the decision-file top level.
  • reviewed_atReview date or timestamp at the decision-file top level.
  • reviewer_attestation.blind_review_completed_before_answer_keyTrue only after the reviewer has completed choices before opening the answer key.
  • reviewer_attestation.answer_key_not_opened_before_decisionsTrue only when the answer key was not opened before decisions were recorded.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

output-eval-expectation

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示哈希
3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • import_decisionspython3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonRequired rationale; do not reveal baseline or with-skill labels before adjudication.
  • reviewerHuman reviewer name or review group at the decision-file top level.
  • reviewed_atReview date or timestamp at the decision-file top level.
  • reviewer_attestation.blind_review_completed_before_answer_keyTrue only after the reviewer has completed choices before opening the answer key.
  • reviewer_attestation.answer_key_not_opened_before_decisionsTrue only when the answer key was not opened before decisions were recorded.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

ir-before-packaging

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示哈希
d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • import_decisionspython3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonRequired rationale; do not reveal baseline or with-skill labels before adjudication.
  • reviewerHuman reviewer name or review group at the decision-file top level.
  • reviewed_atReview date or timestamp at the decision-file top level.
  • reviewer_attestation.blind_review_completed_before_answer_keyTrue only after the reviewer has completed choices before opening the answer key.
  • reviewer_attestation.answer_key_not_opened_before_decisionsTrue only when the answer key was not opened before decisions were recorded.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

near-neighbor-boundary

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示哈希
2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • import_decisionspython3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonRequired rationale; do not reveal baseline or with-skill labels before adjudication.
  • reviewerHuman reviewer name or review group at the decision-file top level.
  • reviewed_atReview date or timestamp at the decision-file top level.
  • reviewer_attestation.blind_review_completed_before_answer_keyTrue only after the reviewer has completed choices before opening the answer key.
  • reviewer_attestation.answer_key_not_opened_before_decisionsTrue only when the answer key was not opened before decisions were recorded.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.
awaiting-decision · 答案隐藏

file-backed-governed-package

Reviewer has not selected A or B yet; answer key remains hidden.

盲评包
reports/output_blind_review_pack.json
决策表
reports/output_review_decisions.json
提示哈希
c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d

操作命令

  • prepare_review_kitpython3 scripts/yao.py output-review-kit
  • write_templatepython3 scripts/adjudicate_output_review.py --write-template
  • import_decisionspython3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • adjudicatepython3 scripts/yao.py output-review
  • refresh_review_studiopython3 scripts/yao.py review-studio .

字段要求

  • winner_variantA or B after reading only the blind review pack.
  • confidenceOptional number from 0 to 1.
  • reasonRequired rationale; do not reveal baseline or with-skill labels before adjudication.
  • reviewerHuman reviewer name or review group at the decision-file top level.
  • reviewed_atReview date or timestamp at the decision-file top level.
  • reviewer_attestation.blind_review_completed_before_answer_keyTrue only after the reviewer has completed choices before opening the answer key.
  • reviewer_attestation.answer_key_not_opened_before_decisionsTrue only when the answer key was not opened before decisions were recorded.

隐私边界

  • Do not paste raw private user data into the decision reason.
  • Do not open the answer key before reviewer choices are recorded.
  • Leave winner_variant blank when the reviewer is not ready to decide.

发布标准

Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。

人工结论

没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。

-

评审方式

先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

+

评审方式

先打开 reports/output_review_kit.html 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

运行方式

reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

运行矩阵

目标数
5
通过数
5
Fail Count
0
-

目标编译

目标数
4
通过数
4
Warn Count
0
Block Count
0
失败数
0
+

目标编译

目标数
5
通过数
5
Warn Count
0
Block Count
0
失败数
0
-

上下文

initial load 987/1000; quality density 131.7

+

上下文

initial load 998/1000; deferred 525864/120000; top deferred scripts 464561; resource governance governed; quality density 140.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
66
网络脚本
2
Help 失败
0
包体哈希
742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f
+

信任报告

Secret
0
脚本数
153
网络脚本
3
Help 失败
0
包体哈希
5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

+
+

Python 兼容

目标 Python
3.11
文件数
232
问题数
0
语法错误
0
F-string 3.11
0
+

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。

+
+

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

批准策略

高权限能力需要 reviewer、scope、reason、expires_at 和 openai/claude/generic 目标端 enforcement 说明。

-

权限探针

3/3 targets probed; native 0; metadata fallback 3; residual risks 3

-

运行边界

目标数
3
通过数
3
原生执行
0
Metadata fallback
3
残余风险
3
失败数
0
+

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

+

运行边界

目标数
4
通过数
4
原生执行
0
Metadata fallback
4
安装拦截
4
安装权限失败
0
残余风险
4
失败数
0
-

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

+

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

下一动作

优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再用运营回路判断真实影响。

-

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

-

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

+

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
-
-

人工批准

0 active waivers cover current warnings

-

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0
+ +
+

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
否
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
+

周度队列

决策
curator-review
Week Id
2026-W25
日报数
2
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
否
Writes Source Files
否
Auto Patch Enabled
否
+
+ + +
+

人工批准

+

warning 可以被有边界地接受，但必须写入 reviewer、理由、范围和到期时间；blocker 与 world-class 完成证据不能通过 waiver 变成通过。

+
+

批准概况

0 active waivers; 1 warning gates still need reviewer decision

+

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0
+
+

批准候选

+
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 10; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .
+
+ +
+

世界证据

+

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 10; token-observed 10
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 10; timing_observed_count: 10; token_observed_count: 10; accepted: True
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 10 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 10 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
+ +
+

提交清单

+

每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界；只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后，才进入 ledger review。

+
awaiting-submission · external

Provider Holdout

No real evidence submission has been provided yet.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
+ +
+

蓝图覆盖

项目数
21
模块数
8
建议 PR
13
通过数
21
Warn Count
0
缺失数
0
Extension Track Count
4
Extension Covered Count
4
Extension Partial Count
0
Extension Planned Count
0
Adaptive Extension Ready
是
本地蓝图
是
世界级
否
待补证据
4
+

覆盖边界

蓝图覆盖只证明 2.0 模块、建议 PR、脚本、报告和测试在本地闭环；public world-class 仍以 world-class evidence ledger 的真人和外部证据为准。

+
+ +
+

公开声明

本地复现
是
发布锁
是
可公开声明
否
声明阻断
3
Provider 证据
是
人审完成
否
世界级就绪
否
+

声明阻断

  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (3 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (11/19 pass, 8 blocked)
+
+ +
+

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

+

证据台账

Ledger Entry Count
4
Accepted Count
0
待审
4
Human Pending Count
1
External Pending Count
3
Overclaim Guard Active
是
Ready To Claim World Class
否
+
+ +
+

证据入口

Schema
是
模板数
4
模板通过
4
提交数
0
有效提交
0
无效提交
0
可收证据
是
可审台账
否
Ready To Claim World Class
否
+

入口边界

intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。

+
+ +
+

声明守卫

台账可声明
否
台账待补
4
声明面
182
违规数
0
Overclaim Guard Active
是
+

声明边界

claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

-

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible
兼容通过
6/6
归档哈希
7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9
+

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
3
Adapter
3
归档存在
是
Zip 条目
488
失败数
0
警告数
0
归档哈希
7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
689
失败数
0
警告数
0
归档哈希
7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2
+
+ +
+

安装模拟

Archive Extracted
是
Entrypoint Loaded
是
Manifest Loaded
是
Interface Loaded
是
Adapter
4
安装权限
12
安装权限失败
0
权限目标
4
权限能力
3
失败数
0
警告数
0
+

权限覆盖

安装模拟会读取解压后的 target adapter，并按 declared_capabilities 核对 security/permission_policy.json 中的有效批准、证据、过期时间和目标端 enforcement 说明。

diff --git a/reports/review-studio.json b/reports/review-studio.json index bee0ceda..a17a2497 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -3,12 +3,13 @@ "ok": true, "skill_dir": ".", "summary": { - "decision": "ready", - "world_class_score": 100, - "gate_count": 13, + "decision": "review", + "world_class_score": 91, + "gate_count": 16, + "gate_contract_ok": true, "blocker_count": 0, - "warning_count": 0, - "action_count": 0, + "warning_count": 3, + "action_count": 3, "annotation_count": 0, "open_annotation_count": 0, "open_annotation_blocker_count": 0, @@ -21,7 +22,14 @@ "status": "pass", "detail": "intent confidence 100/100; Intent is clear enough to package the first routeable version.", "evidence": "reports/intent-confidence.json", - "link": "intent-confidence.md" + "link": "intent-confidence.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "trigger-lab", @@ -29,23 +37,44 @@ "status": "pass", "detail": "13 trigger cases; 0 misroutes; 0 ambiguous", "evidence": "reports/route_scorecard.json", - "link": "route_scorecard.md" + "link": "route_scorecard.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "output-lab", "label": "输出实验", - "status": "pass", - "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5", + "status": "warn", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5", "evidence": "reports/output_quality_scorecard.json", - "link": "output_quality_scorecard.md" + "link": "output_quality_scorecard.md", + "review_action_id": "review-action-output-lab", + "review_action_status": "warn", + "review_action_title": "输出实验", + "review_action_next_step": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。", + "review_action_reason": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "review_action_source_ref_count": 6, + "review_action_verification_command": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review" }, { "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 987/1000; quality density 131.7", + "detail": "initial load 998/1000; deferred 525864/120000; top deferred scripts 464561; resource governance governed; quality density 140.3", "evidence": "reports/context_budget.json", - "link": "context_budget.md" + "link": "context_budget.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "runtime-matrix", @@ -53,15 +82,59 @@ "status": "pass", "detail": "5 / 5 targets pass", "evidence": "reports/conformance_matrix.json", - "link": "conformance_matrix.md" + "link": "conformance_matrix.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 66 scripts; 2 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 153 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", - "link": "security_trust_report.md" + "link": "security_trust_report.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" + }, + { + "key": "python-compat", + "label": "Python 兼容", + "status": "pass", + "detail": "Python 3.11; 232 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "evidence": "reports/python_compatibility.json", + "link": "python_compatibility.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" + }, + { + "key": "architecture-maintainability", + "label": "架构维护", + "status": "pass", + "detail": "229 Python files; 0 hotspots; 0 watchlist files; 6 early watch files; 0 blockers; largest 719 lines; 68 CLI handlers; 18 in entrypoint", + "evidence": "reports/architecture_maintainability.json", + "link": "architecture_maintainability.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "permission-gates", @@ -69,47 +142,104 @@ "status": "pass", "detail": "3/3 permissions approved; gaps 0; required file_write, network, subprocess", "evidence": "reports/security_trust_report.json + security/permission_policy.json", - "link": "../security/permission_policy.md" + "link": "../security/permission_policy.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "permission-runtime", "label": "权限探针", "status": "pass", - "detail": "3/3 targets probed; native 0; metadata fallback 3; residual risks 3", + "detail": "4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4", "evidence": "reports/runtime_permission_probes.json", - "link": "runtime_permission_probes.md" + "link": "runtime_permission_probes.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "skill-atlas", "label": "组合治理", "status": "pass", - "detail": "12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues", + "detail": "12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues", "evidence": "reports/skill_atlas.json", - "link": "skill_atlas.html" + "link": "skill_atlas.html", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low", - "evidence": "reports/adoption_drift_report.json", - "link": "adoption_drift_report.md" + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false", + "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", + "link": "adoption_drift_report.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "review-waivers", "label": "人工批准", - "status": "pass", - "detail": "0 active waivers cover current warnings", + "status": "warn", + "detail": "0 active waivers; 1 warning gates still need reviewer decision", "evidence": "reports/review_waivers.json", - "link": "review_waivers.md" + "link": "review_waivers.md", + "review_action_id": "review-action-review-waivers", + "review_action_status": "warn", + "review_action_title": "人工批准", + "review_action_next_step": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "review_action_reason": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "review_action_source_ref_count": 2, + "review_action_verification_command": "python3 scripts/render_review_waivers.py ." + }, + { + "key": "world-class-evidence", + "label": "世界证据", + "status": "warn", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true", + "evidence": "reports/world_class_evidence_ledger.json", + "link": "world_class_evidence_ledger.md", + "review_action_id": "review-action-world-class-evidence", + "review_action_status": "warn", + "review_action_title": "世界证据", + "review_action_next_step": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", + "review_action_reason": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", + "review_action_source_ref_count": 16, + "review_action_verification_command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio ." }, { "key": "registry-audit", "label": "注册审计", "status": "pass", - "detail": "yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters", + "detail": "yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures", "evidence": "reports/registry_audit.json + reports/install_simulation.json", - "link": "registry_audit.md" + "link": "registry_audit.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" }, { "key": "release-notes", @@ -117,21 +247,1192 @@ "status": "pass", "detail": "0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended", "evidence": "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", - "link": "promotion_decisions.md" + "link": "promotion_decisions.md", + "review_action_id": "", + "review_action_status": "", + "review_action_title": "", + "review_action_next_step": "", + "review_action_reason": "", + "review_action_source_ref_count": 0, + "review_action_verification_command": "" } ], + "gate_contract": { + "ok": true, + "expected_gate_count": 16, + "actual_gate_count": 16, + "expected_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "rendered_gate_keys": [ + "intent-canvas", + "trigger-lab", + "output-lab", + "context-budget", + "runtime-matrix", + "trust-report", + "python-compat", + "architecture-maintainability", + "permission-gates", + "permission-runtime", + "skill-atlas", + "operations-loop", + "review-waivers", + "world-class-evidence", + "registry-audit", + "release-notes" + ], + "missing_gate_keys": [], + "unknown_gate_keys": [], + "duplicate_gate_keys": [], + "unweighted_gate_keys": [] + }, "blockers": [], - "warnings": [], - "review_actions": [], + "warnings": [ + { + "key": "output-lab", + "label": "输出实验", + "status": "warn", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5", + "evidence": "reports/output_quality_scorecard.json", + "link": "output_quality_scorecard.md", + "review_action_id": "review-action-output-lab", + "review_action_status": "warn", + "review_action_title": "输出实验", + "review_action_next_step": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。", + "review_action_reason": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "review_action_source_ref_count": 6, + "review_action_verification_command": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review" + }, + { + "key": "review-waivers", + "label": "人工批准", + "status": "warn", + "detail": "0 active waivers; 1 warning gates still need reviewer decision", + "evidence": "reports/review_waivers.json", + "link": "review_waivers.md", + "review_action_id": "review-action-review-waivers", + "review_action_status": "warn", + "review_action_title": "人工批准", + "review_action_next_step": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "review_action_reason": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "review_action_source_ref_count": 2, + "review_action_verification_command": "python3 scripts/render_review_waivers.py ." + }, + { + "key": "world-class-evidence", + "label": "世界证据", + "status": "warn", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true", + "evidence": "reports/world_class_evidence_ledger.json", + "link": "world_class_evidence_ledger.md", + "review_action_id": "review-action-world-class-evidence", + "review_action_status": "warn", + "review_action_title": "世界证据", + "review_action_next_step": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", + "review_action_reason": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", + "review_action_source_ref_count": 16, + "review_action_verification_command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio ." + } + ], + "review_actions": [ + { + "action_id": "review-action-output-lab", + "gate_key": "output-lab", + "title": "输出实验", + "label": "输出实验", + "status": "warn", + "priority": "warning", + "next_step": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。", + "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。", + "reason": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json", + "source_refs": [ + { + "path": "evals/output/cases.jsonl", + "label": "output eval cases", + "kind": "eval", + "line": 1, + "matched_pattern": "\"id\"", + "excerpt": "{\"id\":\"skill-package-contract\",\"prompt\":\"Turn this repeated workflow into a reusable team skill package.\",\"baseline_output\":\"I can write a prompt for that workflow and include a s…", + "exists": true, + "link": "../evals/output/cases.jsonl" + }, + { + "path": "reports/output_quality_scorecard.md", + "label": "output scorecard", + "kind": "report", + "line": 1, + "matched_pattern": "# Output", + "excerpt": "# Output Quality Scorecard", + "exists": true, + "link": "output_quality_scorecard.md" + }, + { + "path": "reports/output_execution_runs.md", + "label": "output execution runs", + "kind": "report", + "line": 1, + "matched_pattern": "# Output Execution", + "excerpt": "# Output Execution Runs", + "exists": true, + "link": "output_execution_runs.md" + }, + { + "path": "reports/output_blind_review_pack.md", + "label": "blind A/B review pack", + "kind": "report", + "line": 1, + "matched_pattern": "# Output Blind", + "excerpt": "# Output Blind A/B Review Pack", + "exists": true, + "link": "output_blind_review_pack.md" + }, + { + "path": "reports/output_review_kit.html", + "label": "reviewer cockpit", + "kind": "report", + "line": 6, + "matched_pattern": "Output Review Kit", + "excerpt": "Output Review Kit", + "exists": true, + "link": "output_review_kit.html" + }, + { + "path": "reports/output_review_adjudication.md", + "label": "review adjudication", + "kind": "report", + "line": 1, + "matched_pattern": "# Output Review", + "excerpt": "# Output Review Adjudication", + "exists": true, + "link": "output_review_adjudication.md" + } + ], + "evidence": "reports/output_quality_scorecard.json", + "evidence_link": "output_quality_scorecard.md", + "verification_command": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review", + "evidence_steps": [] + }, + { + "action_id": "review-action-review-waivers", + "gate_key": "review-waivers", + "title": "人工批准", + "label": "人工批准", + "status": "warn", + "priority": "warning", + "next_step": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "reason": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "why": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "source_fix": "reports/review_waivers.md", + "source_refs": [ + { + "path": "reports/review_waivers.md", + "label": "waiver ledger", + "kind": "report", + "line": 1, + "matched_pattern": "# Review", + "excerpt": "# Review Waivers", + "exists": true, + "link": "review_waivers.md" + }, + { + "path": "references/review-waiver-method.md", + "label": "waiver method", + "kind": "method", + "line": 1, + "matched_pattern": "# Review", + "excerpt": "# Review Waiver Method", + "exists": true, + "link": "../references/review-waiver-method.md" + } + ], + "evidence": "reports/review_waivers.json", + "evidence_link": "review_waivers.md", + "verification_command": "python3 scripts/render_review_waivers.py .", + "evidence_steps": [] + }, + { + "action_id": "review-action-world-class-evidence", + "gate_key": "world-class-evidence", + "title": "世界证据", + "label": "世界证据", + "status": "warn", + "priority": "warning", + "next_step": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", + "summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", + "reason": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", + "why": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", + "source_fix": "reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", + "source_refs": [ + { + "path": "reports/world_class_evidence_ledger.md", + "label": "world-class evidence ledger", + "kind": "report", + "line": 1, + "matched_pattern": "# World-Class Evidence Ledger", + "excerpt": "# World-Class Evidence Ledger", + "exists": true, + "link": "world_class_evidence_ledger.md" + }, + { + "path": "reports/world_class_evidence_plan.md", + "label": "world-class evidence plan", + "kind": "report", + "line": 1, + "matched_pattern": "# World-Class Evidence Plan", + "excerpt": "# World-Class Evidence Plan", + "exists": true, + "link": "world_class_evidence_plan.md" + }, + { + "path": "reports/world_class_evidence_intake.md", + "label": "world-class evidence intake", + "kind": "report", + "line": 1, + "matched_pattern": "# World-Class Evidence Intake", + "excerpt": "# World-Class Evidence Intake", + "exists": true, + "link": "world_class_evidence_intake.md" + }, + { + "path": "reports/world_class_evidence_preflight.md", + "label": "world-class evidence preflight", + "kind": "report", + "line": 1, + "matched_pattern": "# World-Class Evidence Preflight", + "excerpt": "# World-Class Evidence Preflight", + "exists": true, + "link": "world_class_evidence_preflight.md" + }, + { + "path": "reports/world_class_evidence_preflight.html", + "label": "world-class preflight HTML", + "kind": "report", + "line": 6, + "matched_pattern": "World-Class Evidence Preflight", + "excerpt": "World-Class Evidence Preflight", + "exists": true, + "link": "world_class_evidence_preflight.html" + }, + { + "path": "reports/world_class_submission_review.md", + "label": "world-class submission review", + "kind": "report", + "line": 1, + "matched_pattern": "# World-Class Submission Review", + "excerpt": "# World-Class Submission Review", + "exists": true, + "link": "world_class_submission_review.md" + }, + { + "path": "reports/world_class_claim_guard.md", + "label": "world-class claim guard", + "kind": "report", + "line": 1, + "matched_pattern": "# World-Class Claim Guard", + "excerpt": "# World-Class Claim Guard", + "exists": true, + "link": "world_class_claim_guard.md" + }, + { + "path": "evidence/world_class/intake.schema.json", + "label": "evidence intake schema", + "kind": "schema", + "line": 4, + "matched_pattern": "Yao World-Class Evidence Intake", + "excerpt": "\"title\": \"Yao World-Class Evidence Intake\",", + "exists": true, + "link": "../evidence/world_class/intake.schema.json" + }, + { + "path": "evidence/world_class/templates/provider-holdout.intake.json", + "label": "provider intake template", + "kind": "template", + "line": 3, + "matched_pattern": "provider-holdout", + "excerpt": "\"evidence_key\": \"provider-holdout\",", + "exists": true, + "link": "../evidence/world_class/templates/provider-holdout.intake.json" + }, + { + "path": "evidence/world_class/templates/human-adjudication.intake.json", + "label": "human intake template", + "kind": "template", + "line": 3, + "matched_pattern": "human-adjudication", + "excerpt": "\"evidence_key\": \"human-adjudication\",", + "exists": true, + "link": "../evidence/world_class/templates/human-adjudication.intake.json" + }, + { + "path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "label": "permission intake template", + "kind": "template", + "line": 3, + "matched_pattern": "native-permission-enforcement", + "excerpt": "\"evidence_key\": \"native-permission-enforcement\",", + "exists": true, + "link": "../evidence/world_class/templates/native-permission-enforcement.intake.json" + }, + { + "path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "label": "telemetry intake template", + "kind": "template", + "line": 3, + "matched_pattern": "native-client-telemetry", + "excerpt": "\"evidence_key\": \"native-client-telemetry\",", + "exists": true, + "link": "../evidence/world_class/templates/native-client-telemetry.intake.json" + }, + { + "path": "reports/skill_os2_audit.md", + "label": "Skill OS 2.0 audit", + "kind": "report", + "line": 1, + "matched_pattern": "# Skill OS", + "excerpt": "# Skill OS 2.0 Audit", + "exists": true, + "link": "skill_os2_audit.md" + }, + { + "path": "reports/output_review_decisions.json", + "label": "human review decisions", + "kind": "report", + "line": 30, + "matched_pattern": "winner_variant", + "excerpt": "\"winner_variant\": \"Use A or B after reading the blind review pack. Leave blank when pending.\",", + "exists": true, + "link": "output_review_decisions.json" + }, + { + "path": "reports/runtime_permission_probes.md", + "label": "runtime permission probes", + "kind": "report", + "line": 1, + "matched_pattern": "# Runtime", + "excerpt": "# Runtime Permission Probes", + "exists": true, + "link": "runtime_permission_probes.md" + }, + { + "path": "reports/adoption_drift_report.md", + "label": "adoption drift", + "kind": "report", + "line": 1, + "matched_pattern": "# Adoption", + "excerpt": "# Adoption And Drift Report", + "exists": true, + "link": "adoption_drift_report.md" + } + ], + "evidence": "reports/world_class_evidence_ledger.json", + "evidence_link": "world_class_evidence_ledger.md", + "verification_command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .", + "evidence_steps": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "status": "pending", + "readiness": "awaiting-submission", + "current": "model-executed 10; token-observed 10", + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "source_pass_count": 3, + "source_blocked_count": 0, + "blocked_checks": [], + "repair_rows": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "repair_blocked_count": 1, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "phase_queue_blocked_count": 1, + "phase_queue_counts_as_completion": false, + "commands": [ + { + "key": "prepare_submission", + "label": "准备提交", + "command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions" + }, + { + "key": "validate_intake", + "label": "校验入口", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "submission_review", + "label": "审查提交", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "refresh_ledger", + "label": "刷新台账", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "guard_claim", + "label": "声明守卫", + "command": "python3 scripts/yao.py world-class-claim-guard ." + } + ], + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json" + ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 1, + "submission_ref_total_count": 1, + "supporting_evidence_ready_count": 6, + "supporting_evidence_total_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 1, + "total_count": 1, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 6, + "total_count": 6, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "status": "pending", + "readiness": "awaiting-submission", + "current": "0/5 decisions; pending 5", + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "source_pass_count": 5, + "source_blocked_count": 5, + "blocked_checks": [ + { + "label": "No pending decisions", + "field": "pending_count", + "actual": 5, + "expected": "==0", + "status": "blocked", + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "label": "Judgments complete", + "field": "judgment_count", + "actual": 0, + "expected": "==pair_count", + "status": "blocked", + "next_action": "Every pair needs one valid human judgment." + }, + { + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "actual": false, + "expected": "true", + "status": "blocked", + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "label": "Blind review attested", + "field": "blind_review_attested", + "actual": false, + "expected": "true", + "status": "blocked", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "actual": false, + "expected": "true", + "status": "blocked", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "repair_rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "repair_blocked_count": 6, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "commands": [ + { + "key": "prepare_submission", + "label": "准备提交", + "command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions" + }, + { + "key": "validate_intake", + "label": "校验入口", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "submission_review", + "label": "审查提交", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "refresh_ledger", + "label": "刷新台账", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "guard_claim", + "label": "声明守卫", + "command": "python3 scripts/yao.py world-class-claim-guard ." + } + ], + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md" + ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 2, + "submission_ref_total_count": 2, + "supporting_evidence_ready_count": 8, + "supporting_evidence_total_count": 8, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 2, + "total_count": 2, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 8, + "total_count": 8, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text." + ] + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "status": "pending", + "readiness": "awaiting-submission", + "current": "native-enforced targets 0; installer-enforced targets 4", + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "source_pass_count": 2, + "source_blocked_count": 1, + "blocked_checks": [ + { + "label": "Native enforcement", + "field": "native_enforcement_count", + "actual": 0, + "expected": ">0", + "status": "blocked", + "next_action": "Collect real target-client or external runtime guard proof." + } + ], + "repair_rows": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "repair_blocked_count": 2, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "commands": [ + { + "key": "prepare_submission", + "label": "准备提交", + "command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions" + }, + { + "key": "validate_intake", + "label": "校验入口", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "submission_review", + "label": "审查提交", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "refresh_ledger", + "label": "刷新台账", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "guard_claim", + "label": "声明守卫", + "command": "python3 scripts/yao.py world-class-claim-guard ." + } + ], + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md" + ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 2, + "submission_ref_total_count": 2, + "supporting_evidence_ready_count": 11, + "supporting_evidence_total_count": 11, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 2, + "total_count": 2, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 11, + "total_count": 11, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "status": "pending", + "readiness": "awaiting-submission", + "current": "external source events 0; adoption samples 0", + "next_action": "Install a real client against the native host and import production metadata-only events.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "source_pass_count": 1, + "source_blocked_count": 2, + "blocked_checks": [ + { + "label": "External events", + "field": "external_source_events", + "actual": 0, + "expected": ">0", + "status": "blocked", + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "label": "Adoption sample", + "field": "adoption_sample_count", + "actual": 0, + "expected": ">0", + "status": "blocked", + "next_action": "Telemetry must include adoption outcome evidence." + } + ], + "repair_rows": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "repair_blocked_count": 3, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "commands": [ + { + "key": "prepare_submission", + "label": "准备提交", + "command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions" + }, + { + "key": "validate_intake", + "label": "校验入口", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "submission_review", + "label": "审查提交", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "refresh_ledger", + "label": "刷新台账", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" + }, + { + "key": "guard_claim", + "label": "声明守卫", + "command": "python3 scripts/yao.py world-class-claim-guard ." + } + ], + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json" + ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 2, + "submission_ref_total_count": 2, + "supporting_evidence_ready_count": 6, + "supporting_evidence_total_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 2, + "total_count": 2, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 6, + "total_count": 6, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + } + ] + } + ], "evidence_paths": { "skill_overview": "reports/skill-overview.html", "review_viewer": "reports/review-viewer.html", "output_eval": "reports/output_quality_scorecard.md", "output_execution": "reports/output_execution_runs.md", "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_kit": "reports/output_review_kit.md", + "output_review_kit_html": "reports/output_review_kit.html", + "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", + "benchmark_reproducibility": "reports/benchmark_reproducibility.md", + "skill_os2_coverage": "reports/skill_os2_coverage.md", "runtime_conformance": "reports/conformance_matrix.md", "trust_report": "reports/security_trust_report.md", + "python_compatibility": "reports/python_compatibility.md", + "architecture_maintainability": "reports/architecture_maintainability.md", "permission_policy": "security/permission_policy.md", "runtime_permissions": "reports/runtime_permission_probes.md", "skill_atlas": "reports/skill_atlas.html", @@ -139,11 +1440,26 @@ "adoption_drift": "reports/adoption_drift_report.md", "review_waivers": "reports/review_waivers.md", "review_annotations": "reports/review_annotations.md", + "adaptation_proposals": "reports/adaptation_proposals.md", + "adaptation_approval_ledger": "reports/adaptation_approval_ledger.json", + "adaptation_regression": "reports/adaptation_regression_report.md", + "world_class_evidence_plan": "reports/world_class_evidence_plan.md", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", + "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_evidence_preflight": "reports/world_class_evidence_preflight.md", + "world_class_evidence_preflight_html": "reports/world_class_evidence_preflight.html", + "world_class_submission_review": "reports/world_class_submission_review.md", + "world_class_operator_runbook": "reports/world_class_operator_runbook.md", + "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", + "world_class_claim_guard": "reports/world_class_claim_guard.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", "install_simulation": "reports/install_simulation.md", "upgrade_check": "reports/upgrade_check.md", - "migration": "docs/migration-v2.md" + "migration": "docs/migration-v2.md", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "daily_skillops": "reports/skillops/daily/2026-06-17.md", + "weekly_curator": "reports/skillops/weekly/2026-W25.md" }, "data": { "overview": { @@ -151,7 +1467,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -159,13 +1475,15 @@ "deliverables": [ "SKILL.md", "agents/interface.yaml", - "reports/skill-ir.json", + "skill-ir/examples/yao-meta-skill.json", "reports/compiled_targets.md", "reports/output_quality_scorecard.md", "reports/output_execution_runs.md", "reports/output_blind_review_pack.md", - "reports/output_blind_answer_key.json", + "reports/output_review_kit.md", "reports/output_review_adjudication.md", + "reports/benchmark_reproducibility.md", + "reports/output_blind_answer_key.json", "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/runtime_permission_probes.md", @@ -176,8 +1494,11 @@ "reports/upgrade_check.md", "reports/adoption_drift_report.md", "reports/review_waivers.md", + "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_ledger.md", "reports/review_annotations.md", "reports/review-studio.html", + "reports/skill-interpretation.html", "reports/skill-overview.html" ], "flow": [ @@ -223,7 +1544,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 365 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -244,7 +1565,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 365 个词/字，references 约 14247 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -298,7 +1619,7 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -326,7 +1647,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -348,7 +1669,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -358,10 +1679,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -413,7 +1734,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 365 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -545,6 +1866,78 @@ "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." ] }, + "world_class_readiness": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 19, + "source_pass_count": 11, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete", + "Reviewer metadata" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, "package_assets": { "entries": [ { @@ -575,28 +1968,28 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 32 + "file_count": 35 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 66 + "file_count": 153 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 28 + "file_count": 29 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 170 + "file_count": 235 } ], - "file_count": 300, + "file_count": 456, "folder_count": 4, "distribution": [ { @@ -617,24 +2010,35 @@ }, { "label": "references", - "value": 32 + "value": 35 }, { "label": "scripts", - "value": 66 + "value": 153 }, { "label": "evals", - "value": 28 + "value": 29 }, { "label": "reports", - "value": 170 + "value": 235 } ] }, "iteration_roadmap": { "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, { "title": "Borrow one proven pattern on purpose", "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", @@ -654,16 +2058,6 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift." - }, - { - "title": "Create an iteration evidence loop", - "why": "The package should show what changed and why after the first draft.", - "actions": [ - "Generate the HTML skill report and keep it aligned with the package.", - "Record reference scan choices and non-goals.", - "Capture the next iteration choice explicitly before adding more files." - ], - "unlocks": "A clearer path for the next author or reviewer." } ] }, @@ -703,17 +2097,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -744,25 +2138,25 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 32 + "file_count": 35 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 66 + "file_count": 153 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 28 + "file_count": 29 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 170 + "file_count": 235 } ], "strengths": [ @@ -780,7 +2174,7 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -824,6 +2218,17 @@ } ], "roadmap": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, { "title": "Borrow one proven pattern on purpose", "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", @@ -843,16 +2248,6 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift." - }, - { - "title": "Create an iteration evidence loop", - "why": "The package should show what changed and why after the first draft.", - "actions": [ - "Generate the HTML skill report and keep it aligned with the package.", - "Record reference scan choices and non-goals.", - "Capture the next iteration choice explicitly before adding more files." - ], - "unlocks": "A clearer path for the next author or reviewer." } ], "cards": [], @@ -864,7 +2259,8 @@ "benchmark_highlights": [], "skill_ir": { "schema_version": "2.0.0", - "target_count": 4, + "source_path": "skill-ir/examples/yao-meta-skill.json", + "target_count": 5, "trigger_samples": 8, "output_eval_cases": 3 }, @@ -872,8 +2268,8 @@ "ok": true, "schema_version": "1.0", "summary": { - "target_count": 4, - "pass_count": 4, + "target_count": 5, + "pass_count": 5, "warn_count": 0, "block_count": 0, "failure_count": 0, @@ -938,6 +2334,22 @@ ], "unsupported_features": [], "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] } ], "failures": [], @@ -962,14 +2374,14 @@ "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -978,9 +2390,19 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "failures": [] }, @@ -994,6 +2416,31 @@ "pair_count": 5, "answer_key_separate": true }, + "output_review_kit": { + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "failures": [] + }, "output_review_adjudication": { "ok": true, "summary": { @@ -1003,14 +2450,77 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, - "failure_count": 0 + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "failure_count": 0, + "reviewer_checklist_count": 5, + "reviewer_checklist_pending_count": 5, + "reviewer_checklist_invalid_count": 0, + "reviewer_checklist_ready_count": 0 }, "reviewer": "", "reviewed_at": "", "failures": [] }, + "benchmark_reproducibility": { + "ok": true, + "summary": { + "reproducibility_ready": true, + "release_lock_ready": true, + "methodology_complete": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "evidence_bundle_sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0", + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 23, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 10, + "token_observed_count": 10, + "human_review_complete": false, + "provider_evidence_complete": true, + "world_class_ready": false, + "world_class_open_gap_count": 3, + "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3, + "working_tree_dirty": true, + "changed_file_count": 29, + "source_tree_dirty": false, + "source_changed_file_count": 0, + "generated_tree_dirty": true, + "generated_changed_file_count": 29 + }, + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "missing_artifacts": [], + "limitations": [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims." + ] + }, "runtime_conformance": { "target_count": 5, "pass_count": 5, @@ -1019,14 +2529,22 @@ "runtime_permissions": { "ok": true, "summary": { - "target_count": 3, - "pass_count": 3, + "target_count": 4, + "pass_count": 4, "fail_count": 0, "native_enforcement_count": 0, - "metadata_fallback_count": 3, - "residual_risk_count": 3, + "metadata_fallback_count": 4, + "residual_risk_count": 4, "required_capability_count": 3, - "failure_count": 0 + "failure_count": 0, + "installer_enforcement_source_status": "present", + "installer_enforcement_target_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "installer_permission_capability_count": 3, + "world_class_native_evidence_ready": false, + "installer_enforcement_ready": true }, "expected_capabilities": [ "file_write", @@ -1063,33 +2581,43 @@ "residual_risks": [ "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." ] + }, + { + "target": "vscode", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] } ], "failures": [] }, "trust_security": { - "scanned_files": 150, - "script_count": 66, - "internal_module_count": 7, + "scanned_files": 245, + "script_count": 153, + "internal_module_count": 67, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], - "network_script_count": 2, - "network_policy_covered_count": 2, + "network_script_count": 3, + "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 47, + "file_write_script_count": 73, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 59, + "help_smoke_checked_count": 86, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 150, - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_hash_file_count": 245, + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" }, "skill_atlas": { "skill_count": 12, @@ -1102,6 +2630,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "registry_distribution": { @@ -1110,12 +2641,13 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "maturity": "governed", "owner": "Yao Team", @@ -1123,8 +2655,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f", - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { "openai": "pass", @@ -1155,7 +2687,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1168,11 +2700,11 @@ "package_verification": { "ok": true, "summary": { - "target_count": 3, - "adapter_count": 3, + "target_count": 4, + "adapter_count": 4, "archive_present": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", - "archive_entry_count": 488, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "archive_entry_count": 689, "failure_count": 0, "warning_count": 0 }, @@ -1183,12 +2715,16 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 488, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, "interface_loaded": true, - "adapter_count": 3, + "adapter_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, "install_root_is_temp": false, "failure_count": 0, "warning_count": 0 @@ -1202,7 +2738,7 @@ "declared_bump": "minor", "recommended_bump": "minor", "breaking_change_count": 0, - "added_target_count": 1, + "added_target_count": 2, "metadata_change_count": 1, "checksum_change_count": 2, "failure_count": 0, @@ -1211,7 +2747,8 @@ "upgrade_diff": { "name_changed": false, "added_targets": [ - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "removed_targets": [], "compatibility_changes": [ @@ -1238,25 +2775,25 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ { "field": "archive_sha256", "from": "", - "to": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "to": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "to": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] }, "release_notes": [ "Recommended version bump: minor.", - "Added targets: agent-skills-compatible.", + "Added targets: agent-skills-compatible, vscode.", "Package or archive checksum changed; reviewers should verify package artifacts before release.", "Package verification evidence: reports/package_verification.md." ], @@ -1267,13 +2804,14 @@ "ok": true, "summary": { "event_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1282,9 +2820,13 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, - "failure_types": {} + "failure_types": {}, + "source_types": { + "manual": 1 + }, + "command_counts": {} }, "next_iteration_candidates": [], "privacy_contract": { @@ -1317,25 +2859,68 @@ "expired_count": 0, "invalid_count": 0, "covered_gate_count": 0, - "covered_gate_keys": [] + "covered_gate_keys": [], + "waiver_candidate_count": 2, + "waiverable_open_count": 1, + "non_waivable_count": 1 }, "policy": { "blocker_waivers_allowed": false, "minimum_reason_chars": 20, "expires_required": true, - "known_gate_keys": [ + "review_studio_gate_keys": [ + "architecture-maintainability", "context-budget", "intent-canvas", "operations-loop", "output-lab", "permission-gates", "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "known_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", "registry-audit", "release-notes", "runtime-matrix", "skill-atlas", "trigger-lab", "trust-report" + ], + "waiverable_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "non_waivable_gate_keys": [ + "review-waivers", + "world-class-evidence" ] }, "failures": [], @@ -1357,6 +2942,743 @@ "annotations": [], "failures": [] }, + "world_class_evidence_plan": { + "ok": true, + "summary": { + "audit_decision": "continue-iteration", + "world_class_ready": false, + "audit_world_class_ready": false, + "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, + "task_count": 4, + "human_task_count": 1, + "external_task_count": 3, + "review_task_count": 0, + "decision": "collect-external-evidence" + }, + "tasks": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_audit": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "open_gap_count": 3 + } + }, + "world_class_evidence_ledger": { + "ok": true, + "summary": { + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "source_accepted": false, + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4, + "evidence_requirement_count": 4 + } + }, "synthesis_highlights": [ "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", "Borrow a review checkpoint wherever trust matters more than raw speed.", @@ -1365,10 +3687,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -1397,7 +3719,8 @@ "targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "maturity_tier": "governed", "skill_archetype": "governed", @@ -1553,7 +3876,7 @@ "correct_count": 3, "precision": 1.0, "recall": 1.0, - "average_margin": 0.611 + "average_margin": 0.701 }, "team-frontend-review": { "expected_count": 3, @@ -1569,7 +3892,7 @@ "correct_count": 3, "precision": 1.0, "recall": 1.0, - "average_margin": 0.659 + "average_margin": 0.586 }, "no_route": { "expected_count": 4, @@ -1615,17 +3938,16 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.741, + "margin": 1.0, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.741, + "score": 1.0, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" @@ -1656,17 +3978,16 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.535, + "margin": 0.661, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.535, + "score": 0.661, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package" ], "matched_negative_concepts": [] @@ -1695,23 +4016,22 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.558, + "margin": 0.441, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.558, + "score": 0.441, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ - "build_skill", "reuse_package" ], "matched_negative_concepts": [] }, { "name": "governed-incident-command", - "score": 0.269, + "score": 0.181, "passed_threshold": false, "threshold": 0.48, "matched_desired_concepts": [ @@ -1754,7 +4074,7 @@ }, { "name": "yao-meta-skill", - "score": 0.018, + "score": 0.012, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -1794,7 +4114,7 @@ }, { "name": "yao-meta-skill", - "score": 0.013, + "score": 0.008, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -1867,13 +4187,14 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], "matched_negative_concepts": [] }, { "name": "yao-meta-skill", - "score": 0.021, + "score": 0.016, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -1895,12 +4216,12 @@ "expected_route": "governed-incident-command", "predicted_route": "governed-incident-command", "correct": true, - "margin": 0.805, + "margin": 0.505, "ambiguous": false, "ranked_routes": [ { "name": "governed-incident-command", - "score": 0.805, + "score": 0.505, "passed_threshold": true, "threshold": 0.48, "matched_desired_concepts": [ @@ -1911,7 +4232,7 @@ }, { "name": "yao-meta-skill", - "score": 0.189, + "score": 0.328, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [ @@ -1935,23 +4256,25 @@ "expected_route": "governed-incident-command", "predicted_route": "governed-incident-command", "correct": true, - "margin": 0.172, + "margin": 0.252, "ambiguous": false, "ranked_routes": [ { "name": "governed-incident-command", - "score": 0.62, + "score": 1.0, "passed_threshold": true, "threshold": 0.48, "matched_desired_concepts": [ "comms_and_owners", - "governed_workflow" + "governed_workflow", + "incident_packet", + "severity_assessment" ], "matched_negative_concepts": [] }, { "name": "yao-meta-skill", - "score": 0.448, + "score": 0.748, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ @@ -2021,7 +4344,7 @@ "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.016, + "score": 0.008, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -2950,22 +5273,32 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "summary": { "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -2975,17 +5308,17 @@ "case_id": "skill-package-contract", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.11, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 5980.01, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 16, - "output_tokens": 17, - "total_tokens": 33, - "estimated": true + "input_tokens": 100, + "output_tokens": 384, + "total_tokens": 484, + "estimated": false }, "score": 0.0, "passed_count": 0, @@ -2996,47 +5329,49 @@ "has-report", "has-resource-boundary" ], - "output_sha256": "ece26a6edbbbea86a93faaf9f0eadf88efbae12f1c3ad81da9376cf2a889d101", + "output_sha256": "1aa12d65d55863581eef67aaa173b3a9962524a91533693044cb340a342b760b", "failure": "" }, { "case_id": "skill-package-contract", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.56, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 11217.19, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 16, - "output_tokens": 57, - "total_tokens": 73, - "estimated": true + "input_tokens": 850, + "output_tokens": 998, + "total_tokens": 1848, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "8bee0131e17fdbe87e48457130ac6aeaa55ea30c919a45e77109852609f33eba", + "score": 75.0, + "passed_count": 3, + "failed_count": 1, + "failed_assertions": [ + "has-report" + ], + "output_sha256": "ac2cb822520b3099150ea1df07de59b9c6df382b7251792e0662631a60989d89", "failure": "" }, { "case_id": "output-eval-expectation", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 24.17, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 3178.95, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 24, - "output_tokens": 12, - "total_tokens": 36, - "estimated": true + "input_tokens": 107, + "output_tokens": 129, + "total_tokens": 236, + "estimated": false }, "score": 0.0, "passed_count": 0, @@ -3047,183 +5382,192 @@ "has-scorecard", "has-failure-taxonomy" ], - "output_sha256": "98fa332fb493b5dc14656a9d65a6c0a9c25ff61a14b71b647aad89cbdbd3ab35", + "output_sha256": "a128ff4dbd7870e287ac651b6cab6b7887e347d6477e5bd8c6f40cd9479539f0", "failure": "" }, { "case_id": "output-eval-expectation", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.84, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4611.78, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 24, - "output_tokens": 56, - "total_tokens": 80, - "estimated": true + "input_tokens": 857, + "output_tokens": 264, + "total_tokens": 1121, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "8efdbfefa6a36603682b7063b1753f9f939b3810c082d29686e297906d9f51c8", + "score": 25.0, + "passed_count": 1, + "failed_count": 3, + "failed_assertions": [ + "has-baseline-comparison", + "has-assertion-grading", + "has-failure-taxonomy" + ], + "output_sha256": "4df98d20e2199e56dae2618b6a64e59d5c96484a4f71ae0ebc674b755cf533c3", "failure": "" }, { "case_id": "ir-before-packaging", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 22.51, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 6772.59, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 18, - "output_tokens": 15, - "total_tokens": 33, - "estimated": true + "input_tokens": 104, + "output_tokens": 661, + "total_tokens": 765, + "estimated": false }, - "score": 0.0, - "passed_count": 0, - "failed_count": 4, + "score": 25.0, + "passed_count": 1, + "failed_count": 3, "failed_assertions": [ "has-ir-first", "has-core-ir-fields", - "has-targets", "has-semantic-preservation" ], - "output_sha256": "07a4aa9ee9e48341d4b84d5cb9b6414901aa2c8841340d67447e187240c6c4c2", + "output_sha256": "1696075b4f884ff8b478786b5565aff73ad73235936eb39c2ed615c0746861d2", "failure": "" }, { "case_id": "ir-before-packaging", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 26.03, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 17471.19, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 18, - "output_tokens": 62, - "total_tokens": 80, - "estimated": true + "input_tokens": 854, + "output_tokens": 2132, + "total_tokens": 2986, + "estimated": false }, - "score": 100.0, - "passed_count": 4, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "707fde419b3d61e921561b1c4a8523e1f7b53b1acd5758a47ce21d9ed711fde0", - "failure": "" - }, - { - "case_id": "near-neighbor-boundary", - "variant": "baseline", - "status": "pass", - "execution_mode": "command", - "model_executed": false, - "command_executed": true, - "duration_ms": 28.48, - "provider": "local-output-eval-runner", - "model": "", - "usage": { - "input_tokens": 17, - "output_tokens": 19, - "total_tokens": 36, - "estimated": true - }, - "score": 0.0, - "passed_count": 0, + "score": 25.0, + "passed_count": 1, "failed_count": 3, + "failed_assertions": [ + "has-ir-first", + "has-core-ir-fields", + "has-semantic-preservation" + ], + "output_sha256": "4f87633e53f360948d6403126d49ad3dabcb1b47aff596918650719b7840abcf", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "baseline", + "status": "pass", + "execution_mode": "model", + "model_executed": true, + "command_executed": true, + "duration_ms": 3091.33, + "provider": "deepseek", + "model": "deepseek-v4-flash", + "usage": { + "input_tokens": 105, + "output_tokens": 93, + "total_tokens": 198, + "estimated": false + }, + "score": 33.33, + "passed_count": 1, + "failed_count": 2, "failed_assertions": [ "declines-skill", - "names-near-neighbor", "requires-reuse-signal" ], - "output_sha256": "648ebaa6e09486d05f33fa6639dc973b029d115158588fd192efbe15ccaaeac4", + "output_sha256": "56c8f48f3b2b9963adad3afdb1b6a2bed6b162067506edee1e0f1acd32668a91", "failure": "" }, { "case_id": "near-neighbor-boundary", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.17, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4152.85, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 48, - "total_tokens": 65, - "estimated": true + "input_tokens": 855, + "output_tokens": 160, + "total_tokens": 1015, + "estimated": false }, - "score": 100.0, - "passed_count": 3, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "17a3a5286ad573dea86e51f82c04b596babaecb92cde8879f09e7c519205e2e1", + "score": 33.33, + "passed_count": 1, + "failed_count": 2, + "failed_assertions": [ + "declines-skill", + "requires-reuse-signal" + ], + "output_sha256": "d2d85a2aae5db5c4c7e6e0db5bdd865157a90018afe302b0a703940cc84ba68c", "failure": "" }, { "case_id": "file-backed-governed-package", "variant": "baseline", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 23.94, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 4750.01, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 20, - "total_tokens": 37, - "estimated": true + "input_tokens": 200, + "output_tokens": 302, + "total_tokens": 502, + "estimated": false }, - "score": 0.0, - "passed_count": 0, - "failed_count": 5, + "score": 60.0, + "passed_count": 3, + "failed_count": 2, "failed_assertions": [ "uses-file-backed-evidence", - "has-governance", - "has-output-contract", - "has-trust-and-scorecard", - "does-not-invent-evidence" + "has-trust-and-scorecard" ], - "output_sha256": "6e5827219459dee34143e4f11663415db80c3111291f75fb6c2115c7bda4cf30", + "output_sha256": "d41880b0c2f7e60ee6acb2cb1f4acb9377fd63de72b8e026e6259a962b89af0b", "failure": "" }, { "case_id": "file-backed-governed-package", "variant": "with_skill", "status": "pass", - "execution_mode": "command", - "model_executed": false, + "execution_mode": "model", + "model_executed": true, "command_executed": true, - "duration_ms": 22.69, - "provider": "local-output-eval-runner", - "model": "", + "duration_ms": 9406.25, + "provider": "deepseek", + "model": "deepseek-v4-flash", "usage": { - "input_tokens": 17, - "output_tokens": 81, - "total_tokens": 98, - "estimated": true + "input_tokens": 950, + "output_tokens": 660, + "total_tokens": 1610, + "estimated": false }, - "score": 100.0, - "passed_count": 5, - "failed_count": 0, - "failed_assertions": [], - "output_sha256": "bcfdf036666d406abbc945fc42b93eb74d7074118893e540253c7d8715baad73", + "score": 60.0, + "passed_count": 3, + "failed_count": 2, + "failed_assertions": [ + "has-governance", + "has-output-contract" + ], + "output_sha256": "c670ae2ef81a819a9a0f1b0c046e82ba4c649f56dcbe96a45a6938c7f214fba4", "failure": "" } ], @@ -3437,6 +5781,277 @@ } ] }, + "output_review_kit": { + "schema_version": "1.0", + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "template_written": false, + "review_contract": { + "reviewer_steps": [ + "Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.", + "Record choices in reports/output_review_decisions.json without opening the answer key.", + "Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.", + "Set reviewer_attestation truthfully after choices are recorded and before adjudication.", + "Run python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication after choices are recorded.", + "Refresh python3 scripts/yao.py review-studio . before asking for release approval." + ], + "required_fields": { + "reviewer": "Human reviewer name or review group.", + "reviewed_at": "Review date or timestamp.", + "winner_variant": "A or B for every case.", + "confidence": "Optional numeric confidence from 0 to 1.", + "reason": "Required rationale based on the rubric, not on hidden labels.", + "reviewer_attestation": "Blind-review and answer-key-order attestation after decisions are complete.", + "review_integrity": "Blind pack and prompt fingerprints generated by the template." + }, + "privacy_contract": [ + "The answer key is intentionally withheld from this kit.", + "Do not inspect hidden labels or expected winners before decisions are recorded.", + "Do not paste private user data into decision reasons.", + "Pending decisions must stay pending instead of being counted as human agreement." + ] + }, + "cases": [ + { + "case_id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0 + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0 + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0 + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "skill-package-contract:A", + "output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/." + }, + "variant_b": { + "blind_id": "skill-package-contract:B", + "output": "I can write a prompt for that workflow and include a short checklist." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0 + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0 + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0 + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "output-eval-expectation:A", + "output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md." + }, + "variant_b": { + "blind_id": "output-eval-expectation:B", + "output": "Add more examples and run the trigger eval again." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0 + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0 + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0 + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "ir-before-packaging:A", + "output": "Export folders for each platform and adjust files as needed." + }, + "variant_b": { + "blind_id": "ir-before-packaging:B", + "output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0 + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0 + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "near-neighbor-boundary:A", + "output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it." + }, + "variant_b": { + "blind_id": "near-neighbor-boundary:B", + "output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + }, + { + "case_id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0 + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0 + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0 + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0 + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "file-backed-governed-package:A", + "output": "Draft a release announcement with the changelog, support notes, and owner name." + }, + "variant_b": { + "blind_id": "file-backed-governed-package:B", + "output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them." + }, + "decision_state": { + "status": "awaiting-decision", + "winner_variant_recorded": false, + "confidence_recorded": false, + "reason_recorded": false, + "blocking_reason": "Decision template exists but this row is still blank." + } + } + ], + "failures": [] + }, "output_review_adjudication": { "schema_version": "1.0", "ok": true, @@ -3447,12 +6062,50 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, - "failure_count": 0 + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "failure_count": 0, + "reviewer_checklist_count": 5, + "reviewer_checklist_pending_count": 5, + "reviewer_checklist_invalid_count": 0, + "reviewer_checklist_ready_count": 0 }, "reviewer": "", "reviewed_at": "", + "review_integrity": { + "blind_pack_sha256": "788604c67fa31178c9bba06a52731adcbee9c1e72c00767cf7950643812a6e5c", + "case_count": 5, + "case_ids": [ + "skill-package-contract", + "output-eval-expectation", + "ir-before-packaging", + "near-neighbor-boundary", + "file-backed-governed-package" + ], + "prompt_sha256_by_case": { + "skill-package-contract": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5", + "output-eval-expectation": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4", + "ir-before-packaging": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80", + "near-neighbor-boundary": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0", + "file-backed-governed-package": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d" + } + }, + "reviewer_attestation": { + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded": true, + "reviewer_reason_required": true + }, "artifacts": { "blind_pack": "reports/output_blind_review_pack.json", "answer_key": "reports/output_blind_answer_key.json", @@ -3465,59 +6118,1525 @@ { "case_id": "skill-package-contract", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Turn this repeated workflow into a reusable team skill package." + "prompt_sha256": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5" }, { "case_id": "output-eval-expectation", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline." + "prompt_sha256": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4" }, { "case_id": "ir-before-packaging", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets." + "prompt_sha256": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80" }, { "case_id": "near-neighbor-boundary", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "I only need a one-off summary of these notes, not a reusable process." + "prompt_sha256": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0" }, { "case_id": "file-backed-governed-package", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", - "prompt": "Turn the attached release brief source into a governed skill package." + "prompt_sha256": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d" + } + ], + "reviewer_checklist": [ + { + "case_id": "skill-package-contract", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "95405d805bd516d0998b69d31ece2c1a465b0aa99f04d603628a3939028133b5", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "output-eval-expectation", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "3829e97814ca269ad8e400ec984c56ddcd54b79448d3846c15e7f5ff44c2edc4", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "ir-before-packaging", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "d00333fa41f2e28dc7505f25f259aefb82e65b2df1abda2822686c9a6f07bb80", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "near-neighbor-boundary", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "2d0585e9fa6d03e28786f5e8f0c090781eb209afd344d25e2f91203c49a656d0", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] + }, + { + "case_id": "file-backed-governed-package", + "readiness": "awaiting-decision", + "blocking_reason": "Reviewer has not selected A or B yet; answer key remains hidden.", + "status": "pending", + "reviewer_winner_variant": "", + "answer_key_visible": false, + "prompt_sha256": "c79a554b573517682da48879c1ef54399841d9ebf570088a2174dc38de39cc7d", + "blind_pack_path": "reports/output_blind_review_pack.json", + "decisions_path": "reports/output_review_decisions.json", + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio ." + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded." + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide." + ] } ], "failures": [] }, + "benchmark_reproducibility": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "git_status": { + "available": true, + "dirty": true, + "changed_file_count": 29, + "generated_dirty": true, + "generated_changed_file_count": 29, + "source_dirty": false, + "source_changed_file_count": 0, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adaptation_proposals.json", + " M reports/adaptation_proposals.md", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json" + ], + "source_sample": [], + "generated_sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adaptation_proposals.json", + " M reports/adaptation_proposals.md", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json" + ], + "generated_dirty_prefixes": [ + "dist/", + "registry/index.json", + "registry/packages/", + "reports/", + "skill_atlas/", + "skill-ir/examples/" + ], + "scope": "generation-time status before this report is written" + }, + "summary": { + "reproducibility_ready": true, + "release_lock_ready": true, + "methodology_complete": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "evidence_bundle_sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0", + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 23, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 10, + "token_observed_count": 10, + "human_review_complete": false, + "provider_evidence_complete": true, + "world_class_ready": false, + "world_class_open_gap_count": 3, + "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3, + "working_tree_dirty": true, + "changed_file_count": 29, + "source_tree_dirty": false, + "source_changed_file_count": 0, + "generated_tree_dirty": true, + "generated_changed_file_count": 29 + }, + "beta_test_release": { + "ready": true, + "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "blockers": [], + "allowed_deferred_evidence": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "reason": "Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "reason": "Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "reason": "Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "reason": "Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims." + } + ], + "policy": "Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted.", + "required_wording": "Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline." + }, + "public_claim": { + "ready": false, + "scope": "public benchmark or world-class readiness claim", + "blockers": [ + "human blind-review adjudication is incomplete", + "world-class evidence is not accepted yet (3 open gaps, 4 ledger pending)", + "world-class source checks are not all accepted (11/19 pass, 8 blocked)" + ], + "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." + }, + "release_lock": { + "ready": true, + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "status_scope": "generation-time status before this report is written", + "source_changed_file_count": 0, + "generated_changed_file_count": 29, + "reason": "only generated evidence artifacts were dirty at generation time" + }, + "evidence_bundle": { + "algorithm": "sha256(path,label,exists,artifact_sha256)", + "artifact_count": 25, + "existing_count": 25, + "missing_count": 0, + "missing_paths": [], + "sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0" + }, + "methodology": { + "path": "reports/benchmark_methodology.md", + "exists": true, + "sections": [ + { + "heading": "## Benchmark Types", + "exists": true + }, + { + "heading": "## Sample Sources", + "exists": true + }, + { + "heading": "## Evaluation Dimensions", + "exists": true + }, + { + "heading": "## Weighting Rule", + "exists": true + }, + { + "heading": "## Failure Disclosure", + "exists": true + }, + { + "heading": "## Reproduction", + "exists": true + } + ], + "missing_sections": [] + }, + "artifacts_checked": [ + { + "label": "methodology", + "path": "reports/benchmark_methodology.md", + "exists": true, + "bytes": 2715, + "sha256": "57025e0123ce5d10401c5bff376d2eeeac7943c83897ae1ad3fb22cadf790f92" + }, + { + "label": "failure_disclosure", + "path": "evals/failure-cases.md", + "exists": true, + "bytes": 889, + "sha256": "28833c0d4a217d612879d193fb5de199880dd5b3093ab5757e4315600fa4fb08" + }, + { + "label": "output_cases", + "path": "evals/output/cases.jsonl", + "exists": true, + "bytes": 6555, + "sha256": "a6ae9685711620d7203b73ace4412194ae287689f945b5139ec8ad75b9eefe04" + }, + { + "label": "output_schema", + "path": "evals/output/schema.json", + "exists": true, + "bytes": 2193, + "sha256": "8ee340c95064260c5e952be614e19841ac676162c6bf01d21b107e38cb04e0b9" + }, + { + "label": "output_scorecard", + "path": "reports/output_quality_scorecard.json", + "exists": true, + "bytes": 25530, + "sha256": "0806258a8e084b27e112537faff0de64a8519ca90cfdc78b57c0e4c08a514cca" + }, + { + "label": "output_execution", + "path": "reports/output_execution_runs.json", + "exists": true, + "bytes": 8449, + "sha256": "4df66b63d2e717a559a6881b966fb1b92def7329cc89aa48ee0d8596f87402e5" + }, + { + "label": "blind_review", + "path": "reports/output_blind_review_pack.json", + "exists": true, + "bytes": 7804, + "sha256": "bbe2db8ec2776fe289cd7d6bb78d48c2b8baa106ad37f79c15e43669b10c9390" + }, + { + "label": "review_adjudication", + "path": "reports/output_review_adjudication.json", + "exists": true, + "bytes": 14084, + "sha256": "91fd88dd9b0f8876f68027a87893d1852dac59772613345bbbfabba658c9227e" + }, + { + "label": "trigger_scorecard", + "path": "reports/route_scorecard.json", + "exists": true, + "bytes": 16950, + "sha256": "a649547a7d3d5fe02262ca1b188ed4022d8f557770be425f7b7cf762aa76533a" + }, + { + "label": "runtime_conformance", + "path": "reports/conformance_matrix.json", + "exists": true, + "bytes": 10342, + "sha256": "97f9ba949c23a60b00e9ba2ff279ca03ba517845cc7c62aa8a42645d58006c7e" + }, + { + "label": "trust_report", + "path": "reports/security_trust_report.json", + "exists": true, + "bytes": 137787, + "sha256": "bbff1f1bee0ce6f7014efcd4622ba11c70aacfc81f2e69e225411df7004dd616" + }, + { + "label": "python_compatibility", + "path": "reports/python_compatibility.json", + "exists": true, + "bytes": 30127, + "sha256": "d31dc03c74f17242f4163b8cb3f23120fe3d0712edd08fa183cb94f8120d491c" + }, + { + "label": "registry_audit", + "path": "reports/registry_audit.json", + "exists": true, + "bytes": 3137, + "sha256": "1feb80c7cdc833a655cc0e44150f39f9f6d0633465abaa1ae29ebe7cb0649065" + }, + { + "label": "package_verification", + "path": "reports/package_verification.json", + "exists": true, + "bytes": 19338, + "sha256": "7aa6c25e8154d246ace49f7ab9159fa85f96d1d4ebb7e2bb03c7b223260a908b" + }, + { + "label": "install_simulation", + "path": "reports/install_simulation.json", + "exists": true, + "bytes": 8758, + "sha256": "c1f1122264b868c4998482c580ef94333575c6ff7c5c9c297188b225f69a131d" + }, + { + "label": "skill_os2_audit", + "path": "reports/skill_os2_audit.json", + "exists": true, + "bytes": 13824, + "sha256": "6362e79a8cfd3ff54b1a7ca32f37de8ed1cda500b953fdf1074928c6d2d62379" + }, + { + "label": "world_class_evidence_plan", + "path": "reports/world_class_evidence_plan.json", + "exists": true, + "bytes": 23314, + "sha256": "8eb5e1fbaa7580f107fd9920d13a13b239edb72d50b6959fc8f6dd43150f07d7" + }, + { + "label": "world_class_evidence_ledger", + "path": "reports/world_class_evidence_ledger.json", + "exists": true, + "bytes": 26277, + "sha256": "6038b6cd6e0667f25cfd69a5e40afa314d9efc3a93753a2a3a0509ae9b516a15" + }, + { + "label": "world_class_evidence_intake", + "path": "reports/world_class_evidence_intake.json", + "exists": true, + "bytes": 20922, + "sha256": "aa262da0ecee7232974ce061a51c8966c310de0b0be9f91eb82bb28bf1c7c0a8" + }, + { + "label": "world_class_evidence_preflight", + "path": "reports/world_class_evidence_preflight.json", + "exists": true, + "bytes": 94341, + "sha256": "95cb56581224aab06be84d0b338d6ba9aa4aef0b1ebd480801f1e2226b873b0d" + }, + { + "label": "world_class_submission_review", + "path": "reports/world_class_submission_review.json", + "exists": true, + "bytes": 17295, + "sha256": "12fbb84926294c4bd05af526fcbe0983db26e449d2c305006094baf27332a0ba" + }, + { + "label": "world_class_operator_runbook", + "path": "reports/world_class_operator_runbook.json", + "exists": true, + "bytes": 87257, + "sha256": "f321dc118ee931a1e1510559e5e64774168a58b0446ae62d232009eb42e3b384" + }, + { + "label": "world_class_operator_runbook_markdown", + "path": "reports/world_class_operator_runbook.md", + "exists": true, + "bytes": 25549, + "sha256": "858ee11bbacb143f648e9c00735bc248a0b9f3c9303dc9e0ad5c0a65d9d6e07f" + }, + { + "label": "world_class_operator_runbook_html", + "path": "reports/world_class_operator_runbook.html", + "exists": true, + "bytes": 34702, + "sha256": "31b24658c59ee1dee25aec378e72033b042af88dfdf40c325d361d0c9bec2ca7" + }, + { + "label": "world_class_claim_guard", + "path": "reports/world_class_claim_guard.json", + "exists": true, + "bytes": 18596, + "sha256": "abe7f7d60c0025e140373fadaefbed4063f285c140c74b9c7cfb464e354bb526" + } + ], + "missing_artifacts": [], + "reproduction_commands": [ + { + "label": "source commit", + "command": "git rev-parse HEAD", + "evidence": "git commit hash" + }, + { + "label": "trigger eval", + "command": "make eval-suite", + "evidence": "reports/eval_suite.json" + }, + { + "label": "output eval", + "command": "python3 scripts/yao.py output-eval", + "evidence": "reports/output_quality_scorecard.json" + }, + { + "label": "output execution", + "command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'", + "evidence": "reports/output_execution_runs.json" + }, + { + "label": "blind review adjudication", + "command": "python3 scripts/yao.py output-review", + "evidence": "reports/output_review_adjudication.json" + }, + { + "label": "skill ir", + "command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json", + "evidence": "skill-ir/examples/yao-meta-skill.json" + }, + { + "label": "runtime conformance", + "command": "python3 scripts/yao.py conformance .", + "evidence": "reports/conformance_matrix.json" + }, + { + "label": "trust report", + "command": "python3 scripts/yao.py trust .", + "evidence": "reports/security_trust_report.json" + }, + { + "label": "python compatibility", + "command": "python3 scripts/yao.py python-compat .", + "evidence": "reports/python_compatibility.json" + }, + { + "label": "package", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip", + "evidence": "dist/yao-meta-skill.zip" + }, + { + "label": "package verify", + "command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip", + "evidence": "reports/package_verification.json" + }, + { + "label": "install simulate", + "command": "python3 scripts/yao.py install-simulate . --package-dir dist", + "evidence": "reports/install_simulation.json" + }, + { + "label": "registry audit", + "command": "python3 scripts/yao.py registry-audit .", + "evidence": "reports/registry_audit.json" + }, + { + "label": "skill os audit", + "command": "python3 scripts/yao.py skill-os2-audit .", + "evidence": "reports/skill_os2_audit.json" + }, + { + "label": "world-class evidence plan", + "command": "python3 scripts/yao.py world-class-evidence .", + "evidence": "reports/world_class_evidence_plan.json" + }, + { + "label": "world-class evidence ledger", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_ledger.json" + }, + { + "label": "world-class evidence intake", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_intake.json" + }, + { + "label": "world-class evidence preflight", + "command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_preflight.json" + }, + { + "label": "world-class submission review", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_submission_review.json" + }, + { + "label": "world-class operator runbook", + "command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_operator_runbook.json" + }, + { + "label": "world-class claim guard", + "command": "python3 scripts/yao.py world-class-claim-guard .", + "evidence": "reports/world_class_claim_guard.json" + }, + { + "label": "evidence consistency", + "command": "python3 scripts/yao.py evidence-consistency .", + "evidence": "reports/evidence_consistency.json" + }, + { + "label": "full ci", + "command": "make ci-test", + "evidence": "CI target output" + } + ], + "failure_disclosure": { + "path": "evals/failure-cases.md", + "case_count": 3, + "policy": "Keep representative failures visible and tied to regression checks." + }, + "limitations": [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims." + ], + "artifacts": { + "json": "reports/benchmark_reproducibility.json", + "markdown": "reports/benchmark_reproducibility.md" + } + }, + "skill_os2_coverage": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "item_count": 21, + "module_count": 8, + "recommended_pr_count": 13, + "pass_count": 21, + "warn_count": 0, + "missing_count": 0, + "extension_track_count": 4, + "extension_partial_count": 0, + "extension_planned_count": 0, + "extension_covered_count": 4, + "adaptive_extension_ready": true, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_evidence_pending_count": 4, + "decision": "local-blueprint-covered-evidence-pending" + }, + "status_counts": { + "pass": 21, + "warn": 0, + "missing": 0 + }, + "extension_status_counts": { + "covered": 4 + }, + "modules": [ + { + "key": "skill-ir", + "category": "core-module", + "label": "Skill IR", + "status": "pass", + "objective": "Platform-neutral capability contract exists before platform-specific packaging.", + "current": "schema 2.0.0; targets 5", + "command": "python3 scripts/yao.py skill-ir .", + "test": "python3 tests/verify_skill_ir.py", + "evidence": [ + { + "path": "skill-ir/schema.json", + "exists": true + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "exists": true + }, + { + "path": "scripts/export_skill_ir.py", + "exists": true + }, + { + "path": "tests/verify_skill_ir.py", + "exists": true + } + ], + "next_action": "Keep all target packages compiled from IR rather than hand-maintained per target." + }, + { + "key": "output-eval-lab", + "category": "core-module", + "label": "Output Eval Lab", + "status": "pass", + "objective": "with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs.", + "current": "5 cases; delta 100.0; execution 10", + "command": "python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .", + "test": "python3 tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "evals/output/schema.json", + "exists": true + }, + { + "path": "evals/output/cases.jsonl", + "exists": true + }, + { + "path": "scripts/run_output_eval.py", + "exists": true + }, + { + "path": "scripts/run_output_execution.py", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Add more real-file and adversarial holdout cases as adoption data grows." + }, + { + "key": "runtime-conformance", + "category": "core-module", + "label": "Runtime Conformance", + "status": "pass", + "objective": "Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets.", + "current": "5/5 targets pass", + "command": "python3 scripts/yao.py conformance .", + "test": "python3 tests/verify_conformance_suite.py", + "evidence": [ + { + "path": "runtime/conformance/schema.json", + "exists": true + }, + { + "path": "scripts/run_conformance_suite.py", + "exists": true + }, + { + "path": "reports/conformance_matrix.json", + "exists": true + }, + { + "path": "tests/verify_conformance_suite.py", + "exists": true + } + ], + "next_action": "Keep conformance cases aligned with current platform metadata rules." + }, + { + "key": "trust-security", + "category": "core-module", + "label": "Trust Security", + "status": "pass", + "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", + "current": "153 scripts; secrets 0; help failures 0", + "command": "python3 scripts/yao.py trust .", + "test": "python3 tests/verify_trust_check.py", + "evidence": [ + { + "path": "scripts/trust_check.py", + "exists": true + }, + { + "path": "security/trust_policy.md", + "exists": true + }, + { + "path": "security/script_policy.md", + "exists": true + }, + { + "path": "security/permission_policy.json", + "exists": true + }, + { + "path": "reports/security_trust_report.json", + "exists": true + }, + { + "path": "tests/verify_trust_check.py", + "exists": true + } + ], + "next_action": "Keep high-permission approvals scoped, expiring, and mapped to target enforcement." + }, + { + "key": "skill-atlas", + "category": "core-module", + "label": "Skill Atlas", + "status": "pass", + "objective": "Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities.", + "current": "12 scanned skills; actionable collisions 0", + "command": "python3 scripts/yao.py skill-atlas --workspace-root .", + "test": "python3 tests/verify_skill_atlas.py", + "evidence": [ + { + "path": "scripts/build_skill_atlas.py", + "exists": true + }, + { + "path": "skill_atlas/catalog.json", + "exists": true + }, + { + "path": "skill_atlas/route_overlap_matrix.csv", + "exists": true + }, + { + "path": "skill_atlas/dependency_graph.json", + "exists": true + }, + { + "path": "reports/skill_atlas.json", + "exists": true + }, + { + "path": "tests/verify_skill_atlas.py", + "exists": true + } + ], + "next_action": "Use real telemetry to rank stale or drifting skills by impact, not only by static metadata." + }, + { + "key": "registry-distribution", + "category": "core-module", + "label": "Registry Distribution", + "status": "pass", + "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", + "current": "archive entries 689; install failures 0", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", + "test": "python3 tests/verify_registry_audit.py", + "evidence": [ + { + "path": "registry/index.schema.json", + "exists": true + }, + { + "path": "registry/package.schema.json", + "exists": true + }, + { + "path": "registry/packages/yao-meta-skill.json", + "exists": true + }, + { + "path": "scripts/registry_audit.py", + "exists": true + }, + { + "path": "reports/package_verification.json", + "exists": true + }, + { + "path": "reports/install_simulation.json", + "exists": true + }, + { + "path": "tests/verify_registry_audit.py", + "exists": true + } + ], + "next_action": "Regenerate registry metadata after package verification so source and archive checksums stay aligned." + }, + { + "key": "review-studio", + "category": "core-module", + "label": "Review Studio", + "status": "pass", + "objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.", + "current": "16 gates; decision review; warnings 3", + "command": "python3 scripts/yao.py review-studio .", + "test": "python3 tests/verify_review_studio.py", + "evidence": [ + { + "path": "scripts/render_review_studio.py", + "exists": true + }, + { + "path": "reports/review-studio.html", + "exists": true + }, + { + "path": "reports/review-studio.json", + "exists": true + }, + { + "path": "tests/verify_review_studio.py", + "exists": true + } + ], + "next_action": "Close pending human and external evidence before claiming full release readiness." + }, + { + "key": "telemetry-drift", + "category": "core-module", + "label": "Telemetry Drift", + "status": "pass", + "objective": "Real usage feedback is captured as metadata-only local-first drift signals.", + "current": "events 1; recipes 5; risk low", + "command": "python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .", + "test": "python3 tests/verify_telemetry_hooks.py", + "evidence": [ + { + "path": "scripts/emit_telemetry_event.py", + "exists": true + }, + { + "path": "scripts/import_telemetry_events.py", + "exists": true + }, + { + "path": "scripts/telemetry_native_host.py", + "exists": true + }, + { + "path": "reports/adoption_drift_report.json", + "exists": true + }, + { + "path": "reports/telemetry_hook_recipes.json", + "exists": true + }, + { + "path": "tests/verify_telemetry_hooks.py", + "exists": true + } + ], + "next_action": "Install a real client and import production metadata-only events into the local drift loop." + } + ], + "recommended_prs": [ + { + "key": "benchmark-methodology", + "category": "recommended-pr", + "label": "Benchmark Methodology", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "25 required artifacts checked", + "command": "make ci-test", + "test": "tests/verify_benchmark_reproducibility.py", + "evidence": [ + { + "path": "reports/benchmark_methodology.md", + "exists": true + }, + { + "path": "reports/benchmark_reproducibility.json", + "exists": true + }, + { + "path": "tests/verify_benchmark_reproducibility.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "output-eval-schema", + "category": "recommended-pr", + "label": "Output Eval Schema", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "5 output cases", + "command": "make ci-test", + "test": "tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "evals/output/schema.json", + "exists": true + }, + { + "path": "evals/output/cases.jsonl", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "output-eval-runner", + "category": "recommended-pr", + "label": "Output Eval Runner", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "delta 100.0", + "command": "make ci-test", + "test": "tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "scripts/run_output_eval.py", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "output-quality-scorecard", + "category": "recommended-pr", + "label": "Output Quality Scorecard", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "gate pass True", + "command": "make ci-test", + "test": "tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "reports/output_quality_scorecard.md", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "skill-ir-v0", + "category": "recommended-pr", + "label": "Skill IR V0", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "schema 2.0.0", + "command": "make ci-test", + "test": "tests/verify_skill_ir.py", + "evidence": [ + { + "path": "skill-ir/schema.json", + "exists": true + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "exists": true + }, + { + "path": "tests/verify_skill_ir.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "compiler-refactor", + "category": "recommended-pr", + "label": "Compiler Refactor", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "5/5 compiled targets", + "command": "make ci-test", + "test": "tests/verify_compile_skill.py", + "evidence": [ + { + "path": "scripts/compile_skill.py", + "exists": true + }, + { + "path": "reports/compiled_targets.json", + "exists": true + }, + { + "path": "tests/verify_compile_skill.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "agent-skills-conformance", + "category": "recommended-pr", + "label": "Agent Skills Conformance", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "agent-skills target present", + "command": "make ci-test", + "test": "tests/verify_conformance_suite.py", + "evidence": [ + { + "path": "runtime/conformance/schema.json", + "exists": true + }, + { + "path": "reports/conformance_matrix.json", + "exists": true + }, + { + "path": "tests/verify_conformance_suite.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "trust-check", + "category": "recommended-pr", + "label": "Trust Check", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "secret findings 0", + "command": "make ci-test", + "test": "tests/verify_trust_check.py", + "evidence": [ + { + "path": "scripts/trust_check.py", + "exists": true + }, + { + "path": "reports/security_trust_report.json", + "exists": true + }, + { + "path": "tests/verify_trust_check.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "skill-atlas-generator", + "category": "recommended-pr", + "label": "Skill Atlas Generator", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "12 scanned skills", + "command": "make ci-test", + "test": "tests/verify_skill_atlas.py", + "evidence": [ + { + "path": "scripts/build_skill_atlas.py", + "exists": true + }, + { + "path": "reports/skill_atlas.json", + "exists": true + }, + { + "path": "tests/verify_skill_atlas.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "registry-package-format", + "category": "recommended-pr", + "label": "Registry Package Format", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "registry ok True", + "command": "make ci-test", + "test": "tests/verify_registry_audit.py", + "evidence": [ + { + "path": "registry/package.schema.json", + "exists": true + }, + { + "path": "reports/registry_audit.json", + "exists": true + }, + { + "path": "tests/verify_registry_audit.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "review-studio-2", + "category": "recommended-pr", + "label": "Review Studio 2.0", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "16 review gates", + "command": "make ci-test", + "test": "tests/verify_review_studio.py", + "evidence": [ + { + "path": "scripts/render_review_studio.py", + "exists": true + }, + { + "path": "reports/review-studio.html", + "exists": true + }, + { + "path": "tests/verify_review_studio.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "migration-v2-docs", + "category": "recommended-pr", + "label": "Migration V2 Docs", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "migration guide present", + "command": "make ci-test", + "test": "docs review", + "evidence": [ + { + "path": "docs/migration-v2.md", + "exists": true + }, + { + "path": "reports/skill-os-2-review.md", + "exists": true + }, + { + "path": "README.md", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "evidence-consistency", + "category": "recommended-pr", + "label": "Evidence Consistency", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "39 consistency checks", + "command": "make ci-test", + "test": "tests/verify_evidence_consistency.py", + "evidence": [ + { + "path": "scripts/render_evidence_consistency.py", + "exists": true + }, + { + "path": "reports/evidence_consistency.json", + "exists": true + }, + { + "path": "tests/verify_evidence_consistency.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + } + ], + "reference_extension_tracks": [ + { + "key": "skill-interpretation-report", + "label": "Skill Interpretation Report", + "status": "covered", + "objective": "User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations.", + "current": "Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests.", + "target": "Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests.", + "evidence": [ + { + "path": "reports/skill-overview.html", + "exists": true + }, + { + "path": "reports/skill-overview.json", + "exists": true + }, + { + "path": "scripts/render_skill_overview.py", + "exists": true + }, + { + "path": "scripts/render_skill_interpretation.py", + "exists": true + }, + { + "path": "schemas/skill-interpretation.schema.json", + "exists": true + }, + { + "path": "tests/verify_skill_interpretation.py", + "exists": true + } + ], + "next_action": "Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change." + }, + { + "key": "adaptive-self-iteration", + "label": "Adaptive Self-Iteration", + "status": "covered", + "objective": "Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback.", + "current": "Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata.", + "target": "Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan.", + "evidence": [ + { + "path": "references/autonomous-adaptation.md", + "exists": true + }, + { + "path": "references/user-memory-policy.md", + "exists": true + }, + { + "path": "schemas/adaptation-proposal.schema.json", + "exists": true + }, + { + "path": "scripts/summarize_user_signals.py", + "exists": true + }, + { + "path": "scripts/propose_adaptation.py", + "exists": true + }, + { + "path": "tests/verify_adaptation_safety.py", + "exists": true + }, + { + "path": "scripts/apply_adaptation.py", + "exists": true + }, + { + "path": "reports/adaptation_approval_ledger.json", + "exists": true + }, + { + "path": "reports/adaptation_regression_report.json", + "exists": true + }, + { + "path": "reports/user_patterns.json", + "exists": true + }, + { + "path": "reports/adaptation_proposals.json", + "exists": true + }, + { + "path": "reports/iteration-directions.md", + "exists": true + }, + { + "path": "reports/adoption_drift_report.md", + "exists": true + } + ], + "next_action": "Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path." + }, + { + "key": "daily-skillops-report", + "label": "Daily SkillOps Report", + "status": "covered", + "objective": "Daily operations layer summarizes explicit-source conversation patterns, proposal-only adaptation work, approval state, release locks, and world-class evidence gaps.", + "current": "Daily SkillOps report is a CLI-backed, explicit-source operations cockpit with tests and committed dated evidence.", + "target": "Generate reports/skillops/daily/YYYY-MM-DD.* from explicit sources without private log scanning, source writes, auto-patching, or world-class overclaiming.", + "evidence": [ + { + "path": "scripts/render_daily_skillops_report.py", + "exists": true + }, + { + "path": "tests/verify_daily_skillops.py", + "exists": true + }, + { + "path": "reports/skillops/daily/2026-06-17.json", + "exists": true + }, + { + "path": "reports/skillops/daily/2026-06-17.md", + "exists": true + } + ], + "next_action": "Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves." + }, + { + "key": "weekly-curator-report", + "label": "Weekly Curator Report", + "status": "covered", + "objective": "Weekly curator layer aggregates Daily SkillOps opportunities, Skill Atlas portfolio signals, release locks, and world-class evidence gaps into a maintenance queue.", + "current": "Weekly curator report is CLI-backed, proposal-only, and backed by committed weekly JSON/Markdown evidence.", + "target": "Generate reports/skillops/weekly/YYYY-WNN.* without private log scanning, source writes, auto-patching, or world-class overclaiming.", + "evidence": [ + { + "path": "scripts/render_weekly_curator_report.py", + "exists": true + }, + { + "path": "tests/verify_weekly_curator.py", + "exists": true + }, + { + "path": "reports/skillops/weekly/2026-W25.json", + "exists": true + }, + { + "path": "reports/skillops/weekly/2026-W25.md", + "exists": true + } + ], + "next_action": "Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes." + } + ], + "next_highest_leverage": [ + "Close the four world-class evidence ledger entries with accepted human or external evidence.", + "Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves.", + "Use Daily SkillOps with explicit curated sources to review adaptation proposals without scanning private logs or auto-patching.", + "Keep the blueprint coverage report in CI so 2.0 plan drift is visible." + ], + "source_blueprint": { + "title": "Skill Overview / Skill OS 2.0 upgrade plan", + "core_module_count": 8, + "recommended_pr_count": 13, + "reference_extension_count": 4, + "reference_extensions": [ + "Skill interpretation report", + "Adaptive self-iteration with local preference memory and approval gates", + "Daily SkillOps operations report" + ] + }, + "artifacts": { + "json": "reports/skill_os2_coverage.json", + "markdown": "reports/skill_os2_coverage.md" + } + }, "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { - "target_count": 4, - "pass_count": 4, + "target_count": 5, + "pass_count": 5, "warn_count": 0, "block_count": 0, "failure_count": 0, @@ -3542,17 +7661,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -3588,21 +7708,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -3624,6 +7741,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -3651,39 +7769,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -3698,32 +7848,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -3763,6 +7974,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -3771,15 +7984,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -3816,7 +8030,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -3850,7 +8064,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -3867,18 +8082,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -3887,18 +8104,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -3913,15 +8141,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -3930,15 +8172,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -3953,22 +8200,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -3988,9 +8236,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -4002,17 +8250,17 @@ "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", "activation": { "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/openai/adapter.json", @@ -4021,7 +8269,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -4092,18 +8340,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -4112,18 +8362,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -4138,15 +8399,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -4155,15 +8430,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -4178,22 +8458,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -4211,9 +8492,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -4225,17 +8506,17 @@ "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", "activation": { "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/openai/adapter.json", @@ -4244,7 +8525,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -4331,17 +8612,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -4377,21 +8659,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -4413,6 +8692,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -4440,39 +8720,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -4487,32 +8799,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -4552,6 +8925,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -4560,15 +8935,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -4605,7 +8981,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -4639,7 +9015,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -4656,18 +9033,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -4676,18 +9055,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -4702,15 +9092,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -4719,15 +9123,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -4742,22 +9151,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -4777,9 +9187,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -4791,17 +9201,17 @@ "native_surface": "Claude-compatible neutral source folder with adapter notes", "activation": { "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/claude/adapter.json", @@ -4810,7 +9220,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -4881,18 +9291,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -4901,18 +9313,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -4927,15 +9350,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -4944,15 +9381,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -4967,22 +9409,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -5000,9 +9443,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -5014,17 +9457,17 @@ "native_surface": "Claude-compatible neutral source folder with adapter notes", "activation": { "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/claude/adapter.json", @@ -5033,7 +9476,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -5120,17 +9563,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -5166,21 +9610,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -5202,6 +9643,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -5229,39 +9671,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -5276,32 +9750,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -5341,6 +9876,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -5349,15 +9886,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -5394,7 +9932,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -5428,7 +9966,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -5445,18 +9984,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -5465,18 +10006,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -5491,15 +10043,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -5508,15 +10074,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -5531,22 +10102,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -5566,9 +10138,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -5580,17 +10152,17 @@ "native_surface": "Agent Skills compatible neutral package", "activation": { "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/generic/adapter.json" @@ -5598,7 +10170,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -5663,18 +10235,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -5683,18 +10257,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -5709,15 +10294,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -5726,15 +10325,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -5749,22 +10353,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -5782,9 +10387,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -5796,17 +10401,17 @@ "native_surface": "Agent Skills compatible neutral package", "activation": { "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "targets/generic/adapter.json" @@ -5814,7 +10419,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -5893,17 +10498,18 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "compiled_contract": { "name": "yao-meta-skill", "title": "Yao Meta Skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -5939,21 +10545,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -5975,6 +10578,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -6002,39 +10606,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -6049,32 +10685,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -6114,6 +10811,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -6122,15 +10821,16 @@ "reports/governance_score.json" ], "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 } }, "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -6167,7 +10867,7 @@ ], "baseline": "without_skill", "counts": { - "trigger": 26, + "trigger": 27, "output": 3, "adversarial": 1, "baseline": 1 @@ -6201,7 +10901,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "canonical_format": "agent-skills" }, @@ -6218,18 +10919,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -6238,18 +10941,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -6264,15 +10978,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -6281,15 +11009,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -6304,22 +11037,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -6339,9 +11073,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -6353,17 +11087,17 @@ "native_surface": "Agent Skills standard source tree", "activation": { "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "SKILL.md", @@ -6372,7 +11106,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -6436,18 +11170,20 @@ "capabilities": { "network": { "required": true, - "script_count": 2, + "script_count": 3, "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "review_reason": "Outbound hosts must match security/network_policy.json." }, "file_write": { "required": true, - "script_count": 47, + "script_count": 73, "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -6456,18 +11192,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -6482,15 +11229,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -6499,15 +11260,20 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 12, "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "review_reason": "Scripts spawn local commands and need operator review." }, @@ -6522,22 +11288,23 @@ "source": "security/network_policy.json", "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [] }, "help_smoke": { "enabled": true, - "checked_count": 59, + "checked_count": 86, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, - "network_script_count": 2, - "file_write_script_count": 47, - "subprocess_script_count": 7, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -6555,9 +11322,9 @@ "subprocess" ], "capability_counts": { - "network": 2, - "file_write": 47, - "subprocess": 7, + "network": 3, + "file_write": 73, + "subprocess": 12, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -6569,17 +11336,17 @@ "native_surface": "Agent Skills standard source tree", "activation": { "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "manual_activation_supported": true, "automatic_activation_note": "Depends on the target client route/catalog implementation." }, "resources": { "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { - "references": 32, - "scripts": 66, - "assets": 2, - "reports": 41 + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 }, "generated_files": [ "SKILL.md", @@ -6588,7 +11355,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 66, + "script_count": 153, "help_smoke_failed_count": 0 }, "permissions": { @@ -6647,6 +11414,951 @@ "unsupported_features": [], "warnings": [], "failures": [] + }, + { + "schema_version": "1.0", + "target": "vscode", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/autonomous-adaptation.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/skillops-decision-policy.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md", + "references/user-memory-policy.md" + ], + "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", + "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", + "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", + "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", + "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", + "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", + "scripts/skill_report_layout.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", + "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/trust_check_scripts.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", + "scripts/yao.py", + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" + ], + "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 27, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 3, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 73, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", + "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", + "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 12, + "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", + "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py", + "scripts/yao_cli_runtime.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 86, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "vscode", + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 3, + "file_write": 73, + "subprocess": 12, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 153, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "target_runtime": { + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "VS Code/Copilot Agent Skills project or user scope" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 3, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 73, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", + "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", + "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 12, + "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", + "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", + "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py", + "scripts/yao_cli_runtime.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 86, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 3, + "file_write_script_count": 73, + "subprocess_script_count": 12, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "vscode", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 3, + "file_write": 73, + "subprocess": 12, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "vscode", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation": { + "policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "counts": { + "references": 35, + "scripts": 153, + "assets": 6, + "reports": 43 + }, + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ] + }, + "scripts": { + "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "script_count": 153, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "client-or-workspace-trust", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + "VS Code installation scope is documented but not installed by this compiler" + ] + }, + "install_scope": "VS Code user or project skills directory" + }, + "target_transform": { + "target": "vscode", + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "permission_representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "permission_enforcement": "client-or-workspace-trust" + }, + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [], + "failures": [] } ], "failures": [], @@ -6690,6 +12402,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -6698,8 +12411,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [] @@ -6734,6 +12446,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -6742,8 +12455,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [] @@ -6757,7 +12469,7 @@ "frontmatter description exists", "description length <= 1024", "name is runtime-safe", - "directory name matches skill name", + "package identity derives from skill name", "manifest.name exists", "manifest.version exists", "manifest.owner exists", @@ -6778,6 +12490,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -6786,8 +12499,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [ @@ -6803,14 +12515,14 @@ "frontmatter description exists", "description length <= 1024", "name is runtime-safe", - "directory name matches skill name", + "package identity derives from skill name", "manifest.name exists", "manifest.version exists", "manifest.owner exists", "manifest.status exists", "manifest.maturity_tier exists", "manifest.review_cadence exists", - "manifest declares vscode via agent-skills-compatible", + "manifest declares vscode", "interface.display_name exists", "interface.short_description exists", "interface.default_prompt exists", @@ -6824,6 +12536,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -6832,8 +12545,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [ @@ -6870,6 +12582,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/autonomous-adaptation.md", "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", @@ -6878,8 +12591,7 @@ "references resource resolves: references/intent-dialogue.md", "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", - "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md" + "references resource resolves: references/operating-modes.md" ], "failures": [], "warnings": [] @@ -6906,14 +12618,34 @@ "subprocess" ], "summary": { - "target_count": 3, - "pass_count": 3, + "target_count": 4, + "pass_count": 4, "fail_count": 0, "native_enforcement_count": 0, - "metadata_fallback_count": 3, - "residual_risk_count": 3, + "metadata_fallback_count": 4, + "residual_risk_count": 4, "required_capability_count": 3, - "failure_count": 0 + "failure_count": 0, + "installer_enforcement_source_status": "present", + "installer_enforcement_target_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "installer_permission_capability_count": 3, + "world_class_native_evidence_ready": false, + "installer_enforcement_ready": true + }, + "installer_enforcement": { + "source": "reports/install_simulation.json", + "source_status": "present", + "package_dir_matches": true, + "summary": { + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, + "failure_count": 0 + } }, "targets": [ { @@ -6923,6 +12655,18 @@ "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "openai", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, "assurance": "metadata-fallback-explicit", "declared_capabilities": [ "file_write", @@ -7008,6 +12752,18 @@ "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "claude", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, "assurance": "metadata-fallback-explicit", "declared_capabilities": [ "file_write", @@ -7078,6 +12834,18 @@ "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "generic", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, "assurance": "metadata-fallback-explicit", "declared_capabilities": [ "file_write", @@ -7140,6 +12908,88 @@ "residual_risks": [ "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." ] + }, + { + "target": "vscode", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "vscode", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "vscode adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "vscode adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "vscode adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "vscode permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "vscode target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "vscode target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "vscode target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "vscode target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "vscode target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "vscode review_required matches whether capabilities are required" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] } ], "failures": [], @@ -7152,33 +13002,61 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 150, - "script_count": 66, - "internal_module_count": 7, + "scanned_files": 245, + "script_count": 153, + "internal_module_count": 67, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], - "network_script_count": 2, - "network_policy_covered_count": 2, + "network_script_count": 3, + "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 47, + "file_write_script_count": 73, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 59, + "help_smoke_checked_count": 86, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 150, - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_hash_file_count": 245, + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" }, "failures": [], "warnings": [], "secrets": [], "scripts": [ + { + "path": "scripts/adaptation_patch_safety.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared adaptation patch target parsing and target-file baseline verification helpers.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/adaptation_report_contracts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared adaptation approval and regression report contract decoration helpers.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/adjudicate_output_review.py", "interface": "cli", @@ -7193,6 +13071,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/apply_adaptation.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Approval-gated adaptive patch application with dry-run, allowlist, regression, and rollback evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/benchmark_release_lock.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_benchmark_reproducibility.py to keep release-lock git status classification out of the report renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/build_confusion_matrix.py", "interface": "cli", @@ -7221,6 +13127,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/build_skill_atlas_layout.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by build_skill_atlas.py to render the static Skill Atlas HTML report.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/build_skill_atlas_opportunities.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by build_skill_atlas.py to keep no-route opportunity detection out of the atlas CLI.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/check_update.py", "interface": "cli", @@ -7282,6 +13216,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/compile_skill_targets.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by compile_skill for target platform model and contract builders.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/context_sizer.py", "interface": "cli", @@ -7324,6 +13272,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/cross_packager_contracts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by cross_packager.py to keep platform contract data separate from packaging flow.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/description_optimizer_reporting.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by optimize_description.py to render description optimization reports.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/diff_eval.py", "interface": "cli", @@ -7338,6 +13314,104 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/emit_telemetry_event.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_core.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py for shared report loading, comparison, and Markdown rendering helpers.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_phase_queue.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to prevent preflight and Review Studio phase-queue drift.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_release.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to verify release evidence refresh instructions.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_skill_os2_review.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to keep Skill OS 2.0 review summary drift checks out of the main consistency renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/export_skill_ir.py", "interface": "cli", @@ -7384,6 +13458,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/html_rendering.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Used by report renderers to escape HTML while preserving meaningful falsey values.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/import_output_review_decisions.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Imports human blind A/B reviewer decisions into the canonical output-review decision file.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/import_telemetry_events.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/init_skill.py", "interface": "cli", @@ -7454,6 +13570,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/output_review_privacy.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/prepare_output_review_kit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/prepare_world_class_submission_kit.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Prepares editable world-class evidence intake packets without counting drafts as accepted evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/probe_runtime_permissions.py", "interface": "cli", @@ -7482,6 +13640,68 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/propose_adaptation.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Turns redacted repeated preference patterns into proposal-only adaptation plans.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/provider_output_eval_runner.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": true, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [ + "https://api.openai.com/v1/responses", + "https://api.deepseek.com/chat/completions" + ], + "network_hosts": [ + "api.deepseek.com", + "api.openai.com" + ] + }, + { + "path": "scripts/python_compat_check.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Checks repository Python source for syntax that can pass locally but fail on the supported CI interpreter.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/reference_synthesis_markdown.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_reference_synthesis.py to keep reference synthesis modeling separate from Markdown rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/registry_audit.py", "interface": "cli", @@ -7510,6 +13730,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_architecture_maintainability.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_artifact_design_profile.py", "interface": "cli", @@ -7538,6 +13772,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a release-facing benchmark reproducibility manifest and Markdown report.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_context_reports.py", "interface": "cli", @@ -7552,6 +13800,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_daily_skillops_report.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a Daily SkillOps report that summarizes explicit-source patterns, proposals, approval state, and release evidence without scanning private logs or applying patches.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_description_drift_history.py", "interface": "cli", @@ -7580,6 +13842,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_evidence_consistency.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a cross-report evidence consistency gate for generated Skill OS reports.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_intent_confidence.py", "interface": "cli", @@ -7776,6 +14052,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_skill_interpretation.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders the first-class skill interpretation report while reusing the Skill Overview v2 model and layout.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_skill_os2_audit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_skill_os2_coverage.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a Skill OS 2.0 blueprint-to-evidence coverage audit.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_skill_overview.py", "interface": "cli", @@ -7822,6 +14140,132 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_telemetry_hook_recipes.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_weekly_curator_report.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a weekly SkillOps curator report from generated reports without scanning private logs or writing source files.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_claim_guard.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_evidence_intake.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Validates world-class human and external evidence intake packets before ledger review.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a machine-checkable ledger for world-class external and human evidence gaps.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_evidence_plan.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders an operator runbook for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_preflight.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a preflight checklist for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_submission_review.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a read-only review queue for world-class evidence submissions before ledger acceptance.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/resource_boundary_check.py", "interface": "cli", @@ -7836,6 +14280,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/review_studio_action_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by review_studio_actions.py to keep world-class evidence action cards out of generic Review Studio action wiring.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_actions.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep Review Studio action guidance and source refs out of HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_data.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to load Review Studio source reports and metric cards.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/review_studio_formatting.py", "interface": "internal-module", @@ -7850,6 +14336,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/review_studio_gate_contract.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by review_studio_gates.py for stable gate scoring and contract checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_gate_helpers.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by review_studio_gates.py to keep reusable Review Studio gate helpers out of the main gate sequence.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_gates.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep Review Studio gate evaluation separate from HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/review_studio_layout.py", "interface": "internal-module", @@ -7864,6 +14392,90 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/review_studio_output_review.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to render output review checklist cards.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_panels.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep small Review Studio panel renderers out of the main page composer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_skillops.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep SkillOps summary panels out of the main HTML renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_waivers.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep waiver candidate layout out of the main renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to render world-class evidence cards.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_viewer_data.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_viewer.py to assemble Review Viewer data before HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/run_conformance_suite.py", "interface": "cli", @@ -7948,6 +14560,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/skill_ir_paths.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by compiler, registry, conformance, and report scripts to locate canonical Skill IR artifacts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skill_os2_coverage_markdown.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_os2_coverage.py to keep coverage data assembly separate from Markdown rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/skill_report_charts.py", "interface": "internal-module", @@ -7962,6 +14602,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/skill_report_i18n.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_overview.py to keep bilingual report copy and fallback rules out of HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/skill_report_layout.py", "interface": "internal-module", @@ -8004,6 +14658,76 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/skill_report_sections.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to assemble overview report quality, risk, and asset sections.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skill_report_sources.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to load source files and scan overview package assets.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skill_report_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to summarize world-class evidence readiness and roadmap actions.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skillops_opportunity.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_daily_skillops_report.py to score SkillOps opportunities without writing files.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/summarize_user_signals.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Scans an explicit local source file and summarizes redacted repeated user preference signals.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/sync_local_install.py", "interface": "cli", @@ -8018,6 +14742,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/telemetry_native_host.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/trigger_eval.py", "interface": "cli", @@ -8046,6 +14784,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/trust_check_scripts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Static script inventory helpers imported by trust_check.py.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/upgrade_check.py", "interface": "cli", @@ -8088,6 +14840,188 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_evidence_contract.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to share intake validation and artifact integrity checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_human_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_operator_runbook_coordination.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Builds world-class operator coordination steps and release gates for render_world_class_operator_runbook.py.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_phase_queue.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by prepare_world_class_submission_kit.py to group repair rows into an operator execution queue.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_preflight_layout.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_preflight_markdown.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_world_class_preflight.py to keep Markdown layout separate from preflight data assembly.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_provider_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_repair_checklist.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared by submission kit generation to turn readiness blockers into actionable repair rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_source_checks.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to keep source-evidence readiness checks consistent.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_submission_kit_rendering.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared renderer for world-class submission kit Markdown and HTML artifacts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_submission_matrix.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared by submission kit rendering to summarize draft, artifact, and source-check readiness.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -8102,6 +15036,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/yao_cli_adaptation_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep adaptive scan/proposal/apply command handlers outside the thin CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao_cli_config.py", "interface": "internal-module", @@ -8115,6 +15063,132 @@ "uses_subprocess": false, "network_urls": [], "network_hosts": [] + }, + { + "path": "scripts/yao_cli_create_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_distribution_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep distribution and runtime gate handlers outside the thin CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_output_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep output evaluation and review handlers outside the thin CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_parser.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep CLI parser declarations separate from command orchestration.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_parser_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao_cli_parser.py to keep evidence command declarations out of the main parser module.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_parser_operations.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao_cli_parser.py to keep SkillOps, telemetry, and review-loop command declarations out of the main parser module.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_report_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep report and evidence command handlers out of the CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_runtime.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py and command modules for shared subprocess execution and JSON payload parsing.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_telemetry.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to record opt-in metadata-only CLI run telemetry.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] } ], "dependencies": { @@ -8130,10 +15204,11 @@ "present": true, "path": "security/network_policy.json", "schema_version": "1.0", - "network_script_count": 2, + "network_script_count": 3, "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [], @@ -8146,11 +15221,11 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 59, - "checked_count": 59, - "passed_count": 59, + "candidate_count": 86, + "checked_count": 86, + "passed_count": 86, "failed_count": 0, - "skipped_count": 7, + "skipped_count": 67, "failed_scripts": [], "results": [ { @@ -8163,6 +15238,16 @@ "stdout_excerpt": "usage: adjudicate_output_review.py [-h] [--blind-pack BLIND_PACK]\n [--answer-key ANSWER_KEY]\n [--decisions DECISIONS]\n [--output-json OUTP", "stderr_excerpt": "" }, + { + "path": "scripts/apply_adaptation.py", + "command": "python3 scripts/apply_adaptation.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: apply_adaptation.py [-h] [--proposal-id PROPOSAL_ID]\n [--patch-file PATCH_FILE]\n [--proposals-json PROPOSALS_JSON]\n [--approval-ledger APPROVAL_LEDGER]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/build_confusion_matrix.py", "command": "python3 scripts/build_confusion_matrix.py --help", @@ -8263,6 +15348,16 @@ "stdout_excerpt": "usage: diff_eval.py [-h] before after\n\nDiff two eval reports.\n\npositional arguments:\n before\n after\n\noptions:\n -h, --help show this help message and exit", "stderr_excerpt": "" }, + { + "path": "scripts/emit_telemetry_event.py", + "command": "python3 scripts/emit_telemetry_event.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: emit_telemetry_event.py [-h] [--output-jsonl OUTPUT_JSONL]\n [--event {review_event,script_run,skill_activation,skill_output}]\n [--activation-type {explicit,implicit,manual,u", + "stderr_excerpt": "" + }, { "path": "scripts/export_skill_ir.py", "command": "python3 scripts/export_skill_ir.py --help", @@ -8293,6 +15388,26 @@ "stdout_excerpt": "usage: governance_check.py [-h] [--require-manifest] skill_dir\n\nCheck skill governance metadata and lifecycle readiness.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit\n --require-manifest", "stderr_excerpt": "" }, + { + "path": "scripts/import_output_review_decisions.py", + "command": "python3 scripts/import_output_review_decisions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: import_output_review_decisions.py [-h] --input INPUT\n [--format {auto,json,jsonl,csv}]\n [--blind-pack BLIND_PACK]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/import_telemetry_events.py", + "command": "python3 scripts/import_telemetry_events.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: import_telemetry_events.py [-h] --input-jsonl INPUT_JSONL\n [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OU", + "stderr_excerpt": "" + }, { "path": "scripts/init_skill.py", "command": "python3 scripts/init_skill.py --help", @@ -8343,6 +15458,26 @@ "stdout_excerpt": "usage: optimize_description.py [-h] --description-file DESCRIPTION_FILE\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n --dev-cases DEV_CASES\n ", "stderr_excerpt": "" }, + { + "path": "scripts/prepare_output_review_kit.py", + "command": "python3 scripts/prepare_output_review_kit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: prepare_output_review_kit.py [-h] [--blind-pack-json BLIND_PACK_JSON]\n [--blind-pack-md BLIND_PACK_MD]\n [--decisions DECISIONS]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/prepare_world_class_submission_kit.py", + "command": "python3 scripts/prepare_world_class_submission_kit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: prepare_world_class_submission_kit.py [-h] [--output-dir OUTPUT_DIR]\n [--evidence-key EVIDENCE_KEY]\n [--overwrite]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/probe_runtime_permissions.py", "command": "python3 scripts/probe_runtime_permissions.py --help", @@ -8350,7 +15485,7 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: probe_runtime_permissions.py [-h] [--package-dir PACKAGE_DIR]\n [--target {openai,claude,generic}]\n [--output-json OUTPUT_JSON]\n ", + "stdout_excerpt": "usage: probe_runtime_permissions.py [-h] [--package-dir PACKAGE_DIR]\n [--target {openai,claude,generic,vscode}]\n [--output-json OUTPUT_JSON]\n ", "stderr_excerpt": "" }, { @@ -8363,6 +15498,36 @@ "stdout_excerpt": "usage: promotion_checker.py [-h] [--optimization-suite OPTIMIZATION_SUITE]\n [--route-scorecard ROUTE_SCORECARD]\n [--output-json OUTPUT_JSON]\n [--output-md OUTP", "stderr_excerpt": "" }, + { + "path": "scripts/propose_adaptation.py", + "command": "python3 scripts/propose_adaptation.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: propose_adaptation.py [-h] [--patterns-json PATTERNS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/provider_output_eval_runner.py", + "command": "python3 scripts/provider_output_eval_runner.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: provider_output_eval_runner.py [-h] [--provider PROVIDER]\n [--base-url BASE_URL]\n [--api-format {responses,chat-completions}]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/python_compat_check.py", + "command": "python3 scripts/python_compat_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: python_compat_check.py [-h] [--path PATH]\n [--target-python TARGET_PYTHON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/registry_audit.py", "command": "python3 scripts/registry_audit.py --help", @@ -8383,6 +15548,16 @@ "stdout_excerpt": "usage: render_adoption_drift_report.py [-h] [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_architecture_maintainability.py", + "command": "python3 scripts/render_architecture_maintainability.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_architecture_maintainability.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--warn-lines WARN_LINES]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_artifact_design_profile.py", "command": "python3 scripts/render_artifact_design_profile.py --help", @@ -8403,6 +15578,16 @@ "stdout_excerpt": "usage: render_baseline_compare.py [-h] --entry ENTRY\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a baseline comparison report from description optimization ", "stderr_excerpt": "" }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "command": "python3 scripts/render_benchmark_reproducibility.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_benchmark_reproducibility.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_context_reports.py", "command": "python3 scripts/render_context_reports.py --help", @@ -8413,6 +15598,16 @@ "stdout_excerpt": "usage: render_context_reports.py [-h] [--generated-at GENERATED_AT]\n\nRender context budget reports for root and example skills.\n\noptions:\n -h, --help show this help message and exit\n --generated-at GENERATED_AT\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_daily_skillops_report.py", + "command": "python3 scripts/render_daily_skillops_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_daily_skillops_report.py [-h] [--source SOURCE]\n [--patterns-json PATTERNS_JSON]\n [--proposals-json PROPOSALS_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_description_drift_history.py", "command": "python3 scripts/render_description_drift_history.py --help", @@ -8433,6 +15628,16 @@ "stdout_excerpt": "usage: render_eval_dashboard.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_evidence_consistency.py", + "command": "python3 scripts/render_evidence_consistency.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_evidence_consistency.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [", + "stderr_excerpt": "" + }, { "path": "scripts/render_intent_confidence.py", "command": "python3 scripts/render_intent_confidence.py --help", @@ -8490,7 +15695,7 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: render_portability_report.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a portability score from neutral metadata, contracts, and snapshots.\n\noptions:\n -h, --help ", + "stdout_excerpt": "usage: render_portability_report.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [skill_dir]\n\nRender a portability score from neutral metadata, contracts, a", "stderr_excerpt": "" }, { @@ -8573,6 +15778,36 @@ "stdout_excerpt": "usage: render_review_waivers.py [-h] [--waivers-json WAIVERS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED", "stderr_excerpt": "" }, + { + "path": "scripts/render_skill_interpretation.py", + "command": "python3 scripts/render_skill_interpretation.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_interpretation.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender the first-class HTML skill interpretation rep", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_skill_os2_audit.py", + "command": "python3 scripts/render_skill_os2_audit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_os2_audit.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [skill_dir]\n\nRender a", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_skill_os2_coverage.py", + "command": "python3 scripts/render_skill_os2_coverage.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_os2_coverage.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [skill_di", + "stderr_excerpt": "" + }, { "path": "scripts/render_skill_overview.py", "command": "python3 scripts/render_skill_overview.py --help", @@ -8603,6 +15838,96 @@ "stdout_excerpt": "usage: render_system_model.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a systems-thinking model for a skill package.\n\npositional arguments:\n sk", "stderr_excerpt": "" }, + { + "path": "scripts/render_telemetry_hook_recipes.py", + "command": "python3 scripts/render_telemetry_hook_recipes.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_telemetry_hook_recipes.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--output-jsonl OUTPUT_JSONL]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_weekly_curator_report.py", + "command": "python3 scripts/render_weekly_curator_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_weekly_curator_report.py [-h] [--daily-json DAILY_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_claim_guard.py", + "command": "python3 scripts/render_world_class_claim_guard.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_claim_guard.py [-h] [--claim-surface CLAIM_SURFACE]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_evidence_intake.py", + "command": "python3 scripts/render_world_class_evidence_intake.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_intake.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "command": "python3 scripts/render_world_class_evidence_ledger.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_ledger.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--submissions-dir SUBMISSIONS_DIR]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_evidence_plan.py", + "command": "python3 scripts/render_world_class_evidence_plan.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "command": "python3 scripts/render_world_class_operator_runbook.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_operator_runbook.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_preflight.py", + "command": "python3 scripts/render_world_class_preflight.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_preflight.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n [--output-md OU", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_submission_review.py", + "command": "python3 scripts/render_world_class_submission_review.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_submission_review.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/resource_boundary_check.py", "command": "python3 scripts/resource_boundary_check.py --help", @@ -8673,6 +15998,16 @@ "stdout_excerpt": "usage: simulate_install.py [-h] [--package-dir PACKAGE_DIR]\n [--install-root INSTALL_ROOT]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--generated-at G", "stderr_excerpt": "" }, + { + "path": "scripts/summarize_user_signals.py", + "command": "python3 scripts/summarize_user_signals.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: summarize_user_signals.py [-h] --source SOURCE\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--min-support MIN_SUPPORT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/sync_local_install.py", "command": "python3 scripts/sync_local_install.py --help", @@ -8680,7 +16015,17 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: sync_local_install.py [-h] [--root ROOT] [--install-dir INSTALL_DIR]\n [--dry-run]\n\nSync the current yao-meta-skill source into a managed local skill mirror.\n\noptions:\n -h, --help show this help", + "stdout_excerpt": "usage: sync_local_install.py [-h] [--root ROOT] [--install-dir INSTALL_DIR]\n [--package-dir PACKAGE_DIR]\n [--generated-at GENERATED_AT]\n [--skip-install-pre", + "stderr_excerpt": "" + }, + { + "path": "scripts/telemetry_native_host.py", + "command": "python3 scripts/telemetry_native_host.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: telemetry_native_host.py [-h] [--output-jsonl OUTPUT_JSONL]\n [--message-json MESSAGE_JSON] [--stdio]\n [--dry-run] [--write-manifest WRITE_MANIFEST]\n ", "stderr_excerpt": "" }, { @@ -8740,23 +16085,151 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: yao.py [-h]\n {init,quickstart,validate,optimize-description,promote-check,review,release-snapshot,workspace-flow,report,skill-report,review-viewer,review-studio,reference-scan,github-benchmark-scan,intent-confidence,inte", + "stdout_excerpt": "usage: yao.py [-h] [--record-cli-telemetry] [--no-cli-telemetry]\n [--telemetry-events-jsonl TELEMETRY_EVENTS_JSONL]\n {init,quickstart,validate,optimize-description,promote-check,python-compat,architecture-audit,rev", "stderr_excerpt": "" } ], "skipped": [ + { + "path": "scripts/adaptation_patch_safety.py", + "reason": "internal module" + }, + { + "path": "scripts/adaptation_report_contracts.py", + "reason": "internal module" + }, + { + "path": "scripts/benchmark_release_lock.py", + "reason": "internal module" + }, + { + "path": "scripts/build_skill_atlas_layout.py", + "reason": "internal module" + }, + { + "path": "scripts/build_skill_atlas_opportunities.py", + "reason": "internal module" + }, + { + "path": "scripts/compile_skill_targets.py", + "reason": "internal module" + }, + { + "path": "scripts/cross_packager_contracts.py", + "reason": "internal module" + }, + { + "path": "scripts/description_optimizer_reporting.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_core.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_phase_queue.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_release.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_skill_os2_review.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_world_class.py", + "reason": "internal module" + }, + { + "path": "scripts/html_rendering.py", + "reason": "internal module" + }, + { + "path": "scripts/output_review_privacy.py", + "reason": "internal module" + }, + { + "path": "scripts/reference_synthesis_markdown.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_action_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_actions.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_data.py", + "reason": "internal module" + }, { "path": "scripts/review_studio_formatting.py", "reason": "internal module" }, + { + "path": "scripts/review_studio_gate_contract.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_gate_helpers.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_gates.py", + "reason": "internal module" + }, { "path": "scripts/review_studio_layout.py", "reason": "internal module" }, + { + "path": "scripts/review_studio_output_review.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_panels.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_skillops.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_waivers.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_world_class.py", + "reason": "internal module" + }, + { + "path": "scripts/review_viewer_data.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_ir_paths.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_os2_coverage_markdown.py", + "reason": "internal module" + }, { "path": "scripts/skill_report_charts.py", "reason": "internal module" }, + { + "path": "scripts/skill_report_i18n.py", + "reason": "internal module" + }, { "path": "scripts/skill_report_layout.py", "reason": "internal module" @@ -8769,9 +16242,121 @@ "path": "scripts/skill_report_model.py", "reason": "internal module" }, + { + "path": "scripts/skill_report_sections.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_sources.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_world_class.py", + "reason": "internal module" + }, + { + "path": "scripts/skillops_opportunity.py", + "reason": "internal module" + }, + { + "path": "scripts/trust_check_scripts.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_evidence_contract.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_human_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_operator_runbook_coordination.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_phase_queue.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_preflight_layout.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_preflight_markdown.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_provider_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_repair_checklist.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_source_checks.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_submission_kit_rendering.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_submission_matrix.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_adaptation_commands.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_config.py", "reason": "internal module" + }, + { + "path": "scripts/yao_cli_create_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_distribution_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_output_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_parser.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_parser_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_parser_operations.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_report_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_runtime.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_telemetry.py", + "reason": "internal module" } ] }, @@ -8804,11 +16389,12 @@ "status": "approved", "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "reviewer": "Yao Team", "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", - "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "reason": "Network use is bounded to public GitHub metadata, benchmark discovery, update checks, and reviewed provider output-eval endpoints, with allowed hosts reviewed in security/network_policy.json.", "expires_at": "2026-09-30", "evidence": [ "security/network_policy.json", @@ -8817,7 +16403,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" }, "validation": [] }, @@ -8826,6 +16413,7 @@ "status": "approved", "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -8834,18 +16422,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -8860,15 +16459,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -8885,7 +16498,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" }, "validation": [] }, @@ -8893,13 +16507,18 @@ "capability": "subprocess", "status": "approved", "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "reviewer": "Yao Team", "scope": "Local deterministic validation, packaging, install simulation, and CI orchestration commands.", @@ -8912,7 +16531,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" }, "validation": [] } @@ -8928,6 +16548,1610 @@ "markdown": "reports/security_trust_report.md" } }, + "python_compatibility": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "root": ".", + "summary": { + "target_python": "3.11", + "file_count": 232, + "issue_count": 0, + "syntax_error_count": 0, + "fstring_311_violation_count": 0, + "decision": "pass" + }, + "rules": [ + { + "key": "current-python-syntax", + "reason": "Every scanned Python source file must compile under the running interpreter." + }, + { + "key": "fstring-expression-backslash", + "reason": "Python 3.11 rejects backslashes inside f-string expressions; keep escaping outside the expression." + } + ], + "files": [ + { + "path": "examples/complex-release-orchestrator/generated-skill/scripts/build_release_packet.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "examples/governed-incident-command/generated-skill/scripts/build_incident_packet.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "geo-ranking-article-generator/scripts/build_html_index.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/adaptation_patch_safety.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/adaptation_report_contracts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/adjudicate_output_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/apply_adaptation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/benchmark_release_lock.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_confusion_matrix.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_skill_atlas.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_skill_atlas_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/build_skill_atlas_opportunities.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/check_update.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/ci_test.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/collect_feedback.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/compile_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/compile_skill_targets.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/context_sizer.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/create_iteration_snapshot.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/cross_packager.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/cross_packager_contracts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/description_optimizer_reporting.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/diff_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/emit_telemetry_event.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_core.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_phase_queue.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_release.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_skill_os2_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/evidence_consistency_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/export_skill_ir.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/github_benchmark_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/governance_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/import_output_review_decisions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/import_telemetry_events.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/init_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/judge_blind_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/lint_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/local_output_eval_runner.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/optimize_description.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/output_review_privacy.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/prepare_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/prepare_world_class_submission_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/probe_runtime_permissions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/promotion_checker.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/propose_adaptation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/provider_output_eval_runner.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/python_compat_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/reference_synthesis_markdown.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/registry_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_adoption_drift_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_architecture_maintainability.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_artifact_design_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_baseline_compare.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_context_reports.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_daily_skillops_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_description_drift_history.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_eval_dashboard.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_evidence_consistency.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_intent_confidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_intent_dialogue.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_iteration_directions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_iteration_ledger.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_output_risk_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_portability_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_prompt_quality_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_reference_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_reference_synthesis.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_regression_history.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_annotations.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_studio.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_viewer.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_review_waivers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_interpretation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_os2_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_os2_coverage.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_skill_overview.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_social_preview.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_system_model.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_telemetry_hook_recipes.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_weekly_curator_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_claim_guard.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_evidence_intake.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_evidence_plan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_preflight.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/render_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/resource_boundary_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_action_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_actions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_data.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_formatting.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_gate_contract.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_gate_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_gates.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_output_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_panels.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_skillops.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_waivers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_studio_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/review_viewer_data.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_conformance_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_description_optimization_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_eval_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_output_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/run_output_execution.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/simulate_install.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_ir_paths.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_os2_coverage_markdown.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_charts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_i18n.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_metrics.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_model.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_sections.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_sources.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skill_report_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/skillops_opportunity.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/summarize_user_signals.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/sync_local_install.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/telemetry_native_host.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/trigger_eval.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/trust_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/trust_check_scripts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/upgrade_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/validate_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/verify_package.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_evidence_contract.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_human_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_operator_runbook_coordination.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_phase_queue.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_preflight_layout.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_preflight_markdown.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_provider_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_repair_checklist.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_source_checks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_submission_kit_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/world_class_submission_matrix.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_adaptation_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_config.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_create_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_distribution_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_output_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_parser.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_parser_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_parser_operations.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_report_commands.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_runtime.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "scripts/yao_cli_telemetry.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/review_studio_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/review_studio_world_class_assertions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_adaptation_safety.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_adapter_snapshots.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_adoption_drift.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_architecture_maintainability.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_artifact_design_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_baseline_compare.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_benchmark_reproducibility.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_compile_skill.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_conformance_suite.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_daily_skillops.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_description_optimization.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_evidence_consistency.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_failure_regressions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_feedback.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_github_benchmark_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_html_rendering.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_install_simulation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_intent_confidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_iteration_directions.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_local_install_sync.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_eval_lab.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_execution_runs.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_review_adjudication.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_review_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_output_risk_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_package_verification.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_packager_failures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_portability_report.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_promotion_checker.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_prompt_quality_profile.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_python_compat_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_quality_checks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_reference_scan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_reference_synthesis.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_registry_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_annotations.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_studio.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_viewer.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_review_waivers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_route_confusion.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_runtime_permission_probes.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_security_boundaries.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_atlas.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_interpretation.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_ir.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_ir_paths.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_os2_audit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_os2_coverage.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_overview.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_report_charts.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skill_report_metrics.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_skillops_opportunity.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_system_model.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_emit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_hooks.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_import.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_telemetry_native_host.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_trust_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_upgrade_check.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_weekly_curator.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_claim_guard.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_evidence_ledger.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_evidence_plan.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_operator_runbook.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_preflight.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_submission_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_world_class_submission_review.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_yao_cli.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/verify_yao_cli_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/world_class_human_fixtures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/world_class_native_telemetry_fixtures.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/yao_cli_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, + { + "path": "tests/yao_cli_report_refresh.py", + "ok": true, + "issue_count": 0, + "issues": [] + } + ], + "issues": [], + "artifacts": { + "json": "reports/python_compatibility.json", + "markdown": "reports/python_compatibility.md" + } + }, + "architecture_maintainability": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "python_file_count": 229, + "script_file_count": 153, + "test_file_count": 76, + "internal_module_count": 69, + "cli_script_count": 86, + "command_handler_count": 68, + "entrypoint_command_handler_count": 18, + "command_module_count": 6, + "warn_line_threshold": 900, + "watch_line_threshold": 720, + "early_watch_line_threshold": 600, + "block_line_threshold": 1500, + "largest_file_lines": 719, + "watchlist_count": 0, + "early_watchlist_count": 6, + "hotspot_count": 0, + "blocker_count": 0, + "decision": "pass" + }, + "largest_files": [ + { + "path": "tests/verify_evidence_consistency.py", + "lines": 719, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 703, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_yao_cli.py", + "lines": 700, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "lines": 685, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 676, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "lines": 651, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "tests/verify_output_review_adjudication.py", + "lines": 599, + "kind": "test", + "severity": "pass", + "early_watch": false, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/render_skill_overview.py", + "lines": 588, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "tests/verify_world_class_evidence_ledger.py", + "lines": 587, + "kind": "test", + "severity": "pass", + "early_watch": false, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/build_skill_atlas.py", + "lines": 586, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/optimize_description.py", + "lines": 585, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/trust_check.py", + "lines": 582, + "kind": "cli-script", + "severity": "pass", + "early_watch": false, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + } + ], + "watchlist": [], + "early_watchlist": [ + { + "path": "tests/verify_evidence_consistency.py", + "lines": 719, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 703, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "tests/verify_yao_cli.py", + "lines": 700, + "kind": "test", + "severity": "pass", + "early_watch": true, + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "lines": 685, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 676, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "lines": 651, + "kind": "cli-script", + "severity": "pass", + "early_watch": true, + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + } + ], + "hotspots": [], + "actions": [], + "artifacts": { + "json": "reports/architecture_maintainability.json", + "markdown": "reports/architecture_maintainability.md" + } + }, "context_budget": { "ok": true, "failures": [], @@ -8935,14 +18159,71 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 811, - "other_text_tokens": 869504, - "estimated_initial_load_tokens": 987, - "estimated_total_text_tokens": 870315, - "relevant_file_count": 327, + "skill_body_tokens": 805, + "other_text_tokens": 1156781, + "estimated_initial_load_tokens": 998, + "estimated_total_text_tokens": 1157586, + "deferred_resource_tokens": 525864, + "deferred_resource_warn_threshold": 120000, + "deferred_resource_dirs": [ + { + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153 + }, + { + "path": "references", + "estimated_tokens": 28292, + "file_count": 35 + }, + { + "path": "evals", + "estimated_tokens": 21835, + "file_count": 27 + }, + { + "path": "assets", + "estimated_tokens": 11176, + "file_count": 4 + } + ], + "large_deferred_resource_dirs": [ + { + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153 + } + ], + "deferred_resource_governance": { + "status": "governed", + "large_dir_count": 1, + "governed_large_dir_count": 1, + "directories": [ + { + "status": "governed", + "evidence": [ + "reports/security_trust_report.json", + "reports/architecture_maintainability.json", + "reports/python_compatibility.json" + ], + "reasons": [ + "trust report covers scripts", + "architecture report has no script hotspots or blockers", + "Python compatibility report has no issues" + ], + "missing": [], + "path": "scripts", + "estimated_tokens": 464561, + "file_count": 153, + "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." + } + ], + "summary": "Large deferred resources are indexed and backed by evidence." + }, + "relevant_file_count": 712, "unused_resource_dirs": [], - "quality_signal_points": 130, - "quality_density": 131.7 + "quality_signal_points": 140, + "quality_density": 140.3 } }, "promotion": { @@ -8956,9 +18237,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -8989,44 +18270,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -9117,7 +18398,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.611, + "average_margin": 0.701, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -9140,8 +18421,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -9171,7 +18452,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -9185,7 +18467,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -9485,13 +18768,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -9510,7 +18793,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -9525,49 +18808,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -9630,7 +18913,8 @@ "causes": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", "gate_status": { @@ -9646,7 +18930,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.659, + "average_margin": 0.586, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -9660,8 +18944,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -9671,25 +18955,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -9698,7 +18967,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -9712,7 +18981,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -9726,9 +18995,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } @@ -9751,6 +19035,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "scope_policy": { @@ -9785,7 +19072,7 @@ { "name": "yao-meta-skill", "path": ".", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "owner": "Yao Team", "version": "1.1.0", "status": "active", @@ -9796,37 +19083,68 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "resources": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -9841,32 +19159,90 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", + "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py", "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -9894,35 +19270,48 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", "references/trust-security-method.md", + "references/user-memory-policy.md", + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], "token_set": [ - "add", - "asked", + "creation", + "distribution", "docs", "evals", "evaluate", - "existing", - "improve", + "improvement", "notes", "or", "package", - "process", + "packaging", "prompts", "refactor", - "repeated", - "reuse", - "team", + "team-ready", "transcripts", + "workflow", "workflows" ], "atlas_scope": "release", "actionable": true, - "scope_reason": "default release-actionable skill" + "scope_reason": "default release-actionable skill", + "telemetry": { + "report_present": true, + "report": "reports/adoption_drift_report.json", + "risk_band": "low", + "event_count": 1, + "adoption_sample_count": 1, + "adoption_rate": 100.0, + "candidate_count": 0 + } }, { "name": "release-orchestrator", @@ -9980,7 +19369,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/complex-release-orchestrator/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -10019,7 +19417,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/final/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -10042,7 +19449,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v0/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -10080,7 +19496,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v1/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "incident-command-governor", @@ -10121,7 +19546,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/governed-incident-command/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "note-cleanup", @@ -10158,7 +19592,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/simple-note-cleanup/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -10198,7 +19641,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/team-frontend-review/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "geo-ranking-article-generator", @@ -10258,7 +19710,16 @@ ], "atlas_scope": "embedded-generated-skill", "actionable": false, - "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release.", + "telemetry": { + "report_present": false, + "report": "geo-ranking-article-generator/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "invalid-governance-skill", @@ -10280,7 +19741,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/governance_invalid_manifest/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-yaml-skill", @@ -10304,7 +19774,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_invalid_yaml/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-skill", @@ -10327,7 +19806,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_missing_interface_field/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } } ], "summary": { @@ -10341,6 +19829,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 } }, @@ -10631,6 +20122,8 @@ } ], "actionable_owner_review_gaps": [], + "drift_signals": [], + "actionable_drift_signals": [], "no_route_opportunities": [], "artifacts": { "catalog": "skill_atlas/catalog.json", @@ -10638,6 +20131,7 @@ "dependency_graph": "skill_atlas/dependency_graph.json", "stale_skills": "skill_atlas/stale_skills.json", "owner_review_gaps": "skill_atlas/owner_review_gaps.json", + "drift_signals": "skill_atlas/drift_signals.json", "no_route_opportunities": "skill_atlas/no_route_opportunities.json", "report_json": "reports/skill_atlas.json", "report_html": "reports/skill_atlas.html" @@ -10646,7 +20140,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T10:57:52Z", + "generated_at": "2026-06-17T08:58:53Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -10670,13 +20164,14 @@ }, "summary": { "event_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -10685,73 +20180,995 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, - "failure_types": {} + "failure_types": {}, + "source_types": { + "manual": 1 + }, + "command_counts": {} }, "adoption_by_skill": [ { "skill": "yao-meta-skill", "events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { - "event": "skill_activation", + "command": "unknown", + "event": "review_event", "skill": "yao-meta-skill", + "source": "manual", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } }, + "daily_skillops": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17T08:58:54Z", + "skill_dir": ".", + "decision": "proposal-review", + "source_supplied": false, + "pattern_count": 5, + "proposal_count": 5, + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0, + "summary": { + "decision": "proposal-review", + "source_supplied": false, + "pattern_count": 5, + "proposal_count": 5, + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0 + }, + "operations_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "explicit_source_required_for_scan": true, + "implicit_private_log_scan": false, + "raw_content_stored": false, + "redacted_excerpts_only": true, + "proposal_only": true, + "approval_required_for_writes": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "daily_report_counts_as_world_class_evidence": false + }, + "report_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "source_supplied", + "pattern_count", + "proposal_count", + "approval_count", + "pending_review_count", + "applied_count", + "rollback_count", + "local_blueprint_ready", + "public_world_class_ready", + "world_class_pending_count", + "release_lock_ready", + "evidence_consistency_ok", + "writes_source_files", + "auto_patch_enabled", + "failure_count" + ], + "source_of_truth": [ + "summary", + "operations_contract" + ] + }, + "source": { + "path": "evals/adaptation/user_signals.example.jsonl", + "fingerprint_sha256": "48f94a4f3b82f90db29921c13d09c765ec10aabfde1adce65710c137cc6c5abf", + "record_count": 8, + "explicit_source": true, + "raw_content_stored": false, + "redacted_excerpts_only": true + }, + "patterns": [ + { + "pattern_id": "language_default", + "label": "Default language preference", + "signal_type": "report-language", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated report-language signals.", + "recommended_action": "Keep generated reports Chinese-first with an English switch where user-facing.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + } + ] + }, + { + "pattern_id": "report_ui", + "label": "Report UI and visualization preference", + "signal_type": "artifact-design", + "support_count": 5, + "confidence": 0.95, + "reason": "5 redacted records matched repeated artifact-design signals.", + "recommended_action": "Prioritize white-background Kami-style reports with readable charts and stable navigation.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-3", + "excerpt": "报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。" + } + ] + }, + { + "pattern_id": "approval_safety", + "label": "Approval and privacy boundary", + "signal_type": "governance", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated governance signals.", + "recommended_action": "Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path.", + "evidence": [ + { + "record_id": "line-5", + "excerpt": "自适应升级必须先生成提案，不能直接自动修改源文件。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "delivery_format", + "label": "Delivery format preference", + "signal_type": "artifact-format", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated artifact-format signals.", + "recommended_action": "Surface stable artifact paths and formats in CLI output and generated summaries.", + "evidence": [ + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "evidence_testing", + "label": "Evidence and testing preference", + "signal_type": "quality-gate", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated quality-gate signals.", + "recommended_action": "Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade.", + "evidence": [ + { + "record_id": "line-7", + "excerpt": "每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。" + }, + { + "record_id": "line-8", + "excerpt": "涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。" + } + ] + } + ], + "proposals": [ + { + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ] + }, + { + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "risk_level": "medium", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ] + }, + { + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ] + }, + { + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ] + }, + { + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "risk_level": "medium", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ] + } + ], + "opportunity_summary": { + "opportunity_count": 5, + "top_score": 88, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "action_type_counts": { + "patch_existing_skill": 3, + "agents_update": 1, + "add_eval": 1 + }, + "decision_counts": { + "ready_for_approval_review": 1, + "proposal_review": 4 + } + }, + "opportunities": [ + { + "opportunity_id": "skillops-4ad91c18dd", + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "opportunity_type": "artifact-quality", + "action_type": "patch_existing_skill", + "decision": "ready_for_approval_review", + "priority": "medium", + "score": 88, + "score_reasons": [ + "support_count=5", + "confidence=0.95", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ] + }, + { + "opportunity_id": "skillops-531e13c1c6", + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "opportunity_type": "governance", + "action_type": "agents_update", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated privacy and approval signals should tighten durable operating guidance.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ] + }, + { + "opportunity_id": "skillops-709ef884f3", + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "opportunity_type": "report-experience", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated language preferences can improve existing report templates after approval.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ] + }, + { + "opportunity_id": "skillops-f9e43237aa", + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "opportunity_type": "artifact-discoverability", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ] + }, + { + "opportunity_id": "skillops-0a3a2de7c3", + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "opportunity_type": "quality-gate", + "action_type": "add_eval", + "decision": "proposal_review", + "priority": "medium", + "score": 74, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated verification and evidence requests should become focused checks.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ] + } + ], + "decision_policy": { + "schema_version": "1.0", + "contract": "skillops-opportunity-scoring", + "score_range": [ + 0, + 100 + ], + "score_bands": { + "85-100": "ready_for_approval_review", + "70-84": "proposal_review", + "50-69": "observe_more_evidence", + "0-49": "report_only_or_no_action" + }, + "action_types": [ + "add_eval", + "agents_update", + "patch_existing_skill", + "report_only" + ], + "writes_source_files": false, + "auto_patch_enabled": false, + "approval_required_for_writes": true + }, + "approval": { + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0 + }, + "release_state": { + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true + }, + "actions": [ + { + "key": "review-adaptation-proposals", + "priority": "high", + "action": "Review proposal-only adaptation items before preparing any approval ledger entry." + }, + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for the pending world-class ledger entries." + } + ], + "failures": [], + "source_reports": { + "patterns": "reports/user_patterns.json", + "proposals": "reports/adaptation_proposals.json", + "approval_ledger": "reports/adaptation_approval_ledger.json", + "regression": "reports/adaptation_regression_report.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + "evidence_consistency": "reports/evidence_consistency.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json" + }, + "artifacts": { + "json": "reports/skillops/daily/2026-06-17.json", + "markdown": "reports/skillops/daily/2026-06-17.md" + }, + "source_path": "reports/skillops/daily/2026-06-17.json" + }, + "weekly_curator": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17T08:58:54Z", + "skill_dir": ".", + "decision": "curator-review", + "week_id": "2026-W25", + "daily_report_count": 2, + "opportunity_count": 10, + "unique_opportunity_count": 5, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "top_score": 88, + "skill_count": 12, + "actionable_portfolio_issue_count": 0, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0, + "summary": { + "decision": "curator-review", + "week_id": "2026-W25", + "daily_report_count": 2, + "opportunity_count": 10, + "unique_opportunity_count": 5, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "top_score": 88, + "skill_count": 12, + "actionable_portfolio_issue_count": 0, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0 + }, + "curator_contract": { + "schema_version": "1.0", + "contract": "weekly-skillops-curator-report", + "cadence": "weekly", + "source_of_truth": [ + "reports/skillops/daily/*.json", + "reports/skill_atlas.json", + "reports/benchmark_reproducibility.json", + "reports/evidence_consistency.json", + "reports/world_class_evidence_ledger.json" + ], + "raw_content_stored": false, + "redacted_or_generated_evidence_only": true, + "proposal_only": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "approval_required_for_writes": true, + "counts_as_world_class_evidence": false + }, + "report_contract": { + "schema_version": "1.0", + "contract": "weekly-skillops-curator-report", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "week_id", + "daily_report_count", + "opportunity_count", + "unique_opportunity_count", + "ready_for_approval_review_count", + "proposal_review_count", + "observe_more_evidence_count", + "report_only_count", + "top_score", + "skill_count", + "actionable_portfolio_issue_count", + "release_lock_ready", + "evidence_consistency_ok", + "public_world_class_ready", + "world_class_pending_count", + "writes_source_files", + "auto_patch_enabled", + "failure_count" + ] + }, + "daily_reports": [ + "reports/skillops/daily/2026-06-16.json", + "reports/skillops/daily/2026-06-17.json" + ], + "opportunity_summary": { + "action_type_counts": { + "add_eval": 1, + "agents_update": 1, + "patch_existing_skill": 3 + }, + "decision_counts": { + "proposal_review": 4, + "ready_for_approval_review": 1 + }, + "risk_level_counts": { + "low": 3, + "medium": 2 + } + }, + "curator_queue": [ + { + "opportunity_id": "skillops-4ad91c18dd", + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "opportunity_type": "artifact-quality", + "action_type": "patch_existing_skill", + "decision": "ready_for_approval_review", + "priority": "medium", + "score": 88, + "score_reasons": [ + "support_count=5", + "confidence=0.95", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-531e13c1c6", + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "opportunity_type": "governance", + "action_type": "agents_update", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated privacy and approval signals should tighten durable operating guidance.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-709ef884f3", + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "opportunity_type": "report-experience", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated language preferences can improve existing report templates after approval.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-f9e43237aa", + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "opportunity_type": "artifact-discoverability", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-0a3a2de7c3", + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "opportunity_type": "quality-gate", + "action_type": "add_eval", + "decision": "proposal_review", + "priority": "medium", + "score": 74, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated verification and evidence requests should become focused checks.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ], + "daily_report_date": "2026-06-16" + } + ], + "portfolio": { + "skill_count": 12, + "actionable_issue_count": 0, + "route_collision_count": 5, + "actionable_route_collision_count": 0, + "owner_gap_count": 9, + "actionable_owner_gap_count": 0, + "stale_count": 10, + "actionable_stale_count": 0, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, + "no_route_opportunity_count": 0 + }, + "release_state": { + "release_lock_ready": false, + "evidence_consistency_ok": true, + "public_world_class_ready": false, + "world_class_pending_count": 4 + }, + "actions": [ + { + "key": "review-ready-opportunities", + "priority": "high", + "action": "Review the top ready-for-approval SkillOps opportunities before preparing any approval ledger entry." + }, + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for pending world-class ledger entries before public claims." + } + ], + "failures": [], + "source_reports": { + "daily_dir": "reports/skillops/daily", + "skill_atlas": "reports/skill_atlas.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", + "evidence_consistency": "reports/evidence_consistency.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json" + }, + "artifacts": { + "json": "reports/skillops/weekly/2026-W25.json", + "markdown": "reports/skillops/weekly/2026-W25.md" + }, + "source_path": "reports/skillops/weekly/2026-W25.json" + }, "review_waivers": { "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "summary": { "waiver_count": 0, "active_count": 0, "expired_count": 0, "invalid_count": 0, "covered_gate_count": 0, - "covered_gate_keys": [] + "covered_gate_keys": [], + "waiver_candidate_count": 2, + "waiverable_open_count": 1, + "non_waivable_count": 1 }, "policy": { "blocker_waivers_allowed": false, "minimum_reason_chars": 20, "expires_required": true, - "known_gate_keys": [ + "review_studio_gate_keys": [ + "architecture-maintainability", "context-budget", "intent-canvas", "operations-loop", "output-lab", "permission-gates", "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "known_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", "registry-audit", "release-notes", "runtime-matrix", "skill-atlas", "trigger-lab", "trust-report" + ], + "waiverable_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "non_waivable_gate_keys": [ + "review-waivers", + "world-class-evidence" ] }, "waivers": [], + "waiver_candidates": [ + { + "gate_key": "output-lab", + "label": "Output Lab", + "status": "needs-reviewer-decision", + "waiver_allowed": true, + "decision_options": [ + "accepted-risk", + "false-positive", + "temporary-exception" + ], + "risk_summary": "review pending 5; model-executed 10; output failures 0", + "required_review": [ + "Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.", + "Reviewer names the release scope and expiry date.", + "Reviewer links output_review_adjudication or output_execution evidence." + ], + "suggested_evidence": "reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md", + "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." + }, + { + "gate_key": "world-class-evidence", + "label": "World-Class Evidence", + "status": "cannot-waive", + "waiver_allowed": false, + "risk_summary": "4 pending evidence entries; 1 human pending; 3 external pending", + "required_review": [ + "Do not use a waiver to claim public world-class readiness.", + "Either submit accepted ledger evidence or state that this release does not claim world-class completion.", + "Keep claim guard active until ledger summary.ready_to_claim_world_class is true." + ], + "suggested_evidence": "reports/world_class_evidence_ledger.md", + "suggested_command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .", + "world_class_boundary": "Non-waivable completion boundary." + } + ], "failures": [], "warnings": [], "artifacts": { @@ -10763,7 +21180,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -10783,6 +21200,6419 @@ "markdown": "reports/review_annotations.md" } }, + "adaptation_proposals": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17T08:58:54Z", + "skill_dir": ".", + "source_patterns": "reports/user_patterns.json", + "pattern_count": 5, + "proposal_count": 5, + "apply_supported": true, + "failure_count": 0, + "proposal_only": true, + "approval_required": true, + "writes_repository_files": false, + "allowlisted_targets_required": true, + "target_file_sha256_required_for_apply": true, + "approval_draft_supported": true, + "rollback_required_for_apply": true, + "apply_command_available": true, + "summary": { + "pattern_count": 5, + "proposal_count": 5, + "apply_supported": true, + "failure_count": 0 + }, + "proposal_contract": { + "proposal_only": true, + "approval_required": true, + "writes_repository_files": false, + "allowlisted_targets_required": true, + "target_file_sha256_required_for_apply": true, + "approval_draft_supported": true, + "rollback_required_for_apply": true, + "apply_command_available": true + }, + "report_contract": { + "schema_version": "1.0", + "contract": "adaptation-proposals", + "top_level_mirrors_summary": true, + "top_level_mirrors_proposal_contract": true, + "summary_fields": [ + "pattern_count", + "proposal_count", + "apply_supported", + "failure_count" + ], + "proposal_contract_fields": [ + "proposal_only", + "approval_required", + "writes_repository_files", + "allowlisted_targets_required", + "target_file_sha256_required_for_apply", + "approval_draft_supported", + "rollback_required_for_apply", + "apply_command_available" + ], + "source_of_truth": [ + "summary", + "proposal_contract" + ] + }, + "proposals": [ + { + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "change_type": "report-default-language", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "low", + "reason": "2 redacted records matched repeated report-language signals.", + "support_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "suggested_changes": [ + "Keep user-facing report copy Simplified Chinese by default.", + "Expose English through the existing language switch instead of mixing languages in the default view." + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ], + "rollback_plan": "Revert report language template changes and rerun the overview verifier.", + "evidence_refs": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + } + ] + }, + { + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "change_type": "artifact-ui-polish", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "medium", + "reason": "5 redacted records matched repeated artifact-design signals.", + "support_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "suggested_changes": [ + "Prefer vertical narrative sections with limited two-column layouts only when content has enough width.", + "Keep charts inline SVG, with captions and stable responsive constraints." + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ], + "rollback_plan": "Restore the previous report renderer and regenerate the demo report.", + "evidence_refs": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-3", + "excerpt": "报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。" + } + ] + }, + { + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "change_type": "privacy-governance", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "low", + "reason": "2 redacted records matched repeated governance signals.", + "support_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "suggested_changes": [ + "Require explicit source paths for memory scans.", + "Generate proposals before any source patching.", + "Reserve automatic apply for a future approval ledger and rollback implementation." + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ], + "rollback_plan": "Remove the adaptive proposal artifacts and keep feedback/adoption drift as the only iteration inputs.", + "evidence_refs": [ + { + "record_id": "line-5", + "excerpt": "自适应升级必须先生成提案，不能直接自动修改源文件。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "change_type": "artifact-discoverability", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "low", + "reason": "2 redacted records matched repeated artifact-format signals.", + "support_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "suggested_changes": [ + "Include stable report paths in command output.", + "Document which artifacts are meant for human review." + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ], + "rollback_plan": "Revert CLI copy/documentation changes and keep artifact paths unchanged.", + "evidence_refs": [ + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "change_type": "quality-gate", + "status": "proposal-only", + "requires_approval": true, + "write_allowed_without_approval": false, + "risk_level": "medium", + "reason": "2 redacted records matched repeated quality-gate signals.", + "support_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "suggested_changes": [ + "Add focused verifier coverage for every new adaptive behavior.", + "Refresh Skill OS 2.0 coverage so planned, partial, and covered states remain visible." + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ], + "rollback_plan": "Revert the new verifier and coverage status updates, then regenerate coverage reports.", + "evidence_refs": [ + { + "record_id": "line-7", + "excerpt": "每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。" + }, + { + "record_id": "line-8", + "excerpt": "涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。" + } + ] + } + ], + "failures": [], + "artifacts": { + "json": "reports/adaptation_proposals.json", + "markdown": "reports/adaptation_proposals.md" + } + }, + "adaptation_approval_ledger": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15T00:00:00Z", + "summary": { + "approval_count": 0, + "active_approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0 + }, + "approval_contract": { + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true + }, + "entries": [], + "approval_count": 0, + "active_approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true, + "report_contract": { + "schema_version": "1.0", + "contract": "adaptation-approval-ledger", + "top_level_mirrors_summary": true, + "top_level_mirrors_approval_contract": true, + "summary_fields": [ + "approval_count", + "active_approval_count", + "pending_review_count", + "applied_count", + "rollback_count" + ], + "approval_contract_fields": [ + "approval_required", + "patch_sha256_required", + "allowlisted_targets_required", + "target_file_sha256_required", + "approval_draft_supported", + "dry_run_default", + "writes_repository_files_only_with_apply", + "rollback_required" + ], + "source_of_truth": [ + "summary", + "approval_contract" + ] + } + }, + "adaptation_regression": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-15T00:00:00Z", + "skill_dir": ".", + "summary": { + "apply_supported": true, + "attempt_count": 0, + "approval_draft_count": 0, + "applied_count": 0, + "dry_run_count": 0, + "rollback_count": 0, + "regression_run_count": 0, + "regression_pass_count": 0, + "failure_count": 0 + }, + "apply_contract": { + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true, + "safe_regression_commands_only": true, + "rollback_on_failure_default": true + }, + "attempts": [], + "failures": [], + "artifacts": { + "json": "reports/adaptation_regression_report.json", + "approval_ledger": "reports/adaptation_approval_ledger.json" + }, + "apply_supported": true, + "attempt_count": 0, + "approval_draft_count": 0, + "applied_count": 0, + "dry_run_count": 0, + "rollback_count": 0, + "regression_run_count": 0, + "regression_pass_count": 0, + "failure_count": 0, + "approval_required": true, + "patch_sha256_required": true, + "allowlisted_targets_required": true, + "target_file_sha256_required": true, + "approval_draft_supported": true, + "dry_run_default": true, + "writes_repository_files_only_with_apply": true, + "rollback_required": true, + "safe_regression_commands_only": true, + "rollback_on_failure_default": true, + "report_contract": { + "schema_version": "1.0", + "contract": "adaptation-regression-report", + "top_level_mirrors_summary": true, + "top_level_mirrors_apply_contract": true, + "summary_fields": [ + "apply_supported", + "attempt_count", + "approval_draft_count", + "applied_count", + "dry_run_count", + "rollback_count", + "regression_run_count", + "regression_pass_count", + "failure_count" + ], + "apply_contract_fields": [ + "approval_required", + "patch_sha256_required", + "allowlisted_targets_required", + "target_file_sha256_required", + "approval_draft_supported", + "dry_run_default", + "writes_repository_files_only_with_apply", + "rollback_required", + "safe_regression_commands_only", + "rollback_on_failure_default" + ], + "source_of_truth": [ + "summary", + "apply_contract" + ] + } + }, + "world_class_evidence_ledger": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "decision": "evidence-pending", + "ready_to_claim_world_class": false, + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "summary": { + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "report_contract": { + "schema_version": "1.0", + "contract": "world-class-evidence-ledger", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "ready_to_claim_world_class", + "ledger_entry_count", + "source_accepted_count", + "accepted_count", + "pending_count", + "human_pending_count", + "external_pending_count", + "submitted_entry_count", + "reviewer_approved_submission_count", + "missing_submission_count", + "invalid_submission_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "submitted_but_pending_count", + "source_accepted_without_valid_submission_count", + "overclaim_guard_active" + ], + "source_of_truth": "summary" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "source_accepted": false, + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4, + "evidence_requirement_count": 4 + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "ledger_counts_submission_as_completion": false, + "source_pass_requires_valid_submission": true + }, + "artifacts": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + "intake": "reports/world_class_evidence_intake.md" + } + }, + "world_class_evidence_intake": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "schema_present": true, + "ledger_entry_count": 4, + "template_count": 4, + "template_pass_count": 4, + "submission_count": 0, + "valid_submission_count": 0, + "invalid_submission_count": 0, + "operator_checklist_count": 4, + "operator_checklist_ready_count": 0, + "valid_packet_source_incomplete_count": 0, + "ready_for_external_collection": true, + "ready_for_ledger_review": false, + "ready_to_claim_world_class": false, + "overclaim_guard_active": true, + "decision": "awaiting-submissions" + }, + "schema": "evidence/world_class/intake.schema.json", + "templates": [ + { + "path": "evidence/world_class/templates/provider-holdout.intake.json", + "evidence_key": "provider-holdout", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 1, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + }, + { + "path": "evidence/world_class/templates/human-adjudication.intake.json", + "evidence_key": "human-adjudication", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 2, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + }, + { + "path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "evidence_key": "native-permission-enforcement", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 3, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + }, + { + "path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "evidence_key": "native-client-telemetry", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 2, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + } + ], + "submissions": [], + "operator_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": true, + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": false, + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": false, + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": false, + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_ledger": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + "pending_count": 4 + }, + "artifacts": { + "json": "reports/world_class_evidence_intake.json", + "markdown": "reports/world_class_evidence_intake.md" + } + }, + "world_class_evidence_preflight": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "precheck_count": 14, + "precheck_pass_count": 9, + "precheck_missing_count": 1, + "precheck_optional_count": 1, + "precheck_human_required_count": 1, + "precheck_external_required_count": 2, + "collection_ready_count": 1, + "collection_blocked_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, + "repair_ready_count": 0, + "repair_phase_counts": { + "unblock-access": 4, + "collect-source": 8 + }, + "next_repair_action_id": "human-adjudication-precheck-human-reviewer", + "next_repair_phase": "unblock-access", + "next_repair_owner": "human reviewer", + "next_repair_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "repair_counts_as_completion": false, + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false, + "pending_count": 4, + "ready_to_claim_world_class": false, + "credential_value_exposed": false, + "preflight_counts_as_evidence": false, + "decision": "collection-preflight-blocked" + }, + "status_counts": { + "blocked": 3, + "ready-for-human-review": 1 + }, + "precheck_status_counts": { + "pass": 9, + "missing": 1, + "optional": 1, + "human-required": 1, + "external-required": 2 + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": true, + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-api-key", + "label": "Provider credential", + "kind": "env_any", + "required": true, + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "secret_value_redacted": true, + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], + "status": "missing", + "actual": "not-set", + "set_count": 0, + "set_envs": [] + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "provider-specific model, or pass --provider-model" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + } + ] + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "ready-for-human-review", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices with required rationale.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication`.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + } + ], + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ] + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ] + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ] + } + ], + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-api-key", + "label": "Provider credential", + "kind": "env_any", + "required": true, + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "secret_value_redacted": true, + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], + "status": "missing", + "actual": "not-set", + "set_count": 0, + "set_envs": [] + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "provider-specific model, or pass --provider-model" + }, + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices with required rationale.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication`.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + }, + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + }, + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + }, + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "submissions": { + "directory": "evidence/world_class/submissions", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "submission_kit_prefill_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "preflight_counts_submission_as_completion": false, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 7, + "submission_ref_ready_count": 7, + "supporting_evidence_total_count": 31, + "supporting_evidence_ready_count": 31, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + } + }, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "operator_runbook": "reports/world_class_operator_runbook.json" + }, + "artifacts": { + "json": "reports/world_class_evidence_preflight.json", + "markdown": "reports/world_class_evidence_preflight.md", + "html": "reports/world_class_evidence_preflight.html" + } + }, + "world_class_submission_review": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "decision": "awaiting-submissions", + "ready_to_claim_world_class": false, + "review_item_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "valid_packet_source_incomplete_count": 0, + "ready_for_ledger_review_count": 0, + "fix_submission_count": 0, + "unmatched_submission_count": 0, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "review_counts_submission_as_completion": false, + "summary": { + "review_item_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "valid_packet_source_incomplete_count": 0, + "ready_for_ledger_review_count": 0, + "fix_submission_count": 0, + "unmatched_submission_count": 0, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "review_counts_submission_as_completion": false, + "decision": "awaiting-submissions" + }, + "report_contract": { + "schema_version": "1.0", + "contract": "world-class-submission-review", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "ready_to_claim_world_class", + "review_item_count", + "accepted_count", + "awaiting_submission_count", + "valid_packet_source_incomplete_count", + "ready_for_ledger_review_count", + "fix_submission_count", + "unmatched_submission_count", + "invalid_submission_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "review_counts_submission_as_completion" + ], + "source_of_truth": "summary" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "review_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": true, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "unmatched_submissions": [], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json" + }, + "artifacts": { + "json": "reports/world_class_submission_review.json", + "markdown": "reports/world_class_submission_review.md" + } + }, + "world_class_operator_runbook": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "pending_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "ready_for_ledger_review_count": 0, + "valid_packet_source_incomplete_count": 0, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false, + "coordination_step_count": 6, + "coordination_user_required_step_count": 6, + "coordination_pending_evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "coordination_counts_as_completion": false, + "release_gate_ready": false, + "release_gate_blocked_count": 4, + "release_gate_check_count": 5, + "release_gate_counts_as_completion": false, + "ready_to_claim_world_class": false, + "runbook_counts_as_completion": false, + "decision": "collect-evidence" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "runbook_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": true, + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "current": "model-executed 10; token-observed 10", + "execution_runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "blocked_source_check_count": 0, + "next_source_actions": [], + "repair_checklist": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ], + "repair_blocked_count": 1, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 1, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "current": "0/5 decisions; pending 5", + "execution_runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ] + }, + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "blocked_source_check_count": 5, + "next_source_actions": [ + "Record a reviewer choice and reason for every pair.", + "Every pair needs one valid human judgment.", + "Record reviewer and reviewed_at before adjudication can count.", + "Set reviewer_attestation only after choices are completed before opening the answer key.", + "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + ], + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "repair_blocked_count": 6, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "current": "native-enforced targets 0; installer-enforced targets 4", + "execution_runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "blocked_source_check_count": 1, + "next_source_actions": [ + "Collect real target-client or external runtime guard proof." + ], + "repair_checklist": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "repair_blocked_count": 2, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "current": "external source events 0; adoption samples 0", + "execution_runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + }, + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "blocked_source_check_count": 2, + "next_source_actions": [ + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." + ], + "repair_checklist": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "repair_blocked_count": 3, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + } + ], + "coordination_plan": [ + { + "step_id": "prepare-evidence-session", + "phase": "prepare", + "evidence_key": "", + "owner": "assistant + user", + "requires_user_input": true, + "user_action": "Confirm provider access, reviewer availability, target client path, and telemetry client path before collection starts.", + "assistant_action": "Run preflight and prepare submission drafts without accepting them as evidence.", + "external_dependency": "User-selected provider, reviewer, and real client surfaces.", + "command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "pass_condition": "Preflight lists the same pending evidence keys and no credential values are printed.", + "artifact": "reports/world_class_evidence_preflight.json", + "privacy_boundary": "Drafts and preflight are planning artifacts only; they do not count as evidence.", + "counts_as_completion": false + }, + { + "step_id": "collect-provider-holdout", + "phase": "provider-holdout", + "evidence_key": "provider-holdout", + "owner": "assistant + operator with provider credentials", + "requires_user_input": true, + "user_action": "Provide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.", + "assistant_action": "Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.", + "external_dependency": "Valid provider credentials and a live provider endpoint.", + "command": "python3 scripts/yao.py output-exec . --provider-runner --provider-model --timeout-seconds 60", + "pass_condition": "reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.", + "artifact": "reports/output_execution_runs.json", + "privacy_boundary": "Commit aggregate metadata only; do not commit API keys, raw prompts, raw outputs, or provider request payloads.", + "counts_as_completion": false + }, + { + "step_id": "collect-human-adjudication", + "phase": "human-review", + "evidence_key": "human-adjudication", + "owner": "human reviewer + assistant", + "requires_user_input": true, + "user_action": "Open the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved.", + "assistant_action": "Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet.", + "external_dependency": "A real human reviewer who can make blind A/B judgments.", + "command": "python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .", + "pass_condition": "reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true.", + "artifact": "reports/output_review_adjudication.json", + "privacy_boundary": "Store reviewer choices, reasons, hashes, and attestations; do not store raw prompts or answer-key exposure before decisions.", + "counts_as_completion": false + }, + { + "step_id": "collect-native-permission-enforcement", + "phase": "native-permission", + "evidence_key": "native-permission-enforcement", + "owner": "target client or installer integrator + assistant", + "requires_user_input": true, + "user_action": "Select a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback.", + "assistant_action": "Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet.", + "external_dependency": "A real target runtime, extension, client, or installer guard that can block undeclared capabilities.", + "command": "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "pass_condition": "reports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0.", + "artifact": "reports/runtime_permission_probes.json", + "privacy_boundary": "Do not mark metadata fallback as native enforcement; keep residual risks visible for fallback targets.", + "counts_as_completion": false + }, + { + "step_id": "collect-native-client-telemetry", + "phase": "native-telemetry", + "evidence_key": "native-client-telemetry", + "owner": "real client integrator + assistant", + "requires_user_input": true, + "user_action": "Install the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event.", + "assistant_action": "Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet.", + "external_dependency": "A real client process that emits metadata-only telemetry through the native host.", + "command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source external", + "pass_condition": "reports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0.", + "artifact": "reports/adoption_drift_report.json", + "privacy_boundary": "Telemetry must remain metadata-only; do not store raw prompts, outputs, transcripts, notes, or messages.", + "counts_as_completion": false + }, + { + "step_id": "review-and-release-gate", + "phase": "release-gate", + "evidence_key": "", + "owner": "assistant + ledger reviewer", + "requires_user_input": true, + "user_action": "Approve only validated evidence packets and confirm the release wording after the claim guard passes.", + "assistant_action": "Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish.", + "external_dependency": "Accepted evidence packets for every pending world-class key.", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-test", + "pass_condition": "Ledger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes.", + "artifact": "reports/world_class_evidence_ledger.json", + "privacy_boundary": "Release evidence may include aggregate reports and hashes only; raw content and credentials remain out of git.", + "counts_as_completion": false + } + ], + "release_gate": { + "ready": false, + "decision": "blocked-until-evidence-accepted", + "check_count": 5, + "blocked_count": 4, + "checks": [ + { + "key": "world_class_ledger_ready", + "label": "World-class ledger ready", + "passed": false, + "current": "evidence-pending", + "expected": "ready_to_claim_world_class == true", + "artifact": "reports/world_class_evidence_ledger.json" + }, + { + "key": "claim_guard_clean", + "label": "Claim guard clean", + "passed": false, + "current": "violations 0; ledger ready False", + "expected": "violation_count == 0 and ledger_ready_to_claim_world_class == true", + "artifact": "reports/world_class_claim_guard.json" + }, + { + "key": "benchmark_public_claim_ready", + "label": "Benchmark public claim ready", + "passed": false, + "current": "public_claim_ready False", + "expected": "public_claim_ready == true", + "artifact": "reports/benchmark_reproducibility.json" + }, + { + "key": "review_studio_clean", + "label": "Review Studio clean", + "passed": false, + "current": "blockers 0; warnings 3", + "expected": "blocker_count == 0 and warning_count == 0", + "artifact": "reports/review-studio.json" + }, + { + "key": "evidence_consistency_clean", + "label": "Evidence consistency clean", + "passed": true, + "current": "consistent", + "expected": "decision == consistent and fail_count == 0", + "artifact": "reports/evidence_consistency.json" + } + ], + "blocked_checks": [ + { + "key": "world_class_ledger_ready", + "label": "World-class ledger ready", + "passed": false, + "current": "evidence-pending", + "expected": "ready_to_claim_world_class == true", + "artifact": "reports/world_class_evidence_ledger.json" + }, + { + "key": "claim_guard_clean", + "label": "Claim guard clean", + "passed": false, + "current": "violations 0; ledger ready False", + "expected": "violation_count == 0 and ledger_ready_to_claim_world_class == true", + "artifact": "reports/world_class_claim_guard.json" + }, + { + "key": "benchmark_public_claim_ready", + "label": "Benchmark public claim ready", + "passed": false, + "current": "public_claim_ready False", + "expected": "public_claim_ready == true", + "artifact": "reports/benchmark_reproducibility.json" + }, + { + "key": "review_studio_clean", + "label": "Review Studio clean", + "passed": false, + "current": "blockers 0; warnings 3", + "expected": "blocker_count == 0 and warning_count == 0", + "artifact": "reports/review-studio.json" + } + ], + "final_manual_check": "Run make ci-test in a clean worktree and verify GitHub Actions before converting the PR out of Draft.", + "counts_as_completion": false + }, + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "preflight": "reports/world_class_evidence_preflight.json", + "submission_review": "reports/world_class_submission_review.json", + "claim_guard": "reports/world_class_claim_guard.json" + }, + "artifacts": { + "json": "reports/world_class_operator_runbook.json", + "markdown": "reports/world_class_operator_runbook.md", + "html": "reports/world_class_operator_runbook.html" + } + }, + "world_class_claim_guard": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "ledger_ready_to_claim_world_class": false, + "ledger_pending_count": 4, + "claim_surface_count": 182, + "json_claim_surface_count": 89, + "metadata_claim_surface_count": 90, + "package_claim_surface_count": 17, + "violation_count": 0, + "overclaim_guard_active": true, + "decision": "claim-guard-pass-evidence-pending" + }, + "rules": [ + { + "key": "ready-to-claim-true", + "reason": "ready_to_claim_world_class cannot be true while ledger evidence is pending.", + "pattern": "ready[\\s_-]*to[\\s_-]*claim[\\s_-]*world[\\s_-]*class\\s*[:=]\\s*`?true`?" + }, + { + "key": "world-class-ready-true", + "reason": "world-class readiness cannot be claimed before accepted external and human evidence exists.", + "pattern": "(public\\s+)?world[\\s_-]*class[\\s_-]*ready\\s*[:=]\\s*`?true`?" + }, + { + "key": "json-world-class-ready-true", + "reason": "machine-readable claim fields must remain false until the ledger is ready.", + "pattern": "\"(?:ready_to_claim_world_class|world_class_ready|public_world_class_ready)\"\\s*:\\s*true" + }, + { + "key": "completion-phrase", + "reason": "completion language is blocked until the world-class ledger is accepted.", + "pattern": "(public\\s+)?world[\\s_-]*class\\s+(?:complete|completed|achieved|certified)" + }, + { + "key": "zh-completion-phrase", + "reason": "中文完成态表述必须等到 ledger ready 后才能出现。", + "pattern": "世界级(?:已完成|完成|已达成|达成|准备就绪|可声明|认证通过)" + } + ], + "scanned_surfaces": [ + { + "path": "README.md", + "violation_count": 0 + }, + { + "path": "SKILL.md", + "violation_count": 0 + }, + { + "path": "agents/interface.yaml", + "violation_count": 0 + }, + { + "path": "dist/manifest.json", + "violation_count": 0 + }, + { + "path": "dist/targets/claude/README.md", + "violation_count": 0 + }, + { + "path": "dist/targets/claude/adapter.json", + "violation_count": 0 + }, + { + "path": "dist/targets/generic/adapter.json", + "violation_count": 0 + }, + { + "path": "dist/targets/openai/adapter.json", + "violation_count": 0 + }, + { + "path": "dist/targets/vscode/README.md", + "violation_count": 0 + }, + { + "path": "dist/targets/vscode/adapter.json", + "violation_count": 0 + }, + { + "path": "docs/README.fr-FR.md", + "violation_count": 0 + }, + { + "path": "docs/README.ja-JP.md", + "violation_count": 0 + }, + { + "path": "docs/README.ru-RU.md", + "violation_count": 0 + }, + { + "path": "docs/README.zh-CN.md", + "violation_count": 0 + }, + { + "path": "docs/migration-v2.md", + "violation_count": 0 + }, + { + "path": "evidence/world_class/README.md", + "violation_count": 0 + }, + { + "path": "evidence/world_class/intake.schema.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/human-adjudication.intake.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/provider-holdout.intake.json", + "violation_count": 0 + }, + { + "path": "manifest.json", + "violation_count": 0 + }, + { + "path": "registry/examples/yao-meta-skill-1.0.0.json", + "violation_count": 0 + }, + { + "path": "registry/index.json", + "violation_count": 0 + }, + { + "path": "registry/index.schema.json", + "violation_count": 0 + }, + { + "path": "registry/package.schema.json", + "violation_count": 0 + }, + { + "path": "registry/packages/yao-meta-skill.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_approval_ledger.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_proposals.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_proposals.md", + "violation_count": 0 + }, + { + "path": "reports/adaptation_regression_report.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_regression_report.md", + "violation_count": 0 + }, + { + "path": "reports/adoption_drift_report.json", + "violation_count": 0 + }, + { + "path": "reports/adoption_drift_report.md", + "violation_count": 0 + }, + { + "path": "reports/architecture_maintainability.json", + "violation_count": 0 + }, + { + "path": "reports/architecture_maintainability.md", + "violation_count": 0 + }, + { + "path": "reports/artifact-design-profile.json", + "violation_count": 0 + }, + { + "path": "reports/artifact-design-profile.md", + "violation_count": 0 + }, + { + "path": "reports/baseline-compare.json", + "violation_count": 0 + }, + { + "path": "reports/baseline-compare.md", + "violation_count": 0 + }, + { + "path": "reports/benchmark_methodology.md", + "violation_count": 0 + }, + { + "path": "reports/benchmark_reproducibility.json", + "violation_count": 0 + }, + { + "path": "reports/benchmark_reproducibility.md", + "violation_count": 0 + }, + { + "path": "reports/candidate_registry.json", + "violation_count": 0 + }, + { + "path": "reports/candidate_registry.md", + "violation_count": 0 + }, + { + "path": "reports/compiled_targets.json", + "violation_count": 0 + }, + { + "path": "reports/compiled_targets.md", + "violation_count": 0 + }, + { + "path": "reports/conformance_matrix.json", + "violation_count": 0 + }, + { + "path": "reports/conformance_matrix.md", + "violation_count": 0 + }, + { + "path": "reports/context_budget.json", + "violation_count": 0 + }, + { + "path": "reports/context_budget.md", + "violation_count": 0 + }, + { + "path": "reports/context_budget_summary.json", + "violation_count": 0 + }, + { + "path": "reports/description_drift_history.md", + "violation_count": 0 + }, + { + "path": "reports/description_optimization.json", + "violation_count": 0 + }, + { + "path": "reports/description_optimization.md", + "violation_count": 0 + }, + { + "path": "reports/description_optimization_suite.json", + "violation_count": 0 + }, + { + "path": "reports/description_optimization_suite.md", + "violation_count": 0 + }, + { + "path": "reports/eval_suite.json", + "violation_count": 0 + }, + { + "path": "reports/evidence_consistency.json", + "violation_count": 0 + }, + { + "path": "reports/evidence_consistency.md", + "violation_count": 0 + }, + { + "path": "reports/family_summary.md", + "violation_count": 0 + }, + { + "path": "reports/feishu-ranking-pattern-analysis-2026-04-01.md", + "violation_count": 0 + }, + { + "path": "reports/github-benchmark-scan.json", + "violation_count": 0 + }, + { + "path": "reports/github-benchmark-scan.md", + "violation_count": 0 + }, + { + "path": "reports/governance_score.json", + "violation_count": 0 + }, + { + "path": "reports/install_simulation.json", + "violation_count": 0 + }, + { + "path": "reports/install_simulation.md", + "violation_count": 0 + }, + { + "path": "reports/intent-confidence.json", + "violation_count": 0 + }, + { + "path": "reports/intent-confidence.md", + "violation_count": 0 + }, + { + "path": "reports/intent-context.json", + "violation_count": 0 + }, + { + "path": "reports/intent-dialogue.json", + "violation_count": 0 + }, + { + "path": "reports/intent-dialogue.md", + "violation_count": 0 + }, + { + "path": "reports/iteration-directions.json", + "violation_count": 0 + }, + { + "path": "reports/iteration-directions.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/governed-incident-command/bundle.json", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/governed-incident-command/bundle.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/governed-incident-command/review.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/team-frontend-review/bundle.json", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/team-frontend-review/bundle.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/team-frontend-review/review.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/yao-meta-skill/bundle.json", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/yao-meta-skill/bundle.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/yao-meta-skill/review.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_ledger.md", + "violation_count": 0 + }, + { + "path": "reports/output-risk-profile.json", + "violation_count": 0 + }, + { + "path": "reports/output-risk-profile.md", + "violation_count": 0 + }, + { + "path": "reports/output_blind_answer_key.json", + "violation_count": 0 + }, + { + "path": "reports/output_blind_review_pack.json", + "violation_count": 0 + }, + { + "path": "reports/output_blind_review_pack.md", + "violation_count": 0 + }, + { + "path": "reports/output_execution_runs.json", + "violation_count": 0 + }, + { + "path": "reports/output_execution_runs.md", + "violation_count": 0 + }, + { + "path": "reports/output_quality_scorecard.json", + "violation_count": 0 + }, + { + "path": "reports/output_quality_scorecard.md", + "violation_count": 0 + }, + { + "path": "reports/output_review_adjudication.json", + "violation_count": 0 + }, + { + "path": "reports/output_review_adjudication.md", + "violation_count": 0 + }, + { + "path": "reports/output_review_decisions.json", + "violation_count": 0 + }, + { + "path": "reports/output_review_kit.html", + "violation_count": 0 + }, + { + "path": "reports/output_review_kit.json", + "violation_count": 0 + }, + { + "path": "reports/output_review_kit.md", + "violation_count": 0 + }, + { + "path": "reports/package_verification.json", + "violation_count": 0 + }, + { + "path": "reports/package_verification.md", + "violation_count": 0 + }, + { + "path": "reports/portability_score.json", + "violation_count": 0 + }, + { + "path": "reports/portability_score.md", + "violation_count": 0 + }, + { + "path": "reports/promotion_decisions.json", + "violation_count": 0 + }, + { + "path": "reports/promotion_decisions.md", + "violation_count": 0 + }, + { + "path": "reports/prompt-quality-profile.json", + "violation_count": 0 + }, + { + "path": "reports/prompt-quality-profile.md", + "violation_count": 0 + }, + { + "path": "reports/python_compatibility.json", + "violation_count": 0 + }, + { + "path": "reports/python_compatibility.md", + "violation_count": 0 + }, + { + "path": "reports/reference-scan.json", + "violation_count": 0 + }, + { + "path": "reports/reference-scan.md", + "violation_count": 0 + }, + { + "path": "reports/reference-synthesis.json", + "violation_count": 0 + }, + { + "path": "reports/reference-synthesis.md", + "violation_count": 0 + }, + { + "path": "reports/registry_audit.json", + "violation_count": 0 + }, + { + "path": "reports/registry_audit.md", + "violation_count": 0 + }, + { + "path": "reports/regression_history.md", + "violation_count": 0 + }, + { + "path": "reports/review-studio.html", + "violation_count": 0 + }, + { + "path": "reports/review-studio.json", + "violation_count": 0 + }, + { + "path": "reports/review-viewer.html", + "violation_count": 0 + }, + { + "path": "reports/review-viewer.json", + "violation_count": 0 + }, + { + "path": "reports/review_annotations.json", + "violation_count": 0 + }, + { + "path": "reports/review_annotations.md", + "violation_count": 0 + }, + { + "path": "reports/review_waivers.json", + "violation_count": 0 + }, + { + "path": "reports/review_waivers.md", + "violation_count": 0 + }, + { + "path": "reports/route_scorecard.json", + "violation_count": 0 + }, + { + "path": "reports/route_scorecard.md", + "violation_count": 0 + }, + { + "path": "reports/runtime_permission_probes.json", + "violation_count": 0 + }, + { + "path": "reports/runtime_permission_probes.md", + "violation_count": 0 + }, + { + "path": "reports/security_trust_report.json", + "violation_count": 0 + }, + { + "path": "reports/security_trust_report.md", + "violation_count": 0 + }, + { + "path": "reports/skill-interpretation.html", + "violation_count": 0 + }, + { + "path": "reports/skill-interpretation.json", + "violation_count": 0 + }, + { + "path": "reports/skill-os-2-review.md", + "violation_count": 0 + }, + { + "path": "reports/skill-overview-v2-plan.md", + "violation_count": 0 + }, + { + "path": "reports/skill-overview.html", + "violation_count": 0 + }, + { + "path": "reports/skill-overview.json", + "violation_count": 0 + }, + { + "path": "reports/skill_atlas.html", + "violation_count": 0 + }, + { + "path": "reports/skill_atlas.json", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_audit.json", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_audit.md", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_coverage.json", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_coverage.md", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-16.json", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-16.md", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-17.json", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-17.md", + "violation_count": 0 + }, + { + "path": "reports/skillops/weekly/2026-W25.json", + "violation_count": 0 + }, + { + "path": "reports/skillops/weekly/2026-W25.md", + "violation_count": 0 + }, + { + "path": "reports/system-model.json", + "violation_count": 0 + }, + { + "path": "reports/system-model.md", + "violation_count": 0 + }, + { + "path": "reports/telemetry_hook_recipes.json", + "violation_count": 0 + }, + { + "path": "reports/telemetry_hook_recipes.md", + "violation_count": 0 + }, + { + "path": "reports/upgrade_check.json", + "violation_count": 0 + }, + { + "path": "reports/upgrade_check.md", + "violation_count": 0 + }, + { + "path": "reports/user_patterns.json", + "violation_count": 0 + }, + { + "path": "reports/user_patterns.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_claim_guard.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_intake.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_intake.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_ledger.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_ledger.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_plan.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_plan.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.html", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.html", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_submission_review.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_submission_review.md", + "violation_count": 0 + }, + { + "path": "reports/yao-deploy-bridge-skill-research-plan.md", + "violation_count": 0 + }, + { + "path": "security/dependency_policy.md", + "violation_count": 0 + }, + { + "path": "security/network_policy.json", + "violation_count": 0 + }, + { + "path": "security/network_policy.md", + "violation_count": 0 + }, + { + "path": "security/permission_policy.json", + "violation_count": 0 + }, + { + "path": "security/permission_policy.md", + "violation_count": 0 + }, + { + "path": "security/script_policy.md", + "violation_count": 0 + }, + { + "path": "security/secret_policy.md", + "violation_count": 0 + }, + { + "path": "security/trust_policy.md", + "violation_count": 0 + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "violation_count": 0 + }, + { + "path": "skill-ir/schema.json", + "violation_count": 0 + } + ], + "violations": [], + "source_ledger": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md" + }, + "artifacts": { + "json": "reports/world_class_claim_guard.json", + "markdown": "reports/world_class_claim_guard.md" + } + }, "registry": { "ok": true, "schema_version": "2.0", @@ -10792,12 +27622,13 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "maturity": "governed", "owner": "Yao Team", @@ -10805,8 +27636,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f", - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { "openai": "pass", @@ -10837,7 +27668,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -10858,10 +27689,11 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] }, @@ -10877,15 +27709,15 @@ "package_verification": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "package_dir": "dist", "summary": { - "target_count": 3, - "adapter_count": 3, + "target_count": 4, + "adapter_count": 4, "archive_present": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", - "archive_entry_count": 488, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "archive_entry_count": 689, "failure_count": 0, "warning_count": 0 }, @@ -11300,6 +28132,141 @@ "status": "pass", "detail": "generic adapter includes field: portability_profile" }, + { + "id": "vscode-adapter", + "status": "pass", + "detail": "Adapter exists for target: vscode" + }, + { + "id": "vscode-field-name", + "status": "pass", + "detail": "vscode adapter includes field: name" + }, + { + "id": "vscode-field-description", + "status": "pass", + "detail": "vscode adapter includes field: description" + }, + { + "id": "vscode-field-version", + "status": "pass", + "detail": "vscode adapter includes field: version" + }, + { + "id": "vscode-field-display_name", + "status": "pass", + "detail": "vscode adapter includes field: display_name" + }, + { + "id": "vscode-field-short_description", + "status": "pass", + "detail": "vscode adapter includes field: short_description" + }, + { + "id": "vscode-field-default_prompt", + "status": "pass", + "detail": "vscode adapter includes field: default_prompt" + }, + { + "id": "vscode-field-job_to_be_done", + "status": "pass", + "detail": "vscode adapter includes field: job_to_be_done" + }, + { + "id": "vscode-field-ir_source", + "status": "pass", + "detail": "vscode adapter includes field: ir_source" + }, + { + "id": "vscode-field-ir_schema_version", + "status": "pass", + "detail": "vscode adapter includes field: ir_schema_version" + }, + { + "id": "vscode-field-semantic_contract", + "status": "pass", + "detail": "vscode adapter includes field: semantic_contract" + }, + { + "id": "vscode-field-semantic_parity", + "status": "pass", + "detail": "vscode adapter includes field: semantic_parity" + }, + { + "id": "vscode-field-compiler", + "status": "pass", + "detail": "vscode adapter includes field: compiler" + }, + { + "id": "vscode-field-compiled_contract", + "status": "pass", + "detail": "vscode adapter includes field: compiled_contract" + }, + { + "id": "vscode-field-permission_contract", + "status": "pass", + "detail": "vscode adapter includes field: permission_contract" + }, + { + "id": "vscode-field-target_permission_contract", + "status": "pass", + "detail": "vscode adapter includes field: target_permission_contract" + }, + { + "id": "vscode-field-target_native_contract", + "status": "pass", + "detail": "vscode adapter includes field: target_native_contract" + }, + { + "id": "vscode-field-target_transform", + "status": "pass", + "detail": "vscode adapter includes field: target_transform" + }, + { + "id": "vscode-field-canonical_metadata", + "status": "pass", + "detail": "vscode adapter includes field: canonical_metadata" + }, + { + "id": "vscode-field-canonical_format", + "status": "pass", + "detail": "vscode adapter includes field: canonical_format" + }, + { + "id": "vscode-field-activation_mode", + "status": "pass", + "detail": "vscode adapter includes field: activation_mode" + }, + { + "id": "vscode-field-execution_context", + "status": "pass", + "detail": "vscode adapter includes field: execution_context" + }, + { + "id": "vscode-field-shell", + "status": "pass", + "detail": "vscode adapter includes field: shell" + }, + { + "id": "vscode-field-trust_level", + "status": "pass", + "detail": "vscode adapter includes field: trust_level" + }, + { + "id": "vscode-field-remote_inline_execution", + "status": "pass", + "detail": "vscode adapter includes field: remote_inline_execution" + }, + { + "id": "vscode-field-degradation_strategy", + "status": "pass", + "detail": "vscode adapter includes field: degradation_strategy" + }, + { + "id": "vscode-field-portability_profile", + "status": "pass", + "detail": "vscode adapter includes field: portability_profile" + }, { "id": "openai-file-targets/openai/adapter.json", "status": "pass", @@ -11325,6 +28292,16 @@ "status": "pass", "detail": "Package contains targets/generic/adapter.json" }, + { + "id": "vscode-file-targets/vscode/adapter.json", + "status": "pass", + "detail": "Package contains targets/vscode/adapter.json" + }, + { + "id": "vscode-file-targets/vscode/README.md", + "status": "pass", + "detail": "Package contains targets/vscode/README.md" + }, { "id": "archive-safe-paths", "status": "pass", @@ -11348,7 +28325,7 @@ { "id": "archive-excludes-generated", "status": "pass", - "detail": "Archive excludes generated dist/ and tests/tmp* contents" + "detail": "Archive excludes generated dist/, .previews/, and tests/tmp* contents" }, { "id": "registry-ok", @@ -11379,6 +28356,11 @@ "id": "registry-compat-generic", "status": "pass", "detail": "Registry compatibility is reviewable for target: generic" + }, + { + "id": "registry-compat-vscode", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: vscode" } ], "failures": [], @@ -11395,17 +28377,21 @@ "schema_version": "2.0", "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 488, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, "interface_loaded": true, - "adapter_count": 3, + "adapter_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, "install_root_is_temp": false, "failure_count": 0, "warning_count": 0 @@ -11414,7 +28400,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -11500,15 +28486,170 @@ "id": "adapter-openai-name", "status": "pass", "detail": "openai adapter name matches package manifest" + }, + { + "id": "adapter-vscode", + "status": "pass", + "detail": "vscode adapter is readable after package install simulation" + }, + { + "id": "adapter-vscode-name", + "status": "pass", + "detail": "vscode adapter name matches package manifest" + }, + { + "id": "permission-policy-load", + "status": "pass", + "detail": "Installed permission policy is readable" + }, + { + "id": "permission-claude-contract", + "status": "pass", + "detail": "claude adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-claude-file_write-approved", + "status": "pass", + "detail": "claude capability file_write has active reviewer approval" + }, + { + "id": "permission-claude-file_write-target-enforcement", + "status": "pass", + "detail": "claude capability file_write has target enforcement note" + }, + { + "id": "permission-claude-network-approved", + "status": "pass", + "detail": "claude capability network has active reviewer approval" + }, + { + "id": "permission-claude-network-target-enforcement", + "status": "pass", + "detail": "claude capability network has target enforcement note" + }, + { + "id": "permission-claude-subprocess-approved", + "status": "pass", + "detail": "claude capability subprocess has active reviewer approval" + }, + { + "id": "permission-claude-subprocess-target-enforcement", + "status": "pass", + "detail": "claude capability subprocess has target enforcement note" + }, + { + "id": "permission-generic-contract", + "status": "pass", + "detail": "generic adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-generic-file_write-approved", + "status": "pass", + "detail": "generic capability file_write has active reviewer approval" + }, + { + "id": "permission-generic-file_write-target-enforcement", + "status": "pass", + "detail": "generic capability file_write has target enforcement note" + }, + { + "id": "permission-generic-network-approved", + "status": "pass", + "detail": "generic capability network has active reviewer approval" + }, + { + "id": "permission-generic-network-target-enforcement", + "status": "pass", + "detail": "generic capability network has target enforcement note" + }, + { + "id": "permission-generic-subprocess-approved", + "status": "pass", + "detail": "generic capability subprocess has active reviewer approval" + }, + { + "id": "permission-generic-subprocess-target-enforcement", + "status": "pass", + "detail": "generic capability subprocess has target enforcement note" + }, + { + "id": "permission-openai-contract", + "status": "pass", + "detail": "openai adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-openai-file_write-approved", + "status": "pass", + "detail": "openai capability file_write has active reviewer approval" + }, + { + "id": "permission-openai-file_write-target-enforcement", + "status": "pass", + "detail": "openai capability file_write has target enforcement note" + }, + { + "id": "permission-openai-network-approved", + "status": "pass", + "detail": "openai capability network has active reviewer approval" + }, + { + "id": "permission-openai-network-target-enforcement", + "status": "pass", + "detail": "openai capability network has target enforcement note" + }, + { + "id": "permission-openai-subprocess-approved", + "status": "pass", + "detail": "openai capability subprocess has active reviewer approval" + }, + { + "id": "permission-openai-subprocess-target-enforcement", + "status": "pass", + "detail": "openai capability subprocess has target enforcement note" + }, + { + "id": "permission-vscode-contract", + "status": "pass", + "detail": "vscode adapter exposes target permission contract for installer enforcement" + }, + { + "id": "permission-vscode-file_write-approved", + "status": "pass", + "detail": "vscode capability file_write has active reviewer approval" + }, + { + "id": "permission-vscode-file_write-target-enforcement", + "status": "pass", + "detail": "vscode capability file_write has target enforcement note" + }, + { + "id": "permission-vscode-network-approved", + "status": "pass", + "detail": "vscode capability network has active reviewer approval" + }, + { + "id": "permission-vscode-network-target-enforcement", + "status": "pass", + "detail": "vscode capability network has target enforcement note" + }, + { + "id": "permission-vscode-subprocess-approved", + "status": "pass", + "detail": "vscode capability subprocess has active reviewer approval" + }, + { + "id": "permission-vscode-subprocess-target-enforcement", + "status": "pass", + "detail": "vscode capability subprocess has target enforcement note" } ], "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } }, "upgrade_check": { @@ -11531,14 +28672,15 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "summary": { "declared_bump": "minor", "recommended_bump": "minor", "breaking_change_count": 0, - "added_target_count": 1, + "added_target_count": 2, "metadata_change_count": 1, "checksum_change_count": 2, "failure_count": 0, @@ -11547,7 +28689,8 @@ "upgrade_diff": { "name_changed": false, "added_targets": [ - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "removed_targets": [], "compatibility_changes": [ @@ -11574,25 +28717,25 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ { "field": "archive_sha256", "from": "", - "to": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "to": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "to": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] }, "release_notes": [ "Recommended version bump: minor.", - "Added targets: agent-skills-compatible.", + "Added targets: agent-skills-compatible, vscode.", "Package or archive checksum changed; reviewers should verify package artifacts before release.", "Package verification evidence: reports/package_verification.md." ], @@ -11601,8 +28744,8 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } }, "manifest": { @@ -11619,7 +28762,8 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "factory_components": [ "templates", @@ -11630,10 +28774,9 @@ }, "frontmatter": { "name": "yao-meta-skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "metadata": { - "author": "Yao Team", - "philosophy": "structured design, evaluation loop, template ergonomics, operational packaging" + "author": "Yao Team" } }, "interface": { @@ -11647,7 +28790,8 @@ "adapter_targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "activation": { "mode": "manual", @@ -11665,7 +28809,8 @@ "degradation": { "openai": "metadata-adapter", "claude": "neutral-source-plus-adapter", - "generic": "neutral-source" + "generic": "neutral-source", + "vscode": "agent-skills-source-with-vscode-notes" } } } diff --git a/reports/review-viewer.html b/reports/review-viewer.html index 2094b053..dc661185 100644 --- a/reports/review-viewer.html +++ b/reports/review-viewer.html @@ -5,182 +5,182 @@ Yao Meta Skill Review Viewer @@ -188,29 +188,29 @@

Review Viewer

Yao Meta Skill

-

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

maturity: governed archetype: governed format: agent-skills updated: 2026-03-31 - intent confidence: 30 / 100 + intent confidence: 100 / 100

Architecture at a glance

-
Inputs
workflow, prompt, transcript, docs, or notes
Boundary
Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Logic
Decide whether the request should become a skill and choose the lightest fit.; Capture job, output, exclusions, constraints, and standards.; Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
Usage
Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.; Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Next
Review the top iteration directions before growing the package.
+
Inputs
workflow, prompt, transcript, docs, or notes
Boundary
Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
Logic
For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.; Capture job, output, exclusions, constraints, standards, and the lightest fit.; Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
Usage
Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.; Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
Next
Review the top iteration directions before growing the package.

Core logic

-
  • Decide whether the request should become a skill and choose the lightest fit.
  • Capture job, output, exclusions, constraints, and standards.
  • Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
  • Write the `description` early and test route quality before expanding the package.
  • Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.
+
  • For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.
  • Capture job, output, exclusions, constraints, standards, and the lightest fit.
  • Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
  • Write `description` early, test route quality, then add only earned folders and gates.
  • Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful.

How to use it

-
  • Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  • Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
+
  • Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  • Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
@@ -221,7 +221,7 @@

Why this package is strong

-
  • 触发面保持精简，并锚定在 frontmatter description。
  • 已打包 agents/interface.yaml，便于后续做跨平台适配。
  • 长指导被拆到 references 中，入口文件可以保持轻量。
  • 确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。
  • 包内包含可随 Skill 迁移的质量门禁或触发检查。
+
  • 触发面保持精简，并锚定在 frontmatter description。
  • 已生成 Skill IR，核心语义可先于平台打包被审查和迁移。
  • 已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。
  • 已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。
  • 已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。
  • 已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。
@@ -232,20 +232,20 @@

Compare view

-

Winner: Current

{'priority': ['fewest false positives', 'fewest false negatives', 'highest near-neighbor pass rate', 'highest negative pass rate', 'highest precision', 'highest recall', 'shortest description']}

VariantTokensDev ErrorsHoldout ErrorsStrategy
Baseline800existing
Current6500current
Current6500current
+

Winner: Current

{'priority': ['fewest false positives', 'fewest false negatives', 'highest near-neighbor pass rate', 'highest negative pass rate', 'highest precision', 'highest recall', 'shortest description']}

VariantTokensDev ErrorsHoldout ErrorsStrategy
Baseline800existing
Current5300current
Current5300current

Variant diff studio

-

Baseline

existing

Create and improve agent skills.

tokens 8 (0)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create and improve agent skills.

Drops from baseline

No dropped cues.

Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

tokens 65 (+57)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

Drops from baseline

  • Create and improve agent skills.

Winner — Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

tokens 65 (+57)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

Drops from baseline

  • Create and improve agent skills.
+

Baseline

existing

Create and improve agent skills.

tokens 8 (0)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create and improve agent skills.

Drops from baseline

No dropped cues.

Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

tokens 53 (+45)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

Drops from baseline

  • Create and improve agent skills.

Winner — Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

tokens 53 (+45)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

Drops from baseline

  • Create and improve agent skills.

Evidence readiness

Readiness score: 85/100

-
  • Intent clarity · needs review
    30/100 intent confidence.
  • Benchmark coverage · ready
    3 GitHub benchmark repositories attached.
  • Pattern gate · ready
    3 accepted, 3 deferred using threshold 4/4.
  • Conflict handling · ready
    No material conflicts detected.
  • Output risk profile · ready
    5 output risk families attached.
  • Artifact design profile · ready
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Prompt quality profile · ready
    82.0/100 prompt-facing quality score.
+
  • Intent clarity · ready
    100/100 intent confidence.
  • Benchmark coverage · ready
    3 GitHub benchmark repositories attached.
  • Pattern gate · ready
    5 accepted, 1 deferred using threshold 4/4.
  • Conflict handling · decision needed
    The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.
  • Output risk profile · ready
    5 output risk families attached.
  • Artifact design profile · ready
    Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.
  • Prompt quality profile · ready
    89.0/100 prompt-facing quality score.

Honest boundary check

@@ -271,24 +271,24 @@

Artifact design profile

-

Design system: content-led editorial

-
  • Report or brief
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Code, CLI, or implementation guide
    Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.
+

Design system: metric editorial

+
  • Dashboard or metrics page
    Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.
  • Review viewer
    Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.
  • Code, CLI, or implementation guide
    Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.

Visual quality gates

-
  • Keep the first screen useful without requiring the reader to parse every detail.
  • Use tables only for comparisons; move explanations below the table.
  • Keep source notes readable without flooding the body with markers.
  • Name the working directory and required inputs for commands.
  • Mark destructive, networked, or external side-effect operations.
+
  • Avoid paragraph-heavy table cells.
  • Keep charts tied to one analytical question each.
  • Preserve stable color meaning across metrics and entities.
  • Make differences visible instead of hiding them in prose.
  • Separate author-facing recommendations from reviewer-only evidence.

Prompt quality profile

-

Relevance: prompt-heavy · score 82.0 / 100 · complexity complex

-
  • Completeness · 80 / 100
    Name missing inputs, outputs, constraints, or success standards before deepening the package.
  • Clarity · 80 / 100
    Replace broad verbs with observable actions and define what done means.
  • Consistency · 90 / 100
    Check that role, task, format, exclusions, and examples do not contradict each other.
  • Practicality · 90 / 100
    Add runnable steps, examples, or verification cues instead of abstract advice.
  • Specificity · 70 / 100
    Anchor wording in the user's audience, domain nouns, and target outcome.
+

Relevance: prompt-aware · score 89.0 / 100 · complexity expert

+
  • Completeness · 100 / 100
    Name missing inputs, outputs, constraints, or success standards before deepening the package.
  • Clarity · 85 / 100
    Replace broad verbs with observable actions and define what done means.
  • Consistency · 90 / 100
    Check that role, task, format, exclusions, and examples do not contradict each other.
  • Practicality · 90 / 100
    Add runnable steps, examples, or verification cues instead of abstract advice.
  • Specificity · 80 / 100
    Anchor wording in the user's audience, domain nouns, and target outcome.

RTF to skill mapping

-
  • Role
    Use a conversational role that asks only high-leverage questions and remembers the user's goal.
  • Task
    Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.
  • Format
    Return concise prompts, decision points, and reviewer-visible assumptions.
+
  • Role
    Use an operator role with explicit boundaries, inputs, outputs, and failure handling.
  • Task
    Convert the job into ordered steps with validation checks and stop conditions.
  • Format
    Return a runbook-like handoff with commands, checks, owners, and next actions when relevant.
@@ -310,11 +310,11 @@

Reference synthesis

-

Official workflow product ergonomics

Borrow now

  • Borrow a first-time operator flow that explains itself before it asks for more structure.

Avoid

  • Do not mimic product polish that adds UI bulk without improving clarity.

Hypothesis-test-learn loop

Borrow now

  • Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.

Avoid

  • Do not create experimental overhead that exceeds the skill's real risk tier.

Boundary-first design

Borrow now

  • Borrow the discipline of defining what the skill should not own before growing the package.

Avoid

  • Do not expand execution assets until route boundaries stay clean.
+

Official skill anatomy and context discipline

Borrow now

  • Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.

Avoid

  • Do not let packaging or platform concerns swallow the core job boundary.

Human-in-the-loop verification

Borrow now

  • Borrow a review checkpoint wherever trust matters more than raw speed.

Avoid

  • Do not force every skill through heavyweight review when the risk is low.

Boundary-first design

Borrow now

  • Borrow the discipline of defining what the skill should not own before growing the package.

Avoid

  • Do not expand execution assets until route boundaries stay clean.

Borrow now

-
  • Borrow a first-time operator flow that explains itself before it asks for more structure.
  • Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.
  • Borrow the discipline of defining what the skill should not own before growing the package.
  • Borrow the way it turns a messy workflow into a repeatable operating path.
+
  • Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.
  • Borrow a review checkpoint wherever trust matters more than raw speed.
  • Borrow the discipline of defining what the skill should not own before growing the package.
  • Borrow the way it turns a messy workflow into a repeatable operating path.

Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.

diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 93c362a8..4e482008 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -4,7 +4,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -12,6 +12,30 @@ "deliverables": [ "SKILL.md", "agents/interface.yaml", + "skill-ir/examples/yao-meta-skill.json", + "reports/compiled_targets.md", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_adjudication.md", + "reports/benchmark_reproducibility.md", + "reports/output_blind_answer_key.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_ledger.md", + "reports/review_annotations.md", + "reports/review-studio.html", + "reports/skill-interpretation.html", "reports/skill-overview.html" ], "flow": [ @@ -45,20 +69,19 @@ }, "evidence_score": { "label": "证据充分", - "score": 75, + "score": 100, "reasons": [ - "已生成 6 / 8 类报告证据。", - "intent-confidence.json 已存在。", - "reference-synthesis.json 已存在。", - "artifact-design-profile.json 已存在。", - "证据不足：缺少 intent-dialogue.json, output-risk-profile.json。" + "已生成 20 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" ] }, "maintainability_score": { "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 362 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -79,7 +102,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 362 个词/字，references 约 10018 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -87,7 +110,7 @@ }, "capability_profile": { "archetype": "governed", - "task_family": "Dialogue interaction", + "task_family": "Execution operation", "maturity": "governed", "trigger_strength": "手动触发 + description 路由", "reuse_scope": "跨平台", @@ -100,7 +123,7 @@ "nodes": [ { "title": "意图澄清", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." }, { "title": "边界路由", @@ -120,7 +143,7 @@ }, { "title": "杠杆升级", - "body": "Ask one focused question and update intent context before adding assets." + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." } ], "layers": [ @@ -133,19 +156,19 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" }, "inputs": [ - "用户提供的工作流、提示词、文档、记录或散乱笔记", - "期望沉淀的复用场景、排除项、约束和质量标准" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], "outputs": [ - "可路由的 SKILL.md", - "agents/interface.yaml 元数据", - "必要的 references、scripts、evals、reports 证据", + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "结构化 Skill 目录，共 8 类关键资产。" ], "should_trigger": [ @@ -161,7 +184,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -176,13 +199,14 @@ "quality_review": { "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", - "已打包 agents/interface.yaml，便于后续做跨平台适配。", - "长指导被拆到 references 中，入口文件可以保持轻量。", - "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。", - "包内包含可随 Skill 迁移的质量门禁或触发检查。" + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 362 个词/字，references 约 10018 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -190,33 +214,33 @@ "每次升级后重新生成报告并检查分数原因。" ], "artifact_design": { - "design_system": "content-led editorial", + "design_system": "metric editorial", "highlights": [ - "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "highlights": [ - "Primary prompt task family: Dialogue interaction.", - "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", - "Completeness: 80/100.", - "Clarity: 80/100." + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." ] }, "system_model": { "stability": { - "score": 78, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "highlights": [ - "Stability: stable-first-pass (78/100).", - "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", - "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." ] } }, @@ -232,22 +256,22 @@ { "name": "输出漂移风险", "impact": 2, - "probability": 2, - "signal": "已生成 6 / 8 类报告证据。", + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "证据不足风险", "impact": 3, - "probability": 2, - "signal": "已生成 6 / 8 类报告证据。", + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 362 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -258,13 +282,199 @@ "response": "先补证据和边界，再增加包体复杂度。" } ], - "risk_families": [], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], "human_judgment_boundary": [ "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." ] }, + "world_class_readiness": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 19, + "source_pass_count": 11, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete", + "Reviewer metadata" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, "package_assets": { "entries": [ { @@ -295,28 +505,28 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 23 + "file_count": 35 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 65 + "file_count": 153 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 25 + "file_count": 29 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 118 + "file_count": 235 } ], - "file_count": 235, + "file_count": 456, "folder_count": 4, "distribution": [ { @@ -337,24 +547,45 @@ }, { "label": "references", - "value": 23 + "value": 35 }, { "label": "scripts", - "value": 65 + "value": 153 }, { "label": "evals", - "value": 25 + "value": 29 }, { "label": "reports", - "value": 118 + "value": 235 } ] }, "iteration_roadmap": { "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, { "title": "Harden portability semantics", "why": "The skill already signals reuse across environments, so contract clarity matters early.", @@ -364,16 +595,6 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift." - }, - { - "title": "Create an iteration evidence loop", - "why": "The package should show what changed and why after the first draft.", - "actions": [ - "Generate the HTML skill report and keep it aligned with the package.", - "Record reference scan choices and non-goals.", - "Capture the next iteration choice explicitly before adding more files." - ], - "unlocks": "A clearer path for the next author or reviewer." } ] }, @@ -413,17 +634,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -454,33 +675,34 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 23 + "file_count": 35 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 65 + "file_count": 153 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 25 + "file_count": 29 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 118 + "file_count": 235 } ], "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", - "已打包 agents/interface.yaml，便于后续做跨平台适配。", - "长指导被拆到 references 中，入口文件可以保持轻量。", - "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。", - "包内包含可随 Skill 迁移的质量门禁或触发检查。" + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "scenarios": [ "把重复流程沉淀为可复用的 agent skill。", @@ -489,27 +711,27 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" }, "io_contract": { "inputs": [ - "用户提供的工作流、提示词、文档、记录或散乱笔记", - "期望沉淀的复用场景、排除项、约束和质量标准" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], "outputs": [ - "可路由的 SKILL.md", - "agents/interface.yaml 元数据", - "必要的 references、scripts、evals、reports 证据", + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "结构化 Skill 目录，共 8 类关键资产。" ] }, "principles": [ { "title": "意图澄清", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." }, { "title": "边界路由", @@ -529,10 +751,31 @@ }, { "title": "杠杆升级", - "body": "Ask one focused question and update intent context before adding assets." + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." } ], "roadmap": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, { "title": "Harden portability semantics", "why": "The skill already signals reuse across environments, so contract clarity matters early.", @@ -542,16 +785,6 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift." - }, - { - "title": "Create an iteration evidence loop", - "why": "The package should show what changed and why after the first draft.", - "actions": [ - "Generate the HTML skill report and keep it aligned with the package.", - "Record reference scan choices and non-goals.", - "Capture the next iteration choice explicitly before adding more files." - ], - "unlocks": "A clearer path for the next author or reviewer." } ], "cards": [], @@ -561,39 +794,1461 @@ "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" ], "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "source_path": "skill-ir/examples/yao-meta-skill.json", + "target_count": 5, + "trigger_samples": 8, + "output_eval_cases": 3 + }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, + "output_quality": { + "case_count": 5, + "file_backed_case_count": 1, + "near_neighbor_case_count": 1, + "boundary_case_count": 1, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "gate_pass": true, + "failure_taxonomy": [], + "blind_pair_count": 5 + }, + "output_execution": { + "ok": true, + "summary": { + "case_count": 5, + "variant_run_count": 10, + "command_executed_count": 10, + "model_executed_count": 10, + "recorded_fixture_count": 0, + "timing_observed_count": 10, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, + "regression_count": 0, + "failure_count": 0, + "gate_pass": true + }, + "runner": { + "mode": "command", + "command": [ + "python3", + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" + ], + "timeout_seconds": 180.0 + }, + "failures": [] + }, + "output_blind_review": { + "summary": { + "pair_count": 5, + "answer_key_separate": true, + "with_skill_hidden_count": 5 + }, + "seed": "yao-output-eval-blind-v1", + "pair_count": 5, + "answer_key_separate": true + }, + "output_review_kit": { + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "failures": [] + }, + "output_review_adjudication": { + "ok": true, + "summary": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "agreement_count": 0, + "disagreement_count": 0, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, + "agreement_rate": null, + "needs_review": true, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "failure_count": 0, + "reviewer_checklist_count": 5, + "reviewer_checklist_pending_count": 5, + "reviewer_checklist_invalid_count": 0, + "reviewer_checklist_ready_count": 0 + }, + "reviewer": "", + "reviewed_at": "", + "failures": [] + }, + "benchmark_reproducibility": { + "ok": true, + "summary": { + "reproducibility_ready": true, + "release_lock_ready": true, + "methodology_complete": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "evidence_bundle_sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0", + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 23, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 10, + "token_observed_count": 10, + "human_review_complete": false, + "provider_evidence_complete": true, + "world_class_ready": false, + "world_class_open_gap_count": 3, + "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3, + "working_tree_dirty": true, + "changed_file_count": 29, + "source_tree_dirty": false, + "source_changed_file_count": 0, + "generated_tree_dirty": true, + "generated_changed_file_count": 29 + }, + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "missing_artifacts": [], + "limitations": [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims." + ] + }, + "runtime_conformance": { + "target_count": 5, + "pass_count": 5, + "fail_count": 0 + }, + "runtime_permissions": { + "ok": true, + "summary": { + "target_count": 4, + "pass_count": 4, + "fail_count": 0, + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "required_capability_count": 3, + "failure_count": 0, + "installer_enforcement_source_status": "present", + "installer_enforcement_target_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "installer_permission_capability_count": 3, + "world_class_native_evidence_ready": false, + "installer_enforcement_ready": true + }, + "expected_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "targets": [ + { + "target": "openai", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "claude", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "generic", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "vscode", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + } + ], + "failures": [] + }, + "trust_security": { + "scanned_files": 245, + "script_count": 153, + "internal_module_count": 67, + "secret_findings": 0, + "dependency_files": [ + "requirements-ci.txt" + ], + "network_script_count": 3, + "network_policy_covered_count": 3, + "network_policy_missing_count": 0, + "file_write_script_count": 73, + "permission_required_count": 3, + "permission_approved_count": 3, + "permission_missing_count": 0, + "permission_invalid_count": 0, + "permission_expired_count": 0, + "help_smoke_checked_count": 86, + "help_smoke_failed_count": 0, + "interactive_script_count": 0, + "package_hash_scope": "source-contract-without-generated-reports", + "package_hash_file_count": 245, + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" + }, + "skill_atlas": { + "skill_count": 12, + "actionable_skill_count": 1, + "route_collision_count": 5, + "actionable_route_collision_count": 0, + "owner_gap_count": 9, + "actionable_owner_gap_count": 0, + "stale_count": 10, + "actionable_stale_count": 0, + "shared_resource_count": 0, + "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, + "non_actionable_issue_count": 24 + }, + "registry_distribution": { + "ok": true, + "package": { + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" + }, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": true, + "summary": { + "target_count": 4, + "adapter_count": 4, + "archive_present": true, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "archive_entry_count": 689, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": true, + "summary": { + "archive_present": true, + "archive_entry_count": 696, + "archive_extracted": true, + "entrypoint_loaded": true, + "manifest_loaded": true, + "interface_loaded": true, + "adapter_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, + "install_root_is_temp": false, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": true, + "summary": { + "declared_bump": "minor", + "recommended_bump": "minor", + "breaking_change_count": 0, + "added_target_count": 2, + "metadata_change_count": 1, + "checksum_change_count": 2, + "failure_count": 0, + "warning_count": 0 + }, + "upgrade_diff": { + "name_changed": false, + "added_targets": [ + "agent-skills-compatible", + "vscode" + ], + "removed_targets": [], + "compatibility_changes": [ + { + "target": "agent-skills", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "agent-skills-compatible", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "vscode", + "from": "missing", + "to": "pass", + "direction": "improved" + } + ], + "metadata_changes": [ + { + "field": "description", + "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." + } + ], + "checksum_changes": [ + { + "field": "archive_sha256", + "from": "", + "to": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" + }, + { + "field": "package_sha256", + "from": "0000000000000000000000000000000000000000000000000000000000000000", + "to": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" + } + ] + }, + "release_notes": [ + "Recommended version bump: minor.", + "Added targets: agent-skills-compatible, vscode.", + "Package or archive checksum changed; reviewers should verify package artifacts before release.", + "Package verification evidence: reports/package_verification.md." + ], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "failure_types": {}, + "source_types": { + "manual": 1 + }, + "command_counts": {} + }, + "next_iteration_candidates": [], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 2, + "waiverable_open_count": 1, + "non_waivable_count": 1 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "review_studio_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "known_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "waiverable_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "non_waivable_gate_keys": [ + "review-waivers", + "world-class-evidence" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] + }, + "world_class_evidence_plan": { + "ok": true, + "summary": { + "audit_decision": "continue-iteration", + "world_class_ready": false, + "audit_world_class_ready": false, + "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, + "task_count": 4, + "human_task_count": 1, + "external_task_count": 3, + "review_task_count": 0, + "decision": "collect-external-evidence" + }, + "tasks": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_audit": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "open_gap_count": 3 + } + }, + "world_class_evidence_ledger": { + "ok": true, + "summary": { + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "source_accepted": false, + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4, + "evidence_requirement_count": 4 + } + }, "synthesis_highlights": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", "Borrow the discipline of defining what the skill should not own before growing the package." ], "artifact_design": { - "design_system": "content-led editorial", + "design_system": "metric editorial", "highlights": [ - "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "highlights": [ - "Primary prompt task family: Dialogue interaction.", - "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", - "Completeness: 80/100.", - "Clarity: 80/100." + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." ] }, "system_model": { "stability": { - "score": 78, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "highlights": [ - "Stability: stable-first-pass (78/100).", - "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", - "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." ] }, "metadata": { @@ -601,7 +2256,8 @@ "targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "maturity_tier": "governed", "skill_archetype": "governed", @@ -609,82 +2265,56 @@ } }, "intent_confidence": { - "score": 30, - "band": "low", - "gate_passed": false, + "score": 100, + "band": "high", + "gate_passed": true, "strengths": [ - "The recurring job is concrete enough to anchor the package." + "The recurring job is concrete enough to anchor the package.", + "Real input shape is explicit.", + "The hand-back output is concrete.", + "Boundary exclusions are already explicit.", + "Operational constraints are visible." ], - "gaps": [ - { - "key": "real_inputs", - "label": "Real inputs are missing", - "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", - "severity": "high" - }, - { - "key": "primary_output", - "label": "Primary output is missing", - "reason": "The package does not yet know what it must hand back.", - "severity": "high" - }, - { - "key": "exclusions", - "label": "Near-neighbor exclusions are missing", - "reason": "The route may blur into nearby requests without an exclusion list.", - "severity": "high" - }, - { - "key": "constraints", - "label": "Constraints are missing", - "reason": "The package does not yet know which tradeoffs matter most.", - "severity": "high" - }, - { - "key": "standards", - "label": "Quality bar is implied, not explicit", - "reason": "The first evaluation target is still underspecified.", - "severity": "medium" - } - ], - "follow_up_questions": [ - { - "slot": "real_inputs", - "question": "What material will people actually hand to this skill in practice?", - "why": "Real input shape decides whether references, scripts, or examples are needed.", - "list": true, - "label": "Real inputs are missing", - "severity": "high" - }, - { - "slot": "primary_output", - "question": "What finished hand-back should this skill return so the next person can keep moving?", - "why": "The output is the anchor for package design and review.", - "list": false, - "label": "Primary output is missing", - "severity": "high" - }, - { - "slot": "exclusions", - "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", - "why": "Exclusions are the fastest route to better trigger quality.", - "list": true, - "label": "Near-neighbor exclusions are missing", - "severity": "high" - } - ], - "anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "gaps": [], + "follow_up_questions": [], + "anchor_sentence": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.", + "recommended_action": "Intent is clear enough to package the first routeable version.", "context": { - "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "real_inputs": [], - "primary_output": "", + "job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "real_inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "exclusions": [], - "constraints": [], - "standards": [], - "correction": "", - "user_references": [] + "exclusions": [ + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill", + "raw private material that was not intentionally supplied as skill evidence", + "platform-specific plugin creation when the request is not about an agent skill package" + ], + "constraints": [ + "keep SKILL.md lean and route primarily through frontmatter description", + "put durable guidance in references, executable logic in scripts, and evidence in reports", + "default to the lightest reliable mode before adding governance weight", + "preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets", + "avoid raw prompt, output, transcript, or private content in telemetry" + ], + "standards": [ + "trigger boundaries must be tested with should-trigger and should-not-trigger cases", + "production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence", + "governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation", + "generated reports should be bilingual or reviewer-friendly when they are user-facing", + "each new asset must earn its place by reducing ambiguity, risk, or repeated work" + ], + "correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.", + "user_references": [ + "Yao Meta Skill 2.0 upgrade plan", + "Kami-style white report layout requirements for generated skill reports", + "local Waza engineering habits for review, design, diagnosis, and implementation discipline" + ] } }, "intent": { @@ -862,10 +2492,10 @@ "baseline": { "summary": { "target_count": 3, - "baseline_total_errors": 8, - "current_total_errors": 5, - "winner_total_errors": 5, - "winner_vs_baseline_gain": 3, + "baseline_total_errors": 5, + "current_total_errors": 4, + "winner_total_errors": 4, + "winner_vs_baseline_gain": 1, "winner_vs_current_gain": 0 }, "comparisons": [ @@ -881,7 +2511,7 @@ "adversarial_errors": 0 }, "current": { - "tokens": 65, + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "blind_errors": 0, @@ -889,7 +2519,7 @@ "adversarial_errors": 0 }, "winner": { - "tokens": 65, + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "blind_errors": 0, @@ -940,15 +2570,15 @@ "winner_label": "Current", "baseline": { "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2, - "blind_errors": 1, + "dev_errors": 0, + "holdout_errors": 1, + "blind_errors": 0, "judge_blind_errors": 0, "adversarial_errors": 0 }, "current": { "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "blind_errors": 0, "judge_blind_errors": 0, @@ -956,28 +2586,28 @@ }, "winner": { "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "blind_errors": 0, "judge_blind_errors": 0, "adversarial_errors": 0 }, "delta": { - "current_vs_baseline": 2, - "winner_vs_baseline": 2, + "current_vs_baseline": 0, + "winner_vs_baseline": 0, "winner_vs_current": 0 } } ] }, "compare": { - "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "current_candidate": { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -993,7 +2623,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -3120,9 +4750,9 @@ "winner": { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -3138,7 +4768,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -3293,18 +4923,16 @@ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", - "score": 0.415, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3314,9 +4942,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -3336,33 +4964,32 @@ { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", - "score": 0.394, + "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.327, - "support_score": 0, - "lexical_support": 0.417, + "semantic_coverage": 0.333, + "support_score": 0.16, + "lexical_support": 0.25, "negative_penalty": 0, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ @@ -3387,32 +5014,31 @@ { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", - "score": 0.843, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3443,18 +5069,16 @@ { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", - "score": 0.422, + "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3466,7 +5090,7 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, "lexical_support": 0.4, "negative_penalty": 0, @@ -3492,31 +5116,30 @@ { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", - "score": 0.557, + "score": 0.742, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.16, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, @@ -3543,18 +5166,16 @@ { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", - "score": 0.474, + "score": 0.763, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3565,9 +5186,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0, - "lexical_support": 0.571, + "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3591,32 +5212,31 @@ { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", - "score": 0.868, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0, + "semantic_coverage": 1.0, + "support_score": 0.34, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, @@ -3649,35 +5269,33 @@ { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", - "score": 0.889, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0.06, - "lexical_support": 0.346, + "semantic_coverage": 1.0, + "support_score": 0.4, + "lexical_support": 0.231, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -3724,11 +5342,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3741,9 +5357,9 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { @@ -3770,11 +5386,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -3811,11 +5425,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3832,9 +5444,9 @@ "brainstorm_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.2, + "lexical_support": 0.133, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { @@ -3868,11 +5480,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -3885,7 +5495,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.167, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { @@ -3909,11 +5519,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3926,9 +5534,9 @@ "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.111, + "lexical_support": 0.222, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { @@ -3955,11 +5563,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -3974,9 +5580,9 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.174, + "lexical_support": 0.087, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { @@ -4001,37 +5607,36 @@ { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", - "score": 0.256, + "score": 0.306, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.333, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.222, "negative_penalty": 0.3, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -4066,11 +5671,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4086,7 +5689,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, @@ -4121,11 +5724,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4138,9 +5739,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.143, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { @@ -4167,18 +5768,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -4186,11 +5786,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.3, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -4221,18 +5821,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "iterate_existing_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -4241,11 +5840,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.327, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.16, "lexical_support": 0.267, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "iterate_existing_skill": [ @@ -4276,18 +5875,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -4295,11 +5893,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.286, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -4331,18 +5929,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "future_outline", @@ -4352,11 +5949,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.238, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.19, "negative_penalty": 0.82, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -4395,18 +5992,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -4415,11 +6011,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.34, "lexical_support": 0.263, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -4560,18 +6156,16 @@ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", - "score": 0.468, + "score": 0.767, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4584,9 +6178,9 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0.06, - "lexical_support": 0.316, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4613,33 +6207,32 @@ { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", - "score": 0.727, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.691, - "support_score": 0, - "lexical_support": 0.389, + "semantic_coverage": 1.0, + "support_score": 0.16, + "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4670,18 +6263,16 @@ { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", - "score": 0.568, + "score": 0.936, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4692,9 +6283,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, + "semantic_coverage": 0.933, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4717,34 +6308,32 @@ { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", - "score": 0.858, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0.06, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.4, + "lexical_support": 0.312, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4775,32 +6364,31 @@ { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", - "score": 0.744, + "score": 0.839, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.709, - "support_score": 0, - "lexical_support": 0.4, + "semantic_coverage": 0.733, + "support_score": 0.34, + "lexical_support": 0.24, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -4837,11 +6425,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4857,9 +6443,9 @@ "no_build_directive", "summarize_only" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { @@ -4893,11 +6479,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4913,7 +6497,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, @@ -4949,11 +6533,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -4967,7 +6549,7 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, @@ -5001,19 +6583,18 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "no_build_directive", "translate_only" @@ -5022,11 +6603,11 @@ "no_build_directive", "translate_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.25, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.188, "negative_penalty": 0.77, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -5064,17 +6645,15 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "matched_desired_concepts": [ + "matched_desired_concepts": [], + "extra_positive_concepts": [ "build_skill" ], - "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" @@ -5083,9 +6662,9 @@ "brainstorm_only", "future_outline" ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, + "semantic_coverage": 0.0, + "support_score": 0.34, + "lexical_support": 0.077, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { @@ -5115,11 +6694,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -5134,7 +6711,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.214, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -5162,18 +6739,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -5181,11 +6757,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.615, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.308, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -5211,26 +6787,25 @@ { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", - "score": 0.0, + "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -5238,11 +6813,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.673, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.34, "lexical_support": 0.438, "negative_penalty": 0.6, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -5415,18 +6990,16 @@ { "prompt": "Convert this operations checklist into a reusable skill.", "family": "workflow_to_skill", - "score": 0.415, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5436,9 +7009,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -5458,33 +7031,32 @@ { "prompt": "Add trigger evals to this skill before sharing it with the team.", "family": "iterate_existing_skill", - "score": 0.394, + "score": 0.407, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.327, - "support_score": 0, - "lexical_support": 0.417, + "semantic_coverage": 0.333, + "support_score": 0.16, + "lexical_support": 0.25, "negative_penalty": 0, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "eval_optimize": [ @@ -5509,32 +7081,31 @@ { "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", "family": "complex_multi_asset", - "score": 0.843, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5565,18 +7136,16 @@ { "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", "family": "complex_multi_asset", - "score": 0.422, + "score": 0.7, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5588,7 +7157,7 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, "lexical_support": 0.4, "negative_penalty": 0, @@ -5614,31 +7183,30 @@ { "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", "family": "iterate_existing_skill", - "score": 0.557, + "score": 0.742, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.16, "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, @@ -5665,18 +7233,16 @@ { "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", "family": "paraphrase_trigger", - "score": 0.474, + "score": 0.763, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5687,9 +7253,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0, - "lexical_support": 0.571, + "lexical_support": 0.357, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5713,32 +7279,31 @@ { "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", "family": "paraphrase_trigger", - "score": 0.868, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0, + "semantic_coverage": 1.0, + "support_score": 0.34, "lexical_support": 0.267, "negative_penalty": 0, "coverage_boost": 0.06, @@ -5771,35 +7336,33 @@ { "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", "family": "long_context_trigger", - "score": 0.889, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "team_operationalize", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.855, - "support_score": 0.06, - "lexical_support": 0.346, + "semantic_coverage": 1.0, + "support_score": 0.4, + "lexical_support": 0.231, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -5846,11 +7409,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5863,9 +7424,9 @@ "exclusive_negative_concepts": [ "brainstorm_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { @@ -5892,11 +7453,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -5933,11 +7492,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -5954,9 +7511,9 @@ "brainstorm_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.2, + "lexical_support": 0.133, "negative_penalty": 0.66, "coverage_boost": 0.0, "concept_evidence": { @@ -5990,11 +7547,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -6007,7 +7562,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.167, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { @@ -6031,11 +7586,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6048,9 +7601,9 @@ "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.111, + "lexical_support": 0.222, "negative_penalty": 0.3, "coverage_boost": 0.0, "concept_evidence": { @@ -6077,11 +7630,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6096,9 +7647,9 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.174, + "lexical_support": 0.087, "negative_penalty": 0.68, "coverage_boost": 0.0, "concept_evidence": { @@ -6123,37 +7674,36 @@ { "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", "family": "long_context_summary_only", - "score": 0.256, + "score": 0.306, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "summarize_only" ], "exclusive_negative_concepts": [ "summarize_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.333, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.222, "negative_penalty": 0.3, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -6188,11 +7738,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6208,7 +7756,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0.68, @@ -6243,11 +7791,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6260,9 +7806,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.143, "negative_penalty": 0.42, "coverage_boost": 0.0, "concept_evidence": { @@ -6289,18 +7835,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -6308,11 +7853,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.3, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6343,18 +7888,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "iterate_existing_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -6363,11 +7907,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.327, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.16, "lexical_support": 0.267, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "iterate_existing_skill": [ @@ -6398,18 +7942,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -6417,11 +7960,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.5, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.286, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6453,18 +7996,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "future_outline", @@ -6474,11 +8016,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.238, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.19, "negative_penalty": 0.82, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6517,18 +8059,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "future_outline", "no_build_directive" @@ -6537,11 +8078,11 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, + "semantic_coverage": 0.333, + "support_score": 0.34, "lexical_support": 0.263, "negative_penalty": 0.64, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -6682,18 +8223,16 @@ { "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", "family": "complex_multi_asset", - "score": 0.468, + "score": 0.767, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6706,9 +8245,9 @@ ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.4, + "semantic_coverage": 0.733, "support_score": 0.06, - "lexical_support": 0.316, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6735,33 +8274,32 @@ { "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", "family": "iterate_existing_skill", - "score": 0.727, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "iterate_existing_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.691, - "support_score": 0, - "lexical_support": 0.389, + "semantic_coverage": 1.0, + "support_score": 0.16, + "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6792,18 +8330,16 @@ { "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", "family": "workflow_to_skill", - "score": 0.568, + "score": 0.936, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6814,9 +8350,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.509, + "semantic_coverage": 0.933, "support_score": 0, - "lexical_support": 0.5, + "lexical_support": 0.214, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6839,34 +8375,32 @@ { "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", "family": "paraphrase_trigger", - "score": 0.858, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], "extra_positive_concepts": [ + "build_skill", "multi_asset" ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0.06, - "lexical_support": 0.375, + "semantic_coverage": 0.933, + "support_score": 0.4, + "lexical_support": 0.312, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6897,32 +8431,31 @@ { "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", "family": "long_context_trigger", - "score": 0.744, + "score": 0.839, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "team_operationalize", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.709, - "support_score": 0, - "lexical_support": 0.4, + "semantic_coverage": 0.733, + "support_score": 0.34, + "lexical_support": 0.24, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -6959,11 +8492,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -6979,9 +8510,9 @@ "no_build_directive", "summarize_only" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.286, + "lexical_support": 0.143, "negative_penalty": 0.72, "coverage_boost": 0.04, "concept_evidence": { @@ -7015,11 +8546,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7035,7 +8564,7 @@ "explain_only", "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.091, "negative_penalty": 0.68, @@ -7071,11 +8600,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7089,7 +8616,7 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, "lexical_support": 0.222, "negative_penalty": 0.6, @@ -7123,19 +8650,18 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "no_build_directive", "translate_only" @@ -7144,11 +8670,11 @@ "no_build_directive", "translate_only" ], - "semantic_coverage": 0.673, - "support_score": 0, - "lexical_support": 0.25, + "semantic_coverage": 0.667, + "support_score": 0.34, + "lexical_support": 0.188, "negative_penalty": 0.77, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -7186,17 +8712,15 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], - "matched_desired_concepts": [ + "matched_desired_concepts": [], + "extra_positive_concepts": [ "build_skill" ], - "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only", "future_outline" @@ -7205,9 +8729,9 @@ "brainstorm_only", "future_outline" ], - "semantic_coverage": 0.309, - "support_score": 0, - "lexical_support": 0.308, + "semantic_coverage": 0.0, + "support_score": 0.34, + "lexical_support": 0.077, "negative_penalty": 0.46, "coverage_boost": 0.0, "concept_evidence": { @@ -7237,11 +8761,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7256,7 +8778,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.357, + "lexical_support": 0.214, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -7284,18 +8806,17 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -7303,11 +8824,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.491, - "support_score": 0, - "lexical_support": 0.615, + "semantic_coverage": 0.333, + "support_score": 0.34, + "lexical_support": 0.308, "negative_penalty": 0.6, - "coverage_boost": 0.04, + "coverage_boost": 0.0, "concept_evidence": { "positive": { "build_skill": [ @@ -7333,26 +8854,25 @@ { "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", "family": "long_context_near_neighbor", - "score": 0.0, + "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [ "document_only", "no_build_directive" @@ -7360,11 +8880,11 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.673, - "support_score": 0, + "semantic_coverage": 0.667, + "support_score": 0.34, "lexical_support": 0.438, "negative_penalty": 0.6, - "coverage_boost": 0.06, + "coverage_boost": 0.04, "concept_evidence": { "positive": { "build_skill": [ @@ -7467,18 +8987,16 @@ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", - "score": 0.392, + "score": 0.665, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7488,9 +9006,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.214, + "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -7505,24 +9023,21 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", - "score": 0.405, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7532,9 +9047,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.375, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -7550,26 +9065,23 @@ }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", - "score": 0.028, + "score": 0.16, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7582,9 +9094,9 @@ "document_only" ], "exclusive_negative_concepts": [], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.318, + "lexical_support": 0.227, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { @@ -7614,11 +9126,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7656,11 +9166,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -7675,7 +9183,7 @@ ], "semantic_coverage": 0.0, "support_score": 0.06, - "lexical_support": 0.308, + "lexical_support": 0.231, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { @@ -7695,18 +9203,16 @@ { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", - "score": 0.026, + "score": 0.305, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7720,7 +9226,7 @@ "exclusive_negative_concepts": [ "future_outline" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, @@ -7740,7 +9246,8 @@ ] } } - } + }, + "boundary_case": true } ] } @@ -7819,18 +9326,16 @@ { "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", "family": "blind_operationalize", - "score": 0.392, + "score": 0.665, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7840,9 +9345,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.214, + "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -7857,24 +9362,21 @@ }, "negative": {} } - }, - "boundary_case": true + } }, { "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", "family": "blind_multi_asset", - "score": 0.405, + "score": 0.673, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7884,9 +9386,9 @@ "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, - "lexical_support": 0.375, + "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { @@ -7902,26 +9404,23 @@ }, "negative": {} } - }, - "boundary_case": true + } } ], "should_not_trigger": [ { "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", "family": "blind_document_only", - "score": 0.028, + "score": 0.16, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -7934,9 +9433,9 @@ "document_only" ], "exclusive_negative_concepts": [], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0.06, - "lexical_support": 0.318, + "lexical_support": 0.227, "negative_penalty": 0.18, "coverage_boost": 0.0, "concept_evidence": { @@ -7966,11 +9465,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -8008,11 +9505,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -8027,7 +9522,7 @@ ], "semantic_coverage": 0.0, "support_score": 0.06, - "lexical_support": 0.308, + "lexical_support": 0.231, "negative_penalty": 0.22, "coverage_boost": 0.0, "concept_evidence": { @@ -8047,18 +9542,16 @@ { "prompt": "Review this process and suggest improvements for later; no packaging yet.", "family": "blind_brainstorm_only", - "score": 0.026, + "score": 0.305, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -8072,7 +9565,7 @@ "exclusive_negative_concepts": [ "future_outline" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0, "lexical_support": 0.273, "negative_penalty": 0.22, @@ -8092,7 +9585,8 @@ ] } } - } + }, + "boundary_case": true } ] } @@ -9558,32 +11052,31 @@ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", - "score": 0.834, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.263, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -9610,32 +11103,31 @@ { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", - "score": 0.842, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.368, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -9670,11 +11162,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9691,9 +11181,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, - "lexical_support": 0.429, + "lexical_support": 0.357, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { @@ -9729,11 +11219,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9751,9 +11239,9 @@ "summarize_only", "translate_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.188, + "lexical_support": 0.125, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { @@ -9792,11 +11280,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -9811,7 +11297,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.2, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -9840,11 +11326,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -9861,9 +11345,9 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.182, + "lexical_support": 0.273, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { @@ -9963,32 +11447,31 @@ { "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", "family": "adversarial_noisy_operationalize", - "score": 0.834, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.263, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.211, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -10015,32 +11498,31 @@ { "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", "family": "adversarial_multi_asset_boundary", - "score": 0.842, + "score": 1.0, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ - "build_skill", "eval_optimize", "reuse_package", "transform_workflow" ], - "extra_positive_concepts": [], + "extra_positive_concepts": [ + "build_skill" + ], "matched_negative_concepts": [], "exclusive_negative_concepts": [], - "semantic_coverage": 0.818, - "support_score": 0, - "lexical_support": 0.368, + "semantic_coverage": 0.933, + "support_score": 0.34, + "lexical_support": 0.263, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { @@ -10075,11 +11557,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -10096,9 +11576,9 @@ "exclusive_negative_concepts": [ "no_build_directive" ], - "semantic_coverage": 0.364, + "semantic_coverage": 0.667, "support_score": 0.06, - "lexical_support": 0.429, + "lexical_support": 0.357, "negative_penalty": 0.6, "coverage_boost": 0.04, "concept_evidence": { @@ -10134,11 +11614,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -10156,9 +11634,9 @@ "summarize_only", "translate_only" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.188, + "lexical_support": 0.125, "negative_penalty": 1.25, "coverage_boost": 0.0, "concept_evidence": { @@ -10197,11 +11675,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [], @@ -10216,7 +11692,7 @@ ], "semantic_coverage": 0.0, "support_score": 0, - "lexical_support": 0.25, + "lexical_support": 0.2, "negative_penalty": 0.64, "coverage_boost": 0.0, "concept_evidence": { @@ -10245,11 +11721,9 @@ "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ - "build_skill", "transform_workflow", "reuse_package", "eval_optimize", - "iterate_existing_skill", "team_operationalize" ], "matched_desired_concepts": [ @@ -10266,9 +11740,9 @@ "future_outline", "no_build_directive" ], - "semantic_coverage": 0.182, + "semantic_coverage": 0.333, "support_score": 0, - "lexical_support": 0.182, + "lexical_support": 0.273, "negative_penalty": 0.88, "coverage_boost": 0.0, "concept_evidence": { @@ -10697,9 +12171,9 @@ { "id": "current", "label": "Current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "strategy": "current", - "estimated_tokens": 65, + "estimated_tokens": 53, "dev": { "false_positives": 0, "false_negatives": 0, @@ -10715,7 +12189,7 @@ -1.0, -1.0, -1.0, - 65 + 53 ], "holdout": { "false_positives": 0, @@ -11049,30 +12523,30 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" @@ -11150,33 +12624,33 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "baseline_calibration": { "threshold": 0.33, @@ -11325,14 +12799,14 @@ }, "winner_calibration": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -11340,14 +12814,14 @@ }, "current_calibration": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -11402,8 +12876,8 @@ }, "summary": { "winner_label": "Current", - "winner_tokens": 65, - "current_tokens": 65, + "winner_tokens": 53, + "current_tokens": 53, "winner_dev_total_errors": 0, "current_dev_total_errors": 0, "winner_holdout_total_errors": 0, @@ -11416,7 +12890,7 @@ "current_adversarial_holdout_total_errors": 0, "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", - "winner_adversarial_score_gap": 0.834, + "winner_adversarial_score_gap": 1.0, "candidate_count": 6, "baseline_tokens": 8, "baseline_dev_total_errors": 1, @@ -11437,9 +12911,9 @@ { "name": "yao-meta-skill", "winner_label": "Current", - "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "winner_tokens": 65, - "current_tokens": 65, + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "winner_tokens": 53, + "current_tokens": 53, "winner_holdout_fp": 0, "winner_holdout_fn": 0, "current_holdout_fp": 0, @@ -11470,44 +12944,44 @@ "calibration": { "holdout": { "threshold": 0.33, - "mean_positive_score": 0.673, - "mean_non_trigger_score": 0.0, - "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.468, - "max_non_trigger_score": 0.0, - "score_gap": 0.468, - "threshold_margin": 0.138, - "positive_threshold_buffer": 0.138, - "negative_threshold_buffer": 0.33, + "mean_positive_score": 0.908, + "mean_non_trigger_score": 0.003, + "mean_near_neighbor_score": 0.006, + "min_positive_score": 0.767, + "max_non_trigger_score": 0.023, + "score_gap": 0.744, + "threshold_margin": 0.307, + "positive_threshold_buffer": 0.437, + "negative_threshold_buffer": 0.307, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "blind_holdout": { "threshold": 0.33, - "mean_positive_score": 0.399, - "mean_non_trigger_score": 0.013, - "mean_near_neighbor_score": 0.013, - "min_positive_score": 0.392, - "max_non_trigger_score": 0.028, - "score_gap": 0.364, - "threshold_margin": 0.062, - "positive_threshold_buffer": 0.062, - "negative_threshold_buffer": 0.302, - "boundary_case_count": 2, - "boundary_case_rate": 0.333, - "risk_band": "watch" + "mean_positive_score": 0.669, + "mean_non_trigger_score": 0.116, + "mean_near_neighbor_score": 0.152, + "min_positive_score": 0.665, + "max_non_trigger_score": 0.305, + "score_gap": 0.36, + "threshold_margin": 0.025, + "positive_threshold_buffer": 0.335, + "negative_threshold_buffer": 0.025, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" }, "adversarial_holdout": { "threshold": 0.33, - "mean_positive_score": 0.838, + "mean_positive_score": 1.0, "mean_non_trigger_score": 0.0, "mean_near_neighbor_score": 0.0, - "min_positive_score": 0.834, + "min_positive_score": 1.0, "max_non_trigger_score": 0.0, - "score_gap": 0.834, + "score_gap": 1.0, "threshold_margin": 0.33, - "positive_threshold_buffer": 0.504, + "positive_threshold_buffer": 0.67, "negative_threshold_buffer": 0.33, "boundary_case_count": 0, "boundary_case_rate": 0.0, @@ -11598,7 +13072,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.611, + "average_margin": 0.701, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -11621,8 +13095,8 @@ "label": "Current", "ranking_state": "selected_by_dev", "promotion_state": "kept_current", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "tokens": 65, + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "tokens": 53, "dev_errors": 0, "holdout_errors": 0, "reason_tags": [] @@ -11652,7 +13126,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -11666,7 +13141,8 @@ "dev_errors": 2, "holdout_errors": 0, "reason_tags": [ - "weaker_dev_fit" + "weaker_dev_fit", + "longer_without_gain" ] }, { @@ -11966,13 +13442,13 @@ "current_holdout_fp": 0, "current_holdout_fn": 1, "baseline_holdout_fp": 0, - "baseline_holdout_fn": 2, + "baseline_holdout_fn": 1, "winner_blind_holdout_fp": 0, "winner_blind_holdout_fn": 0, "current_blind_holdout_fp": 0, "current_blind_holdout_fn": 0, "baseline_blind_holdout_fp": 0, - "baseline_blind_holdout_fn": 1, + "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_fp": 0, "winner_judge_blind_holdout_fn": 0, @@ -11991,7 +13467,7 @@ "calibration": { "holdout": { "threshold": 0.48, - "mean_positive_score": 0.37, + "mean_positive_score": 0.45, "mean_non_trigger_score": 0.019, "mean_near_neighbor_score": 0.037, "min_positive_score": 0.08, @@ -12006,49 +13482,49 @@ }, "blind_holdout": { "threshold": 0.48, - "mean_positive_score": 0.902, - "mean_non_trigger_score": 0.168, - "mean_near_neighbor_score": 0.138, - "min_positive_score": 0.804, - "max_non_trigger_score": 0.386, - "score_gap": 0.418, - "threshold_margin": 0.094, - "positive_threshold_buffer": 0.324, - "negative_threshold_buffer": 0.094, - "boundary_case_count": 1, - "boundary_case_rate": 0.167, + "mean_positive_score": 0.82, + "mean_non_trigger_score": 0.059, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.645, + "max_non_trigger_score": 0.228, + "score_gap": 0.417, + "threshold_margin": 0.165, + "positive_threshold_buffer": 0.165, + "negative_threshold_buffer": 0.252, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, "risk_band": "healthy" }, "adversarial_holdout": { "threshold": 0.34, - "mean_positive_score": 0.827, - "mean_non_trigger_score": 0.027, - "mean_near_neighbor_score": 0.011, - "min_positive_score": 0.654, - "max_non_trigger_score": 0.056, - "score_gap": 0.598, - "threshold_margin": 0.284, - "positive_threshold_buffer": 0.314, - "negative_threshold_buffer": 0.284, - "boundary_case_count": 0, - "boundary_case_rate": 0.0, - "risk_band": "healthy" + "mean_positive_score": 0.907, + "mean_non_trigger_score": 0.083, + "mean_near_neighbor_score": 0.139, + "min_positive_score": 0.815, + "max_non_trigger_score": 0.277, + "score_gap": 0.538, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.475, + "negative_threshold_buffer": 0.063, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" } }, "judge_blind": { "winner": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "current": { "agreement_rate": 1.0, - "mean_confidence": 0.657, + "mean_confidence": 0.677, "rubric_version": "blind-v1" }, "baseline": { "agreement_rate": 1.0, - "mean_confidence": 0.69, + "mean_confidence": 0.677, "rubric_version": "blind-v1" } }, @@ -12111,7 +13587,8 @@ "causes": [ "no_candidate_outperformed_current", "current_holdout_gap_present", - "current_holdout_risk" + "current_holdout_risk", + "adversarial_watch_risk" ], "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", "gate_status": { @@ -12127,7 +13604,7 @@ "route_metrics": { "precision": 1.0, "recall": 1.0, - "average_margin": 0.659, + "average_margin": 0.586, "misroute_count": 0, "ambiguous_case_count": 0 } @@ -12141,8 +13618,8 @@ "promotion_state": "reference", "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", "tokens": 93, - "dev_errors": 1, - "holdout_errors": 2 + "dev_errors": 0, + "holdout_errors": 1 }, { "name": "governed-incident-command", @@ -12152,25 +13629,10 @@ "promotion_state": "kept_current", "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 37, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [] }, - { - "name": "governed-incident-command", - "role": "candidate", - "label": "Guardrail", - "ranking_state": "not_selected", - "promotion_state": "blocked", - "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", - "tokens": 51, - "dev_errors": 1, - "holdout_errors": 2, - "reason_tags": [ - "visible_holdout_regression", - "longer_without_gain" - ] - }, { "name": "governed-incident-command", "role": "candidate", @@ -12179,7 +13641,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 54, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -12193,7 +13655,7 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", "tokens": 78, - "dev_errors": 1, + "dev_errors": 0, "holdout_errors": 1, "reason_tags": [ "longer_without_gain" @@ -12207,9 +13669,24 @@ "promotion_state": "blocked", "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", "tokens": 78, + "dev_errors": 0, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, "dev_errors": 1, "holdout_errors": 1, "reason_tags": [ + "weaker_dev_fit", "longer_without_gain" ] } @@ -12408,11 +13885,11 @@ }, "reference_synthesis": { "skill_name": "yao-meta-skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "intent_confidence": { - "score": 30, - "band": "low", - "gate_passed": false + "score": 100, + "band": "high", + "gate_passed": true }, "github_benchmarks": [ { @@ -12446,49 +13923,51 @@ "source_tracks": [ { "source_type": "official", - "name": "Official workflow product ergonomics", + "name": "Official skill anatomy and context discipline", "evidence_mode": "curated-pattern-track", "matched_keywords": [ - "workflow" + "metadata", + "references" ], - "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", - "why_relevant": "This track matches: workflow." + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "why_relevant": "This track matches: metadata, references." }, { "source_type": "research", - "name": "Hypothesis-test-learn loop", + "name": "Human-in-the-loop verification", "evidence_mode": "curated-pattern-track", "matched_keywords": [ - "general fit" + "review", + "govern" ], - "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", - "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", - "why_relevant": "This track matches: general fit." + "borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.", + "avoid": "Do not force every skill through heavyweight review when the risk is low.", + "why_relevant": "This track matches: review, govern." }, { "source_type": "principles", "name": "Boundary-first design", "evidence_mode": "curated-pattern-track", "matched_keywords": [ - "general fit" + "route" ], "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", "avoid": "Do not expand execution assets until route boundaries stay clean.", - "why_relevant": "This track matches: general fit." + "why_relevant": "This track matches: route." } ], "synthesis": { "borrow_now": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", "Borrow the discipline of defining what the skill should not own before growing the package.", "Borrow the way it turns a messy workflow into a repeatable operating path.", "Borrow the clear execution entrypoints and command structure." ], "avoid_now": [ - "Do not mimic product polish that adds UI bulk without improving clarity.", - "Do not create experimental overhead that exceeds the skill's real risk tier.", + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not force every skill through heavyweight review when the risk is low.", "Do not expand execution assets until route boundaries stay clean.", "Do not import process overhead that only exists for that project's scale.", "Do not copy repo-specific commands or environment assumptions verbatim." @@ -12504,10 +13983,50 @@ "source_count": 6, "accepted": [ { - "name": "Official workflow product ergonomics", + "name": "Official skill anatomy and context discipline", "source_type": "official", - "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "Human-in-the-loop verification", + "source_type": "research", + "borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.", + "avoid": "Do not force every skill through heavyweight review when the risk is low.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "Boundary-first design", + "source_type": "principles", + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", "gates": { "recurrence": true, "generativity": true, @@ -12565,48 +14084,6 @@ } ], "deferred": [ - { - "name": "Hypothesis-test-learn loop", - "source_type": "research", - "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", - "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", - "gates": { - "recurrence": true, - "generativity": true, - "distinctiveness": false, - "boundary": true - }, - "passed": [ - "recurrence", - "generativity", - "boundary" - ], - "missing": [ - "distinctiveness" - ], - "score": 3 - }, - { - "name": "Boundary-first design", - "source_type": "principles", - "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", - "avoid": "Do not expand execution assets until route boundaries stay clean.", - "gates": { - "recurrence": true, - "generativity": true, - "distinctiveness": false, - "boundary": true - }, - "passed": [ - "recurrence", - "generativity", - "boundary" - ], - "missing": [ - "distinctiveness" - ], - "score": 3 - }, { "name": "multica-ai/andrej-karpathy-skills", "source_type": "github", @@ -12629,27 +14106,34 @@ "score": 3 } ], - "summary": "3 accepted, 3 deferred using threshold 4/4." + "summary": "5 accepted, 1 deferred using threshold 4/4." }, - "conflicts": [], + "conflicts": [ + { + "key": "lightweight_vs_governance", + "summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_preference": "lightweight or speed-first", + "benchmark_pressure": "governance or evaluation-heavy patterns" + } + ], "recommendation": { - "summary": "Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.", + "summary": "Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.", "borrow_now": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed." + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed." ], "avoid_for_now": [ - "Do not mimic product polish that adds UI bulk without improving clarity.", - "Do not create experimental overhead that exceeds the skill's real risk tier." + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not force every skill through heavyweight review when the risk is low." ], - "why": "Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package.", + "why": "There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", "user_decision_required": true }, "visibility": { "mode": "explicit", "user_decision_required": true, "reasons": [ - "intent_uncertain" + "design_conflict" ], "user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.", "reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers." @@ -12807,17 +14291,53 @@ }, "artifact_design": { "skill_name": "yao-meta-skill", - "design_system": "content-led editorial", + "design_system": "metric editorial", "primary_artifact": { - "key": "report", - "label": "Report or brief", - "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "key": "dashboard", + "label": "Dashboard or metrics page", + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", "matched_keywords": [ - "report", - "summary" + "metric", + "score", + "table", + "scorecard" ] }, "artifact_families": [ + { + "key": "dashboard", + "label": "Dashboard or metrics page", + "score": 4, + "matched_keywords": [ + "metric", + "score", + "table", + "scorecard" + ], + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." + }, + { + "key": "review_viewer", + "label": "Review viewer", + "score": 3, + "matched_keywords": [ + "review", + "viewer", + "audit" + ], + "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." + }, + { + "key": "code_or_cli", + "label": "Code, CLI, or implementation guide", + "score": 3, + "matched_keywords": [ + "code", + "script", + "command" + ], + "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit." + }, { "key": "report", "label": "Report or brief", @@ -12829,24 +14349,24 @@ "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." }, { - "key": "code_or_cli", - "label": "Code, CLI, or implementation guide", + "key": "visual_capture", + "label": "Screenshot or visual evidence", "score": 1, "matched_keywords": [ - "script" + "screenshot" ], - "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit." + "direction": "Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect." } ], "layout_patterns": [ - "thesis", - "evidence blocks", - "decision table", - "risks", - "next actions", - "prerequisites", - "commands", - "expected output" + "metric board", + "ranked signals", + "comparison rows", + "interpretation", + "action queue", + "summary", + "variant comparison", + "evidence" ], "design_tokens": { "type": [ @@ -12871,12 +14391,14 @@ ] }, "quality_gates": [ - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities.", + "Make differences visible instead of hiding them in prose.", + "Separate author-facing recommendations from reviewer-only evidence.", + "Surface conflicts clearly and keep routine benchmark synthesis quiet.", "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Mark destructive, networked, or external side-effect operations." ], "anti_patterns": [ "Do not copy Kami's fixed parchment background as a default.", @@ -12889,85 +14411,86 @@ }, "prompt_quality": { "skill_name": "yao-meta-skill", - "relevance": "prompt-heavy", + "relevance": "prompt-aware", "primary_task_family": { - "key": "dialogue_interaction", - "label": "Dialogue interaction", + "key": "execution_operation", + "label": "Execution operation", "matched_keywords": [ - "dialogue", - "conversation" + "workflow", + "runbook" ] }, "task_families": [ - { - "key": "dialogue_interaction", - "label": "Dialogue interaction", - "score": 2, - "matched_keywords": [ - "dialogue", - "conversation" - ], - "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", - "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", - "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." - }, { "key": "execution_operation", "label": "Execution operation", - "score": 1, + "score": 2, "matched_keywords": [ - "workflow" + "workflow", + "runbook" ], "role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." }, { - "key": "teaching_guidance", - "label": "Teaching guidance", + "key": "creative_generation", + "label": "Creative generation", "score": 1, "matched_keywords": [ - "teach" + "content" ], - "role_guidance": "Use a teacher role that adapts to learner level and avoids overloading the first pass.", - "task_guidance": "Explain through progressive steps, examples, and visible success checks.", - "format_guidance": "Return learner-facing sections, worked examples, checkpoints, and common mistakes." + "role_guidance": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.", + "task_guidance": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.", + "format_guidance": "Return options with rationale, selection criteria, and refinement paths." }, { - "key": "prompt_engineering", - "label": "Prompt engineering", + "key": "analytical_reasoning", + "label": "Analytical reasoning", "score": 1, "matched_keywords": [ - "prompt" + "decision" ], - "role_guidance": "Use a prompt engineer role only when role design materially improves execution.", - "task_guidance": "Map Role, Task, and Format into skill behavior rather than copying a large prompt template.", - "format_guidance": "Return a compact prompt contract plus tests, quality matrix, and usage notes." + "role_guidance": "Use an analyst role that separates evidence, inference, uncertainty, and recommendation.", + "task_guidance": "State assumptions, compare alternatives, and make the decision path inspectable.", + "format_guidance": "Return findings, evidence, tradeoffs, recommendation, and residual risks." + }, + { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "score": 1, + "matched_keywords": [ + "dialogue" + ], + "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." } ], "complexity": { - "band": "complex", - "score": 6, - "reason": "multiple inputs, constraints, or task families require tradeoff handling" + "band": "expert", + "score": 27, + "reason": "multiple task families plus governance, evaluation, or expert-level constraints" }, "need_model": { - "explicit_need": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "explicit_need": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.", - "scenario": "not yet explicit", + "scenario": "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions, an existing skill directory that needs refactor, evaluation, packaging, or governance hardening, target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution, benchmark references, local constraints, desired maturity tier, and review standards", "user_level": "infer from examples and standards; ask only if it changes output depth", - "success_standard": "usable output with clear validation cues" + "success_standard": "trigger boundaries must be tested with should-trigger and should-not-trigger cases, production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence, governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation, generated reports should be bilingual or reviewer-friendly when they are user-facing, each new asset must earn its place by reducing ambiguity, risk, or repeated work" }, "rtf_to_skill": { - "role": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", - "task": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", - "format": "Return concise prompts, decision points, and reviewer-visible assumptions." + "role": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", + "task": "Convert the job into ordered steps with validation checks and stop conditions.", + "format": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." }, "quality_matrix": [ { "key": "completeness", "label": "Completeness", - "score": 80, + "score": 100, "matched_signals": [ + "input", "output", "constraint", "standard" @@ -12977,8 +14500,10 @@ { "key": "clarity", "label": "Clarity", - "score": 80, - "matched_signals": [], + "score": 85, + "matched_signals": [ + "clear" + ], "repair": "Replace broad verbs with observable actions and define what done means." }, { @@ -13004,12 +14529,12 @@ { "key": "specificity", "label": "Specificity", - "score": 70, + "score": 80, "matched_signals": [], "repair": "Anchor wording in the user's audience, domain nouns, and target outcome." } ], - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "self_repair_checks": [ "Check explicit need, implicit need, scenario, user level, and success standard before deepening.", "Map Role, Task, and Format into skill behavior, not decorative prompt labels.", @@ -13024,8 +14549,8 @@ "checks": [ { "label": "Intent clarity", - "status": "needs review", - "detail": "30/100 intent confidence." + "status": "ready", + "detail": "100/100 intent confidence." }, { "label": "Benchmark coverage", @@ -13035,12 +14560,12 @@ { "label": "Pattern gate", "status": "ready", - "detail": "3 accepted, 3 deferred using threshold 4/4." + "detail": "5 accepted, 1 deferred using threshold 4/4." }, { "label": "Conflict handling", - "status": "ready", - "detail": "No material conflicts detected." + "status": "decision needed", + "detail": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure." }, { "label": "Output risk profile", @@ -13050,12 +14575,12 @@ { "label": "Artifact design profile", "status": "ready", - "detail": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." + "detail": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." }, { "label": "Prompt quality profile", "status": "ready", - "detail": "82.0/100 prompt-facing quality score." + "detail": "89.0/100 prompt-facing quality score." } ], "reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery." diff --git a/reports/review_annotations.json b/reports/review_annotations.json index 66f2b1d9..76fadc1f 100644 --- a/reports/review_annotations.json +++ b/reports/review_annotations.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, diff --git a/reports/review_waivers.json b/reports/review_waivers.json index c3a9d456..aaa4a9d5 100644 --- a/reports/review_waivers.json +++ b/reports/review_waivers.json @@ -2,35 +2,115 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "summary": { "waiver_count": 0, "active_count": 0, "expired_count": 0, "invalid_count": 0, "covered_gate_count": 0, - "covered_gate_keys": [] + "covered_gate_keys": [], + "waiver_candidate_count": 2, + "waiverable_open_count": 1, + "non_waivable_count": 1 }, "policy": { "blocker_waivers_allowed": false, "minimum_reason_chars": 20, "expires_required": true, - "known_gate_keys": [ + "review_studio_gate_keys": [ + "architecture-maintainability", "context-budget", "intent-canvas", "operations-loop", "output-lab", "permission-gates", "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "known_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", "registry-audit", "release-notes", "runtime-matrix", "skill-atlas", "trigger-lab", "trust-report" + ], + "waiverable_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "non_waivable_gate_keys": [ + "review-waivers", + "world-class-evidence" ] }, "waivers": [], + "waiver_candidates": [ + { + "gate_key": "output-lab", + "label": "Output Lab", + "status": "needs-reviewer-decision", + "waiver_allowed": true, + "decision_options": [ + "accepted-risk", + "false-positive", + "temporary-exception" + ], + "risk_summary": "review pending 5; model-executed 10; output failures 0", + "required_review": [ + "Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.", + "Reviewer names the release scope and expiry date.", + "Reviewer links output_review_adjudication or output_execution evidence." + ], + "suggested_evidence": "reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md", + "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." + }, + { + "gate_key": "world-class-evidence", + "label": "World-Class Evidence", + "status": "cannot-waive", + "waiver_allowed": false, + "risk_summary": "4 pending evidence entries; 1 human pending; 3 external pending", + "required_review": [ + "Do not use a waiver to claim public world-class readiness.", + "Either submit accepted ledger evidence or state that this release does not claim world-class completion.", + "Keep claim guard active until ledger summary.ready_to_claim_world_class is true." + ], + "suggested_evidence": "reports/world_class_evidence_ledger.md", + "suggested_command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .", + "world_class_boundary": "Non-waivable completion boundary." + } + ], "failures": [], "warnings": [], "artifacts": { diff --git a/reports/review_waivers.md b/reports/review_waivers.md index dc684bb3..c303e8bf 100644 --- a/reports/review_waivers.md +++ b/reports/review_waivers.md @@ -6,17 +6,63 @@ - Expired: `0` - Invalid: `0` - Covered gates: `none` +- Waiver candidates: `2` +- Open waiverable candidates: `1` +- Non-waivable boundaries: `1` ## Policy - Blocker waivers allowed: `False` - Minimum reason chars: `20` - Expiry is required for every waiver. +- World-class evidence completion cannot be waived; it can only be proven by accepted ledger evidence. +- Review Studio gates: `architecture-maintainability, context-budget, intent-canvas, operations-loop, output-lab, permission-gates, permission-runtime, python-compat, registry-audit, release-notes, review-waivers, runtime-matrix, skill-atlas, trigger-lab, trust-report, world-class-evidence` +- Waiverable gates: `architecture-maintainability, context-budget, intent-canvas, operations-loop, output-lab, permission-gates, permission-runtime, python-compat, registry-audit, release-notes, runtime-matrix, skill-atlas, trigger-lab, trust-report` +- Non-waivable gates: `review-waivers, world-class-evidence` ## Waivers - None +## Candidate Actions + +| Gate | Status | Waiver | Risk | Evidence | +| --- | --- | --- | --- | --- | +| `output-lab` | `needs-reviewer-decision` | `true` | review pending 5; model-executed 10; output failures 0 | `reports/output_review_adjudication.md` | +| `world-class-evidence` | `cannot-waive` | `false` | 4 pending evidence entries; 1 human pending; 3 external pending | `reports/world_class_evidence_ledger.md` | + +### Output Lab + +- gate: `output-lab` +- status: `needs-reviewer-decision` +- waiver allowed: `true` +- risk: review pending 5; model-executed 10; output failures 0 +- evidence: `reports/output_review_adjudication.md` +- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md` +- world-class boundary: Does not count as provider, human, or public world-class completion evidence. + +#### Required Review + +- Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority. +- Reviewer names the release scope and expiry date. +- Reviewer links output_review_adjudication or output_execution evidence. + +### World-Class Evidence + +- gate: `world-class-evidence` +- status: `cannot-waive` +- waiver allowed: `false` +- risk: 4 pending evidence entries; 1 human pending; 3 external pending +- evidence: `reports/world_class_evidence_ledger.md` +- verification: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .` +- world-class boundary: Non-waivable completion boundary. + +#### Required Review + +- Do not use a waiver to claim public world-class readiness. +- Either submit accepted ledger evidence or state that this release does not claim world-class completion. +- Keep claim guard active until ledger summary.ready_to_claim_world_class is true. + ## Failures - None diff --git a/reports/route_scorecard.json b/reports/route_scorecard.json index 18dce57e..9a15898c 100644 --- a/reports/route_scorecard.json +++ b/reports/route_scorecard.json @@ -15,7 +15,7 @@ "correct_count": 3, "precision": 1.0, "recall": 1.0, - "average_margin": 0.611 + "average_margin": 0.701 }, "team-frontend-review": { "expected_count": 3, @@ -31,7 +31,7 @@ "correct_count": 3, "precision": 1.0, "recall": 1.0, - "average_margin": 0.659 + "average_margin": 0.586 }, "no_route": { "expected_count": 4, @@ -77,17 +77,16 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.741, + "margin": 1.0, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.741, + "score": 1.0, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package", "team_operationalize", "transform_workflow" @@ -118,17 +117,16 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.535, + "margin": 0.661, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.535, + "score": 0.661, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ "eval_optimize", - "iterate_existing_skill", "reuse_package" ], "matched_negative_concepts": [] @@ -157,23 +155,22 @@ "expected_route": "yao-meta-skill", "predicted_route": "yao-meta-skill", "correct": true, - "margin": 0.558, + "margin": 0.441, "ambiguous": false, "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.558, + "score": 0.441, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ - "build_skill", "reuse_package" ], "matched_negative_concepts": [] }, { "name": "governed-incident-command", - "score": 0.269, + "score": 0.181, "passed_threshold": false, "threshold": 0.48, "matched_desired_concepts": [ @@ -216,7 +213,7 @@ }, { "name": "yao-meta-skill", - "score": 0.018, + "score": 0.012, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -256,7 +253,7 @@ }, { "name": "yao-meta-skill", - "score": 0.013, + "score": 0.008, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -329,13 +326,14 @@ "matched_desired_concepts": [ "comms_and_owners", "governed_workflow", + "incident_packet", "severity_assessment" ], "matched_negative_concepts": [] }, { "name": "yao-meta-skill", - "score": 0.021, + "score": 0.016, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], @@ -357,12 +355,12 @@ "expected_route": "governed-incident-command", "predicted_route": "governed-incident-command", "correct": true, - "margin": 0.805, + "margin": 0.505, "ambiguous": false, "ranked_routes": [ { "name": "governed-incident-command", - "score": 0.805, + "score": 0.505, "passed_threshold": true, "threshold": 0.48, "matched_desired_concepts": [ @@ -373,7 +371,7 @@ }, { "name": "yao-meta-skill", - "score": 0.189, + "score": 0.328, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [ @@ -397,23 +395,25 @@ "expected_route": "governed-incident-command", "predicted_route": "governed-incident-command", "correct": true, - "margin": 0.172, + "margin": 0.252, "ambiguous": false, "ranked_routes": [ { "name": "governed-incident-command", - "score": 0.62, + "score": 1.0, "passed_threshold": true, "threshold": 0.48, "matched_desired_concepts": [ "comms_and_owners", - "governed_workflow" + "governed_workflow", + "incident_packet", + "severity_assessment" ], "matched_negative_concepts": [] }, { "name": "yao-meta-skill", - "score": 0.448, + "score": 0.748, "passed_threshold": true, "threshold": 0.33, "matched_desired_concepts": [ @@ -483,7 +483,7 @@ "ranked_routes": [ { "name": "yao-meta-skill", - "score": 0.016, + "score": 0.008, "passed_threshold": false, "threshold": 0.33, "matched_desired_concepts": [], diff --git a/reports/route_scorecard.md b/reports/route_scorecard.md index b54eb657..ba62c9f4 100644 --- a/reports/route_scorecard.md +++ b/reports/route_scorecard.md @@ -9,9 +9,9 @@ | Route | Expected | Predicted | Precision | Recall | Avg Margin | | --- | ---: | ---: | ---: | ---: | ---: | -| `yao-meta-skill` | 3 | 3 | 1.0 | 1.0 | 0.611 | +| `yao-meta-skill` | 3 | 3 | 1.0 | 1.0 | 0.701 | | `team-frontend-review` | 3 | 3 | 1.0 | 1.0 | 0.805 | -| `governed-incident-command` | 3 | 3 | 1.0 | 1.0 | 0.659 | +| `governed-incident-command` | 3 | 3 | 1.0 | 1.0 | 0.586 | | `no_route` | 4 | 4 | 1.0 | 1.0 | - | ## Confusion Matrix diff --git a/reports/runtime_permission_probes.json b/reports/runtime_permission_probes.json index e68842ee..59c9a1a4 100644 --- a/reports/runtime_permission_probes.json +++ b/reports/runtime_permission_probes.json @@ -9,14 +9,34 @@ "subprocess" ], "summary": { - "target_count": 3, - "pass_count": 3, + "target_count": 4, + "pass_count": 4, "fail_count": 0, "native_enforcement_count": 0, - "metadata_fallback_count": 3, - "residual_risk_count": 3, + "metadata_fallback_count": 4, + "residual_risk_count": 4, "required_capability_count": 3, - "failure_count": 0 + "failure_count": 0, + "installer_enforcement_source_status": "present", + "installer_enforcement_target_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "installer_permission_capability_count": 3, + "world_class_native_evidence_ready": false, + "installer_enforcement_ready": true + }, + "installer_enforcement": { + "source": "reports/install_simulation.json", + "source_status": "present", + "package_dir_matches": true, + "summary": { + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, + "failure_count": 0 + } }, "targets": [ { @@ -26,6 +46,18 @@ "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "openai", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, "assurance": "metadata-fallback-explicit", "declared_capabilities": [ "file_write", @@ -111,6 +143,18 @@ "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "claude", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, "assurance": "metadata-fallback-explicit", "declared_capabilities": [ "file_write", @@ -181,6 +225,18 @@ "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "generic", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, "assurance": "metadata-fallback-explicit", "declared_capabilities": [ "file_write", @@ -243,6 +299,88 @@ "residual_risks": [ "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." ] + }, + { + "target": "vscode", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", + "permission_model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "installer_enforcement": { + "target": "vscode", + "source_status": "present", + "enforced": true, + "enforced_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "failure_details": [] + }, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "vscode adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "vscode adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "vscode adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "vscode permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "vscode target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "vscode target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "vscode target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "vscode target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "vscode target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "vscode review_required matches whether capabilities are required" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] } ], "failures": [], diff --git a/reports/runtime_permission_probes.md b/reports/runtime_permission_probes.md index 1e68c336..5e242e99 100644 --- a/reports/runtime_permission_probes.md +++ b/reports/runtime_permission_probes.md @@ -1,22 +1,35 @@ # Runtime Permission Probes -Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit. +Runtime permission probes verify that generated target adapters expose high-permission capabilities, make native-enforcement limits explicit, and link installer enforcement evidence when available. ## Summary - OK: `True` -- Targets probed: `3` -- Passed: `3` +- Targets probed: `4` +- Passed: `4` - Failed: `0` - Native enforcement targets: `0` -- Explicit metadata fallbacks: `3` +- Explicit metadata fallbacks: `4` +- Installer enforcement source: `present` +- Installer-enforced targets: `4` +- Installer permission failures: `0` +- World-class native evidence ready: `False` - Required capabilities: `file_write, network, subprocess` -| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk | -| --- | --- | --- | --- | --- | --- | -| `openai` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | -| `claude` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | -| `generic` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | +| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Installer Enforcement | Residual Risk | +| --- | --- | --- | --- | --- | --- | --- | +| `openai` | `pass` | `metadata-fallback-explicit` | `False` | `True` | `pass` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | +| `claude` | `pass` | `metadata-fallback-explicit` | `False` | `True` | `pass` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | +| `generic` | `pass` | `metadata-fallback-explicit` | `False` | `True` | `pass` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | +| `vscode` | `pass` | `metadata-fallback-explicit` | `False` | `True` | `pass` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | + +## Installer Enforcement + +- Source: `reports/install_simulation.json` +- Source status: `present` +- Package dir matches probe: `True` + +Installer enforcement means the package install simulation blocks missing capability approvals or target enforcement notes. It is supporting local distribution evidence, not proof of target-client native enforcement. ## Failures diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 75a03a72..547450a8 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,33 +2,61 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 150, - "script_count": 66, - "internal_module_count": 7, + "scanned_files": 245, + "script_count": 153, + "internal_module_count": 67, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], - "network_script_count": 2, - "network_policy_covered_count": 2, + "network_script_count": 3, + "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 47, + "file_write_script_count": 73, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 59, + "help_smoke_checked_count": 86, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 150, - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_hash_file_count": 245, + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" }, "failures": [], "warnings": [], "secrets": [], "scripts": [ + { + "path": "scripts/adaptation_patch_safety.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared adaptation patch target parsing and target-file baseline verification helpers.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/adaptation_report_contracts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared adaptation approval and regression report contract decoration helpers.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/adjudicate_output_review.py", "interface": "cli", @@ -43,6 +71,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/apply_adaptation.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Approval-gated adaptive patch application with dry-run, allowlist, regression, and rollback evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/benchmark_release_lock.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_benchmark_reproducibility.py to keep release-lock git status classification out of the report renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/build_confusion_matrix.py", "interface": "cli", @@ -71,6 +127,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/build_skill_atlas_layout.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by build_skill_atlas.py to render the static Skill Atlas HTML report.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/build_skill_atlas_opportunities.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by build_skill_atlas.py to keep no-route opportunity detection out of the atlas CLI.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/check_update.py", "interface": "cli", @@ -132,6 +216,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/compile_skill_targets.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by compile_skill for target platform model and contract builders.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/context_sizer.py", "interface": "cli", @@ -174,6 +272,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/cross_packager_contracts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by cross_packager.py to keep platform contract data separate from packaging flow.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/description_optimizer_reporting.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by optimize_description.py to render description optimization reports.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/diff_eval.py", "interface": "cli", @@ -188,6 +314,104 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/emit_telemetry_event.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_core.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py for shared report loading, comparison, and Markdown rendering helpers.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_phase_queue.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to prevent preflight and Review Studio phase-queue drift.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_release.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to verify release evidence refresh instructions.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_skill_os2_review.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to keep Skill OS 2.0 review summary drift checks out of the main consistency renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/evidence_consistency_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/export_skill_ir.py", "interface": "cli", @@ -234,6 +458,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/html_rendering.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Used by report renderers to escape HTML while preserving meaningful falsey values.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/import_output_review_decisions.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Imports human blind A/B reviewer decisions into the canonical output-review decision file.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/import_telemetry_events.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/init_skill.py", "interface": "cli", @@ -304,6 +570,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/output_review_privacy.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/prepare_output_review_kit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/prepare_world_class_submission_kit.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Prepares editable world-class evidence intake packets without counting drafts as accepted evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/probe_runtime_permissions.py", "interface": "cli", @@ -332,6 +640,68 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/propose_adaptation.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Turns redacted repeated preference patterns into proposal-only adaptation plans.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/provider_output_eval_runner.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": true, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [ + "https://api.openai.com/v1/responses", + "https://api.deepseek.com/chat/completions" + ], + "network_hosts": [ + "api.deepseek.com", + "api.openai.com" + ] + }, + { + "path": "scripts/python_compat_check.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Checks repository Python source for syntax that can pass locally but fail on the supported CI interpreter.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/reference_synthesis_markdown.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_reference_synthesis.py to keep reference synthesis modeling separate from Markdown rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/registry_audit.py", "interface": "cli", @@ -360,6 +730,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_architecture_maintainability.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_artifact_design_profile.py", "interface": "cli", @@ -388,6 +772,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a release-facing benchmark reproducibility manifest and Markdown report.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_context_reports.py", "interface": "cli", @@ -402,6 +800,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_daily_skillops_report.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a Daily SkillOps report that summarizes explicit-source patterns, proposals, approval state, and release evidence without scanning private logs or applying patches.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_description_drift_history.py", "interface": "cli", @@ -430,6 +842,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_evidence_consistency.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a cross-report evidence consistency gate for generated Skill OS reports.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_intent_confidence.py", "interface": "cli", @@ -626,6 +1052,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_skill_interpretation.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders the first-class skill interpretation report while reusing the Skill Overview v2 model and layout.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_skill_os2_audit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_skill_os2_coverage.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a Skill OS 2.0 blueprint-to-evidence coverage audit.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_skill_overview.py", "interface": "cli", @@ -672,6 +1140,132 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_telemetry_hook_recipes.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_weekly_curator_report.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a weekly SkillOps curator report from generated reports without scanning private logs or writing source files.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_claim_guard.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_evidence_intake.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Validates world-class human and external evidence intake packets before ledger review.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a machine-checkable ledger for world-class external and human evidence gaps.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_evidence_plan.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders an operator runbook for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_preflight.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a preflight checklist for collecting pending world-class evidence without accepting evidence.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_world_class_submission_review.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a read-only review queue for world-class evidence submissions before ledger acceptance.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/resource_boundary_check.py", "interface": "cli", @@ -686,6 +1280,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/review_studio_action_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by review_studio_actions.py to keep world-class evidence action cards out of generic Review Studio action wiring.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_actions.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep Review Studio action guidance and source refs out of HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_data.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to load Review Studio source reports and metric cards.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/review_studio_formatting.py", "interface": "internal-module", @@ -700,6 +1336,48 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/review_studio_gate_contract.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by review_studio_gates.py for stable gate scoring and contract checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_gate_helpers.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by review_studio_gates.py to keep reusable Review Studio gate helpers out of the main gate sequence.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_gates.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep Review Studio gate evaluation separate from HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/review_studio_layout.py", "interface": "internal-module", @@ -714,6 +1392,90 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/review_studio_output_review.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to render output review checklist cards.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_panels.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep small Review Studio panel renderers out of the main page composer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_skillops.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep SkillOps summary panels out of the main HTML renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_waivers.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to keep waiver candidate layout out of the main renderer.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_studio_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_studio.py to render world-class evidence cards.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/review_viewer_data.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_review_viewer.py to assemble Review Viewer data before HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/run_conformance_suite.py", "interface": "cli", @@ -798,6 +1560,34 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/skill_ir_paths.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by compiler, registry, conformance, and report scripts to locate canonical Skill IR artifacts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skill_os2_coverage_markdown.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_os2_coverage.py to keep coverage data assembly separate from Markdown rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/skill_report_charts.py", "interface": "internal-module", @@ -812,6 +1602,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/skill_report_i18n.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_overview.py to keep bilingual report copy and fallback rules out of HTML rendering.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/skill_report_layout.py", "interface": "internal-module", @@ -854,6 +1658,76 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/skill_report_sections.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to assemble overview report quality, risk, and asset sections.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skill_report_sources.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to load source files and scan overview package assets.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skill_report_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to summarize world-class evidence readiness and roadmap actions.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/skillops_opportunity.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_daily_skillops_report.py to score SkillOps opportunities without writing files.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/summarize_user_signals.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Scans an explicit local source file and summarizes redacted repeated user preference signals.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/sync_local_install.py", "interface": "cli", @@ -868,6 +1742,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/telemetry_native_host.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/trigger_eval.py", "interface": "cli", @@ -896,6 +1784,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/trust_check_scripts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Static script inventory helpers imported by trust_check.py.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/upgrade_check.py", "interface": "cli", @@ -938,6 +1840,188 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_evidence_contract.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to share intake validation and artifact integrity checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_human_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_operator_runbook_coordination.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Builds world-class operator coordination steps and release gates for render_world_class_operator_runbook.py.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_phase_queue.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by prepare_world_class_submission_kit.py to group repair rows into an operator execution queue.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_preflight_layout.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_preflight_markdown.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_world_class_preflight.py to keep Markdown layout separate from preflight data assembly.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_provider_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_repair_checklist.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared by submission kit generation to turn readiness blockers into actionable repair rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_source_checks.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to keep source-evidence readiness checks consistent.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_submission_kit_rendering.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared renderer for world-class submission kit Markdown and HTML artifacts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/world_class_submission_matrix.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Shared by submission kit rendering to summarize draft, artifact, and source-check readiness.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -952,6 +2036,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/yao_cli_adaptation_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep adaptive scan/proposal/apply command handlers outside the thin CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao_cli_config.py", "interface": "internal-module", @@ -965,6 +2063,132 @@ "uses_subprocess": false, "network_urls": [], "network_hosts": [] + }, + { + "path": "scripts/yao_cli_create_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_distribution_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep distribution and runtime gate handlers outside the thin CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_output_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep output evaluation and review handlers outside the thin CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_parser.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep CLI parser declarations separate from command orchestration.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_parser_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao_cli_parser.py to keep evidence command declarations out of the main parser module.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_parser_operations.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao_cli_parser.py to keep SkillOps, telemetry, and review-loop command declarations out of the main parser module.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_report_commands.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to keep report and evidence command handlers out of the CLI orchestrator.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_runtime.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py and command modules for shared subprocess execution and JSON payload parsing.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/yao_cli_telemetry.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by yao.py to record opt-in metadata-only CLI run telemetry.", + "has_argparse": true, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] } ], "dependencies": { @@ -980,10 +2204,11 @@ "present": true, "path": "security/network_policy.json", "schema_version": "1.0", - "network_script_count": 2, + "network_script_count": 3, "covered_scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "missing_scripts": [], "mismatches": [], @@ -996,11 +2221,11 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 59, - "checked_count": 59, - "passed_count": 59, + "candidate_count": 86, + "checked_count": 86, + "passed_count": 86, "failed_count": 0, - "skipped_count": 7, + "skipped_count": 67, "failed_scripts": [], "results": [ { @@ -1013,6 +2238,16 @@ "stdout_excerpt": "usage: adjudicate_output_review.py [-h] [--blind-pack BLIND_PACK]\n [--answer-key ANSWER_KEY]\n [--decisions DECISIONS]\n [--output-json OUTP", "stderr_excerpt": "" }, + { + "path": "scripts/apply_adaptation.py", + "command": "python3 scripts/apply_adaptation.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: apply_adaptation.py [-h] [--proposal-id PROPOSAL_ID]\n [--patch-file PATCH_FILE]\n [--proposals-json PROPOSALS_JSON]\n [--approval-ledger APPROVAL_LEDGER]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/build_confusion_matrix.py", "command": "python3 scripts/build_confusion_matrix.py --help", @@ -1113,6 +2348,16 @@ "stdout_excerpt": "usage: diff_eval.py [-h] before after\n\nDiff two eval reports.\n\npositional arguments:\n before\n after\n\noptions:\n -h, --help show this help message and exit", "stderr_excerpt": "" }, + { + "path": "scripts/emit_telemetry_event.py", + "command": "python3 scripts/emit_telemetry_event.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: emit_telemetry_event.py [-h] [--output-jsonl OUTPUT_JSONL]\n [--event {review_event,script_run,skill_activation,skill_output}]\n [--activation-type {explicit,implicit,manual,u", + "stderr_excerpt": "" + }, { "path": "scripts/export_skill_ir.py", "command": "python3 scripts/export_skill_ir.py --help", @@ -1143,6 +2388,26 @@ "stdout_excerpt": "usage: governance_check.py [-h] [--require-manifest] skill_dir\n\nCheck skill governance metadata and lifecycle readiness.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit\n --require-manifest", "stderr_excerpt": "" }, + { + "path": "scripts/import_output_review_decisions.py", + "command": "python3 scripts/import_output_review_decisions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: import_output_review_decisions.py [-h] --input INPUT\n [--format {auto,json,jsonl,csv}]\n [--blind-pack BLIND_PACK]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/import_telemetry_events.py", + "command": "python3 scripts/import_telemetry_events.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: import_telemetry_events.py [-h] --input-jsonl INPUT_JSONL\n [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OU", + "stderr_excerpt": "" + }, { "path": "scripts/init_skill.py", "command": "python3 scripts/init_skill.py --help", @@ -1193,6 +2458,26 @@ "stdout_excerpt": "usage: optimize_description.py [-h] --description-file DESCRIPTION_FILE\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n --dev-cases DEV_CASES\n ", "stderr_excerpt": "" }, + { + "path": "scripts/prepare_output_review_kit.py", + "command": "python3 scripts/prepare_output_review_kit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: prepare_output_review_kit.py [-h] [--blind-pack-json BLIND_PACK_JSON]\n [--blind-pack-md BLIND_PACK_MD]\n [--decisions DECISIONS]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/prepare_world_class_submission_kit.py", + "command": "python3 scripts/prepare_world_class_submission_kit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: prepare_world_class_submission_kit.py [-h] [--output-dir OUTPUT_DIR]\n [--evidence-key EVIDENCE_KEY]\n [--overwrite]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/probe_runtime_permissions.py", "command": "python3 scripts/probe_runtime_permissions.py --help", @@ -1200,7 +2485,7 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: probe_runtime_permissions.py [-h] [--package-dir PACKAGE_DIR]\n [--target {openai,claude,generic}]\n [--output-json OUTPUT_JSON]\n ", + "stdout_excerpt": "usage: probe_runtime_permissions.py [-h] [--package-dir PACKAGE_DIR]\n [--target {openai,claude,generic,vscode}]\n [--output-json OUTPUT_JSON]\n ", "stderr_excerpt": "" }, { @@ -1213,6 +2498,36 @@ "stdout_excerpt": "usage: promotion_checker.py [-h] [--optimization-suite OPTIMIZATION_SUITE]\n [--route-scorecard ROUTE_SCORECARD]\n [--output-json OUTPUT_JSON]\n [--output-md OUTP", "stderr_excerpt": "" }, + { + "path": "scripts/propose_adaptation.py", + "command": "python3 scripts/propose_adaptation.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: propose_adaptation.py [-h] [--patterns-json PATTERNS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/provider_output_eval_runner.py", + "command": "python3 scripts/provider_output_eval_runner.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: provider_output_eval_runner.py [-h] [--provider PROVIDER]\n [--base-url BASE_URL]\n [--api-format {responses,chat-completions}]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/python_compat_check.py", + "command": "python3 scripts/python_compat_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: python_compat_check.py [-h] [--path PATH]\n [--target-python TARGET_PYTHON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/registry_audit.py", "command": "python3 scripts/registry_audit.py --help", @@ -1233,6 +2548,16 @@ "stdout_excerpt": "usage: render_adoption_drift_report.py [-h] [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_architecture_maintainability.py", + "command": "python3 scripts/render_architecture_maintainability.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_architecture_maintainability.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--warn-lines WARN_LINES]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_artifact_design_profile.py", "command": "python3 scripts/render_artifact_design_profile.py --help", @@ -1253,6 +2578,16 @@ "stdout_excerpt": "usage: render_baseline_compare.py [-h] --entry ENTRY\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a baseline comparison report from description optimization ", "stderr_excerpt": "" }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "command": "python3 scripts/render_benchmark_reproducibility.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_benchmark_reproducibility.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_context_reports.py", "command": "python3 scripts/render_context_reports.py --help", @@ -1263,6 +2598,16 @@ "stdout_excerpt": "usage: render_context_reports.py [-h] [--generated-at GENERATED_AT]\n\nRender context budget reports for root and example skills.\n\noptions:\n -h, --help show this help message and exit\n --generated-at GENERATED_AT\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_daily_skillops_report.py", + "command": "python3 scripts/render_daily_skillops_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_daily_skillops_report.py [-h] [--source SOURCE]\n [--patterns-json PATTERNS_JSON]\n [--proposals-json PROPOSALS_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_description_drift_history.py", "command": "python3 scripts/render_description_drift_history.py --help", @@ -1283,6 +2628,16 @@ "stdout_excerpt": "usage: render_eval_dashboard.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_evidence_consistency.py", + "command": "python3 scripts/render_evidence_consistency.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_evidence_consistency.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [", + "stderr_excerpt": "" + }, { "path": "scripts/render_intent_confidence.py", "command": "python3 scripts/render_intent_confidence.py --help", @@ -1340,7 +2695,7 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: render_portability_report.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a portability score from neutral metadata, contracts, and snapshots.\n\noptions:\n -h, --help ", + "stdout_excerpt": "usage: render_portability_report.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [skill_dir]\n\nRender a portability score from neutral metadata, contracts, a", "stderr_excerpt": "" }, { @@ -1423,6 +2778,36 @@ "stdout_excerpt": "usage: render_review_waivers.py [-h] [--waivers-json WAIVERS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED", "stderr_excerpt": "" }, + { + "path": "scripts/render_skill_interpretation.py", + "command": "python3 scripts/render_skill_interpretation.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_interpretation.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender the first-class HTML skill interpretation rep", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_skill_os2_audit.py", + "command": "python3 scripts/render_skill_os2_audit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_os2_audit.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [skill_dir]\n\nRender a", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_skill_os2_coverage.py", + "command": "python3 scripts/render_skill_os2_coverage.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_os2_coverage.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [skill_di", + "stderr_excerpt": "" + }, { "path": "scripts/render_skill_overview.py", "command": "python3 scripts/render_skill_overview.py --help", @@ -1453,6 +2838,96 @@ "stdout_excerpt": "usage: render_system_model.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a systems-thinking model for a skill package.\n\npositional arguments:\n sk", "stderr_excerpt": "" }, + { + "path": "scripts/render_telemetry_hook_recipes.py", + "command": "python3 scripts/render_telemetry_hook_recipes.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_telemetry_hook_recipes.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--output-jsonl OUTPUT_JSONL]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_weekly_curator_report.py", + "command": "python3 scripts/render_weekly_curator_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_weekly_curator_report.py [-h] [--daily-json DAILY_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_claim_guard.py", + "command": "python3 scripts/render_world_class_claim_guard.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_claim_guard.py [-h] [--claim-surface CLAIM_SURFACE]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_evidence_intake.py", + "command": "python3 scripts/render_world_class_evidence_intake.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_intake.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "command": "python3 scripts/render_world_class_evidence_ledger.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_ledger.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--submissions-dir SUBMISSIONS_DIR]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_evidence_plan.py", + "command": "python3 scripts/render_world_class_evidence_plan.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_plan.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_operator_runbook.py", + "command": "python3 scripts/render_world_class_operator_runbook.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_operator_runbook.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_preflight.py", + "command": "python3 scripts/render_world_class_preflight.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_preflight.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n [--output-md OU", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_world_class_submission_review.py", + "command": "python3 scripts/render_world_class_submission_review.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_submission_review.py [-h]\n [--submissions-dir SUBMISSIONS_DIR]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/resource_boundary_check.py", "command": "python3 scripts/resource_boundary_check.py --help", @@ -1523,6 +2998,16 @@ "stdout_excerpt": "usage: simulate_install.py [-h] [--package-dir PACKAGE_DIR]\n [--install-root INSTALL_ROOT]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--generated-at G", "stderr_excerpt": "" }, + { + "path": "scripts/summarize_user_signals.py", + "command": "python3 scripts/summarize_user_signals.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: summarize_user_signals.py [-h] --source SOURCE\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--min-support MIN_SUPPORT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/sync_local_install.py", "command": "python3 scripts/sync_local_install.py --help", @@ -1530,7 +3015,17 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: sync_local_install.py [-h] [--root ROOT] [--install-dir INSTALL_DIR]\n [--dry-run]\n\nSync the current yao-meta-skill source into a managed local skill mirror.\n\noptions:\n -h, --help show this help", + "stdout_excerpt": "usage: sync_local_install.py [-h] [--root ROOT] [--install-dir INSTALL_DIR]\n [--package-dir PACKAGE_DIR]\n [--generated-at GENERATED_AT]\n [--skip-install-pre", + "stderr_excerpt": "" + }, + { + "path": "scripts/telemetry_native_host.py", + "command": "python3 scripts/telemetry_native_host.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: telemetry_native_host.py [-h] [--output-jsonl OUTPUT_JSONL]\n [--message-json MESSAGE_JSON] [--stdio]\n [--dry-run] [--write-manifest WRITE_MANIFEST]\n ", "stderr_excerpt": "" }, { @@ -1590,23 +3085,151 @@ "timed_out": false, "passed": true, "has_help_text": true, - "stdout_excerpt": "usage: yao.py [-h]\n {init,quickstart,validate,optimize-description,promote-check,review,release-snapshot,workspace-flow,report,skill-report,review-viewer,review-studio,reference-scan,github-benchmark-scan,intent-confidence,inte", + "stdout_excerpt": "usage: yao.py [-h] [--record-cli-telemetry] [--no-cli-telemetry]\n [--telemetry-events-jsonl TELEMETRY_EVENTS_JSONL]\n {init,quickstart,validate,optimize-description,promote-check,python-compat,architecture-audit,rev", "stderr_excerpt": "" } ], "skipped": [ + { + "path": "scripts/adaptation_patch_safety.py", + "reason": "internal module" + }, + { + "path": "scripts/adaptation_report_contracts.py", + "reason": "internal module" + }, + { + "path": "scripts/benchmark_release_lock.py", + "reason": "internal module" + }, + { + "path": "scripts/build_skill_atlas_layout.py", + "reason": "internal module" + }, + { + "path": "scripts/build_skill_atlas_opportunities.py", + "reason": "internal module" + }, + { + "path": "scripts/compile_skill_targets.py", + "reason": "internal module" + }, + { + "path": "scripts/cross_packager_contracts.py", + "reason": "internal module" + }, + { + "path": "scripts/description_optimizer_reporting.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_core.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_phase_queue.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_release.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_skill_os2_review.py", + "reason": "internal module" + }, + { + "path": "scripts/evidence_consistency_world_class.py", + "reason": "internal module" + }, + { + "path": "scripts/html_rendering.py", + "reason": "internal module" + }, + { + "path": "scripts/output_review_privacy.py", + "reason": "internal module" + }, + { + "path": "scripts/reference_synthesis_markdown.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_action_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_actions.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_data.py", + "reason": "internal module" + }, { "path": "scripts/review_studio_formatting.py", "reason": "internal module" }, + { + "path": "scripts/review_studio_gate_contract.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_gate_helpers.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_gates.py", + "reason": "internal module" + }, { "path": "scripts/review_studio_layout.py", "reason": "internal module" }, + { + "path": "scripts/review_studio_output_review.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_panels.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_skillops.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_waivers.py", + "reason": "internal module" + }, + { + "path": "scripts/review_studio_world_class.py", + "reason": "internal module" + }, + { + "path": "scripts/review_viewer_data.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_ir_paths.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_os2_coverage_markdown.py", + "reason": "internal module" + }, { "path": "scripts/skill_report_charts.py", "reason": "internal module" }, + { + "path": "scripts/skill_report_i18n.py", + "reason": "internal module" + }, { "path": "scripts/skill_report_layout.py", "reason": "internal module" @@ -1619,9 +3242,121 @@ "path": "scripts/skill_report_model.py", "reason": "internal module" }, + { + "path": "scripts/skill_report_sections.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_sources.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_world_class.py", + "reason": "internal module" + }, + { + "path": "scripts/skillops_opportunity.py", + "reason": "internal module" + }, + { + "path": "scripts/trust_check_scripts.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_evidence_contract.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_human_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_native_permission_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_native_telemetry_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_operator_runbook_coordination.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_phase_queue.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_preflight_layout.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_preflight_markdown.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_provider_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_repair_checklist.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_source_checks.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_submission_kit_rendering.py", + "reason": "internal module" + }, + { + "path": "scripts/world_class_submission_matrix.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_adaptation_commands.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_config.py", "reason": "internal module" + }, + { + "path": "scripts/yao_cli_create_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_distribution_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_output_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_parser.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_parser_evidence.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_parser_operations.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_report_commands.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_runtime.py", + "reason": "internal module" + }, + { + "path": "scripts/yao_cli_telemetry.py", + "reason": "internal module" } ] }, @@ -1654,11 +3389,12 @@ "status": "approved", "scripts": [ "scripts/check_update.py", - "scripts/github_benchmark_scan.py" + "scripts/github_benchmark_scan.py", + "scripts/provider_output_eval_runner.py" ], "reviewer": "Yao Team", "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", - "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "reason": "Network use is bounded to public GitHub metadata, benchmark discovery, update checks, and reviewed provider output-eval endpoints, with allowed hosts reviewed in security/network_policy.json.", "expires_at": "2026-09-30", "evidence": [ "security/network_policy.json", @@ -1667,7 +3403,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" }, "validation": [] }, @@ -1676,6 +3413,7 @@ "status": "approved", "scripts": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -1684,18 +3422,29 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/python_compat_check.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -1710,15 +3459,29 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trust_check.py", "scripts/upgrade_check.py", "scripts/verify_package.py" @@ -1735,7 +3498,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" }, "validation": [] }, @@ -1743,13 +3507,18 @@ "capability": "subprocess", "status": "approved", "scripts": [ + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", "scripts/trust_check.py", - "scripts/yao.py" + "scripts/yao.py", + "scripts/yao_cli_runtime.py" ], "reviewer": "Yao Team", "scope": "Local deterministic validation, packaging, install simulation, and CI orchestration commands.", @@ -1762,7 +3531,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" }, "validation": [] } diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index df2c282c..e221d69c 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `150` -- Scripts: `66` -- Internal script modules: `7` +- Scanned files: `245` +- Scripts: `153` +- Internal script modules: `67` - Secret findings: `0` -- Network-capable scripts: `2` -- Network policy covered scripts: `2` +- Network-capable scripts: `3` +- Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `47` +- File-write scripts: `73` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `59` +- CLI help smoke checked: `86` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `150` -- Package SHA256: `742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f` +- Package hash files: `245` +- Package SHA256: `5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257` ## Failures @@ -36,7 +36,7 @@ - Policy file: `security/network_policy.json` - Present: `True` -- Covered scripts: `2` +- Covered scripts: `3` - Missing scripts: `none` - Mismatches: `0` @@ -54,42 +54,72 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `59` -- Passed scripts: `59` +- Checked scripts: `86` +- Passed scripts: `86` - Failed scripts: `none` ## Script Surface | Script | Interface | Declared | Argparse | Main Guard | Input | Network | File Write | Subprocess | Reason | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| scripts/adaptation_patch_safety.py | internal-module | True | False | False | False | False | False | False | Shared adaptation patch target parsing and target-file baseline verification helpers. | +| scripts/adaptation_report_contracts.py | internal-module | True | False | False | False | False | False | False | Shared adaptation approval and regression report contract decoration helpers. | | scripts/adjudicate_output_review.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/apply_adaptation.py | cli | True | True | True | False | False | True | True | Approval-gated adaptive patch application with dry-run, allowlist, regression, and rollback evidence. | +| scripts/benchmark_release_lock.py | internal-module | True | False | False | False | False | False | True | Imported by render_benchmark_reproducibility.py to keep release-lock git status classification out of the report renderer. | | scripts/build_confusion_matrix.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/build_skill_atlas.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/build_skill_atlas_layout.py | internal-module | True | False | False | False | False | False | False | Imported by build_skill_atlas.py to render the static Skill Atlas HTML report. | +| scripts/build_skill_atlas_opportunities.py | internal-module | True | False | False | False | False | False | False | Imported by build_skill_atlas.py to keep no-route opportunity detection out of the atlas CLI. | | scripts/check_update.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/ci_test.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/collect_feedback.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/compile_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/compile_skill_targets.py | internal-module | True | False | False | False | False | False | False | Imported by compile_skill for target platform model and contract builders. | | scripts/context_sizer.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/create_iteration_snapshot.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/cross_packager.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/cross_packager_contracts.py | internal-module | True | False | False | False | False | False | False | Imported by cross_packager.py to keep platform contract data separate from packaging flow. | +| scripts/description_optimizer_reporting.py | internal-module | True | False | False | False | False | False | False | Imported by optimize_description.py to render description optimization reports. | | scripts/diff_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/emit_telemetry_event.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/evidence_consistency_artifact_roles.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts. | +| scripts/evidence_consistency_core.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py for shared report loading, comparison, and Markdown rendering helpers. | +| scripts/evidence_consistency_phase_queue.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to prevent preflight and Review Studio phase-queue drift. | +| scripts/evidence_consistency_release.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to verify release evidence refresh instructions. | +| scripts/evidence_consistency_skill_os2_review.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to keep Skill OS 2.0 review summary drift checks out of the main consistency renderer. | +| scripts/evidence_consistency_world_class.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks. | | scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/html_rendering.py | internal-module | True | False | False | False | False | False | False | Used by report renderers to escape HTML while preserving meaningful falsey values. | +| scripts/import_output_review_decisions.py | cli | True | True | True | False | False | True | False | Imports human blind A/B reviewer decisions into the canonical output-review decision file. | +| scripts/import_telemetry_events.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/init_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/judge_blind_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/local_output_eval_runner.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/optimize_description.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/output_review_privacy.py | internal-module | True | False | False | False | False | False | False | Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively. | +| scripts/prepare_output_review_kit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/prepare_world_class_submission_kit.py | cli | True | True | True | False | False | True | False | Prepares editable world-class evidence intake packets without counting drafts as accepted evidence. | | scripts/probe_runtime_permissions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/promotion_checker.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/propose_adaptation.py | cli | True | True | True | False | False | True | False | Turns redacted repeated preference patterns into proposal-only adaptation plans. | +| scripts/provider_output_eval_runner.py | cli | False | True | True | False | True | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/python_compat_check.py | cli | True | True | True | False | False | True | False | Checks repository Python source for syntax that can pass locally but fail on the supported CI interpreter. | +| scripts/reference_synthesis_markdown.py | internal-module | True | False | False | False | False | False | False | Imported by render_reference_synthesis.py to keep reference synthesis modeling separate from Markdown rendering. | | scripts/registry_audit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_adoption_drift_report.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_architecture_maintainability.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_artifact_design_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_baseline_compare.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_benchmark_reproducibility.py | cli | True | True | True | False | False | True | True | Renders a release-facing benchmark reproducibility manifest and Markdown report. | | scripts/render_context_reports.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_daily_skillops_report.py | cli | True | True | True | False | False | True | False | Renders a Daily SkillOps report that summarizes explicit-source patterns, proposals, approval state, and release evidence without scanning private logs or applying patches. | | scripts/render_description_drift_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_eval_dashboard.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_evidence_consistency.py | cli | True | True | True | False | False | True | True | Renders a cross-report evidence consistency gate for generated Skill OS reports. | | scripts/render_intent_confidence.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_intent_dialogue.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_iteration_directions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | @@ -104,27 +134,84 @@ | scripts/render_review_studio.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_review_viewer.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_review_waivers.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_skill_interpretation.py | cli | True | True | True | False | False | True | False | Renders the first-class skill interpretation report while reusing the Skill Overview v2 model and layout. | +| scripts/render_skill_os2_audit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_skill_os2_coverage.py | cli | True | True | True | False | False | True | False | Renders a Skill OS 2.0 blueprint-to-evidence coverage audit. | | scripts/render_skill_overview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_social_preview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_system_model.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_telemetry_hook_recipes.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_weekly_curator_report.py | cli | True | True | True | False | False | True | False | Renders a weekly SkillOps curator report from generated reports without scanning private logs or writing source files. | +| scripts/render_world_class_claim_guard.py | cli | True | True | True | False | False | True | False | Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists. | +| scripts/render_world_class_evidence_intake.py | cli | True | True | True | False | False | True | False | Validates world-class human and external evidence intake packets before ledger review. | +| scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. | +| scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_world_class_operator_runbook.py | cli | True | True | True | False | False | True | False | Renders an operator runbook for collecting pending world-class evidence without accepting evidence. | +| scripts/render_world_class_preflight.py | cli | True | True | True | False | False | True | False | Renders a preflight checklist for collecting pending world-class evidence without accepting evidence. | +| scripts/render_world_class_submission_review.py | cli | True | True | True | False | False | True | False | Renders a read-only review queue for world-class evidence submissions before ledger acceptance. | | scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/review_studio_action_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by review_studio_actions.py to keep world-class evidence action cards out of generic Review Studio action wiring. | +| scripts/review_studio_actions.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to keep Review Studio action guidance and source refs out of HTML rendering. | +| scripts/review_studio_data.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to load Review Studio source reports and metric cards. | | scripts/review_studio_formatting.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to format report dictionaries as audit UI panels. | +| scripts/review_studio_gate_contract.py | internal-module | True | False | False | False | False | False | False | Imported by review_studio_gates.py for stable gate scoring and contract checks. | +| scripts/review_studio_gate_helpers.py | internal-module | True | False | False | False | False | False | False | Imported by review_studio_gates.py to keep reusable Review Studio gate helpers out of the main gate sequence. | +| scripts/review_studio_gates.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to keep Review Studio gate evaluation separate from HTML rendering. | | scripts/review_studio_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to keep Review Studio layout and CSS out of gate logic. | +| scripts/review_studio_output_review.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to render output review checklist cards. | +| scripts/review_studio_panels.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to keep small Review Studio panel renderers out of the main page composer. | +| scripts/review_studio_skillops.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to keep SkillOps summary panels out of the main HTML renderer. | +| scripts/review_studio_waivers.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to keep waiver candidate layout out of the main renderer. | +| scripts/review_studio_world_class.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to render world-class evidence cards. | +| scripts/review_viewer_data.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_viewer.py to assemble Review Viewer data before HTML rendering. | | scripts/run_conformance_suite.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/run_description_optimization_suite.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/run_eval_suite.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/run_output_eval.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/run_output_execution.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/simulate_install.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/skill_ir_paths.py | internal-module | True | False | False | False | False | False | False | Imported by compiler, registry, conformance, and report scripts to locate canonical Skill IR artifacts. | +| scripts/skill_os2_coverage_markdown.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_os2_coverage.py to keep coverage data assembly separate from Markdown rendering. | | scripts/skill_report_charts.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to render inline SVG report charts. | +| scripts/skill_report_i18n.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to keep bilingual report copy and fallback rules out of HTML rendering. | | scripts/skill_report_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to keep overview report layout and CSS out of data rendering. | | scripts/skill_report_metrics.py | internal-module | True | False | False | False | False | False | False | Imported by skill_report_model.py to calculate overview report metrics. | | scripts/skill_report_model.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to build the v2 report data model. | +| scripts/skill_report_sections.py | internal-module | True | False | False | False | False | False | False | Imported by skill_report_model.py to assemble overview report quality, risk, and asset sections. | +| scripts/skill_report_sources.py | internal-module | True | False | False | False | False | False | False | Imported by skill_report_model.py to load source files and scan overview package assets. | +| scripts/skill_report_world_class.py | internal-module | True | False | False | False | False | False | False | Imported by skill_report_model.py to summarize world-class evidence readiness and roadmap actions. | +| scripts/skillops_opportunity.py | internal-module | True | False | False | False | False | False | False | Imported by render_daily_skillops_report.py to score SkillOps opportunities without writing files. | +| scripts/summarize_user_signals.py | cli | True | True | True | False | False | True | False | Scans an explicit local source file and summarizes redacted repeated user preference signals. | | scripts/sync_local_install.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/telemetry_native_host.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/trigger_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/trust_check.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/trust_check_scripts.py | internal-module | True | True | True | False | False | False | False | Static script inventory helpers imported by trust_check.py. | | scripts/upgrade_check.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | +| scripts/world_class_human_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows. | +| scripts/world_class_native_permission_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows. | +| scripts/world_class_native_telemetry_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows. | +| scripts/world_class_operator_runbook_coordination.py | internal-module | True | False | False | False | False | False | False | Builds world-class operator coordination steps and release gates for render_world_class_operator_runbook.py. | +| scripts/world_class_phase_queue.py | internal-module | True | False | False | False | False | False | False | Imported by prepare_world_class_submission_kit.py to group repair rows into an operator execution queue. | +| scripts/world_class_preflight_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly. | +| scripts/world_class_preflight_markdown.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep Markdown layout separate from preflight data assembly. | +| scripts/world_class_provider_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows. | +| scripts/world_class_repair_checklist.py | internal-module | True | False | False | False | False | False | False | Shared by submission kit generation to turn readiness blockers into actionable repair rows. | +| scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | +| scripts/world_class_submission_kit_rendering.py | internal-module | True | False | False | False | False | False | False | Shared renderer for world-class submission kit Markdown and HTML artifacts. | +| scripts/world_class_submission_matrix.py | internal-module | True | False | False | False | False | False | False | Shared by submission kit rendering to summarize draft, artifact, and source-check readiness. | | scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/yao_cli_adaptation_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep adaptive scan/proposal/apply command handlers outside the thin CLI orchestrator. | | scripts/yao_cli_config.py | internal-module | True | False | False | False | False | False | False | Imported by yao.py for CLI target maps and side-effect-free shaping helpers. | +| scripts/yao_cli_create_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator. | +| scripts/yao_cli_distribution_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep distribution and runtime gate handlers outside the thin CLI orchestrator. | +| scripts/yao_cli_output_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep output evaluation and review handlers outside the thin CLI orchestrator. | +| scripts/yao_cli_parser.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep CLI parser declarations separate from command orchestration. | +| scripts/yao_cli_parser_evidence.py | internal-module | True | True | False | False | False | False | False | Imported by yao_cli_parser.py to keep evidence command declarations out of the main parser module. | +| scripts/yao_cli_parser_operations.py | internal-module | True | True | False | False | False | False | False | Imported by yao_cli_parser.py to keep SkillOps, telemetry, and review-loop command declarations out of the main parser module. | +| scripts/yao_cli_report_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep report and evidence command handlers out of the CLI orchestrator. | +| scripts/yao_cli_runtime.py | internal-module | True | False | False | False | False | False | True | Imported by yao.py and command modules for shared subprocess execution and JSON payload parsing. | +| scripts/yao_cli_telemetry.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to record opt-in metadata-only CLI run telemetry. | diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html new file mode 100644 index 00000000..8366ba87 --- /dev/null +++ b/reports/skill-interpretation.html @@ -0,0 +1,1005 @@ + + + + + + yao-meta-skill Skill 生成审计报告 + + + + +
+ +
+
+ Skill audit + +
+
+
+
+
+
+
+
+

YAO Skill 生成审计报告YAO Skill Generation Audit

+

yao-meta-skill

+

技能审计报告Yao Meta Skill Audit Report

+

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

+
技能名称：yao-meta-skillSkill name: yao-meta-skill成熟度：治理Maturity: governed格式：Agent SkillsFormat: agent-skills更新时间：2026-03-31Updated: 2026-03-31
+
openaiopenaiclaudeclaudegenericgenericvscodevscode
+
+ +
+
完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
+
+ +
+
+
+

技能概述Overview

+

先用一屏说明这个 Skill 是什么、给谁用、交付什么。A first-screen explanation of what this skill is, who it serves, and what it delivers.

+
+
+
+

作用定位Role

+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html, reports/skill-interpretation.jsonDeliverables: SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html, reports/skill-interpretation.json
+
+
交付流程Delivery Flow输入材料Input materialSkill 包体Skill package可复用能力Reusable capability
交付流程把用户输入、生成的包体和可复用能力放在一条线上。The delivery flow places user input, generated package, and reusable capability on one path.
+
+
+
+ +
+
+
+

总览指标Metrics

+

分数来自本地文件和 reports 证据，缺失时明确标为证据不足。Scores are derived from package files and reports; missing inputs are shown as evidence gaps.

+
+
+

指标判读Reading

+

先看雷达图判断能力短板，再看每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

+
+
+
+
+
+
评分雷达Rating Radar完整度Completeness触发清晰Trigger clarity证据充分Evidence depth可维护性Maintainability可迁移性Portability
评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。The radar chart compares completeness, trigger clarity, evidence, maintainability, and portability.
+
+

成熟度条Maturity Bar

+

这里把每个指标压缩为状态、分数和第一条证据，先给出整体判断，再进入下方明细。This compresses each metric into status, score, and the first evidence item before the detailed cards below.

+
  1. 稳定Stable完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 稳定Stable证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability100SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 关注Watch上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
+
+
+
+
指标明细Metric Details
+
完整度Completeness100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth100
  • 已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability100
  • SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost42
  • 入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本偏高，建议压缩入口或拆分 references。Context cost is high; compress the entrypoint or split references further.
+
+
+
+
+ +
+
+
+

能力画像Capability

+

判断这个 Skill 在能力地图中的位置和复用范围。Places this skill on a capability map and clarifies reuse scope.

+
+
+
能力矩阵Capability Matrix执行确定性Execution certainty知识密度Knowledge densityExecution operationExecution operation
能力矩阵说明这个 Skill 更偏知识密集还是执行确定。The capability matrix shows whether the skill leans toward knowledge density or execution certainty.
+
+

画像摘要Profile

+
  • 能力类型：Execution operationCapability type: Execution operation
  • 成熟度：governedMaturity: governed
  • 触发强度：手动触发 + description 路由Trigger strength: Manual activation plus description-based routing
  • 复用范围：跨平台Reuse scope: Cross-platform
+
+
+
+
+ +
+
+
+

原理结构Principle

+

说明入口、参考、脚本、评估和报告如何组成一个稳定闭环。Explains how entrypoint, references, scripts, evals, and reports form a stable loop.

+
+
+
分层结构Layered Structure入口层Entrypoint layer参考层Reference layer脚本层Script layer评估层Evaluation layer报告层Report layer
分层结构展示入口、参考、脚本、评估和报告如何各司其职。The layered structure shows how entrypoint, references, scripts, evals, and reports each carry a distinct role.
+
+
+

执行流程Execution Flow

+
  1. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.
  2. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Capture job, output, exclusions, constraints, standards, and the lightest fit.
  3. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
  4. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Write `description` early, test route quality, then add only earned folders and gates.
  5. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful.
+
+
+

调用方式How To Use

+
  1. 当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  2. 当用户请求与该 Skill 的触发描述匹配时使用。Use this skill when the request matches the frontmatter description.
+
+
+
+
+
+ +
+
+
+

契约边界Contract

+

把触发、输入、输出和排除场景放在同一屏。Keeps trigger, inputs, outputs, and exclusions on the same screen.

+
+
+
+

触发描述Trigger

+

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

+

输入材料Inputs

+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。an existing skill directory that needs refactor, evaluation, packaging, or governance hardening
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。benchmark references, local constraints, desired maturity tier, and review standards
+
+
+

输出结果Outputs

+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.
  • 结构化 Skill 目录，共 8 类关键资产。Structured skill directory with 8 key asset groups.
+

不应触发Should Not Trigger

+
  • 只需要一次性回答、没有复用价值的临时请求。One-off requests that do not need reusable skill behavior.
  • 要求直接执行相邻任务，而不是沉淀或使用这个 Skill。Requests to perform an adjacent task directly rather than create or use this skill.
  • 缺少必要事实且用户不允许澄清的场景。Cases that lack required facts and do not allow clarification.
+
+
+
+
+ +
+
+
+

质量评估Quality

+

展示强项、缺口和建议，避免只给分不解释。Shows strengths, gaps, and recommendations instead of scores without explanation.

+
+
+ + + +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。Context cost needs improvement: Entrypoint is about 339 words/characters; references are about 16934.纳入下一轮修复Fix next
+
+
+

执行证据Execution Evidence

+
  • 变体运行：10Variant runs: 10
  • 模型执行：10Model executions: 10
  • 记录样本：0Recorded fixtures: 0
  • Token 估算：0Token estimates: 0
+
+
+

盲评审定Blind Adjudication

+
  • 评审进度：0 / 5Review progress: 0 / 5
  • 待评审：5Pending review: 5
  • 一致率：暂无Agreement rate: not available yet
  • 非法决策：0Invalid decisions: 0
+
+
+
+
+

评审原则Review Rule

+
  • 先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。Record the reviewer's A/B choice before opening the answer key and calculating agreement.
  • 缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.
+
+
+

运行原则Run Rule

+
  • recorded fixture 只能证明可复现样本，不等同于模型执行。A recorded fixture proves reproducible samples only; it is not model execution.
  • 只有 provider runner 返回 model metadata 时才计入 model-executed。Only provider runners that return model metadata count as model-executed.
+
+
+
+
+
+ +
+
+
+

风险治理Risk

+

提前暴露误触发、漂移、证据不足和迁移风险。Surfaces trigger, drift, evidence, and portability risks before the package grows.

+
+
+
风险热力Risk Heatmap000200300发生概率Probability影响程度Impact
风险热力图用影响程度和发生概率标出当前治理重点。The risk heatmap marks governance priorities by impact and probability.
+
+ + + +
风险Risk信号Signal应对Response
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
+
+
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks11 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks

当前没有阻塞检查。No blocked checks right now.

人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
+
+
+ +
+
+
+

包体资产Assets

+

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

+
+
+
资产分布Asset Distribution456项456 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+ + + +
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
+
+
+
+ +
+
+
+

迭代路线Roadmap

+

把下一步升级收束为少数高价值动作。Keeps next iteration moves focused and actionable.

+
+
+
迭代时间Iteration Timeline下一步 1Next 1补齐世界证据Close world-class evide...下一步 2Next 2Borrow one proven…Borrow one proven…下一步 3Next 3Harden portabilit…Harden portabilit…
迭代时间线把下一步升级收束成少数可执行动作。The iteration timeline narrows the next upgrade into a few executable moves.
+
下一步 1Next 1

补齐世界证据Close world-class evidence

世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。World-class evidence still has 4 pending item(s); public completion claims must stay blocked.

  • 补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。Close provider holdout evidence: Missing a real provider model run and token metadata.
  • 补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。Close human adjudication evidence: Blind-review pairs still need real reviewer decisions.
  • 按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。Close all 4 pending evidence item(s) in the ledger (external 3, human 1; 2 not shown here) and keep the claim guard pending.

全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。Only after the ledger accepts all external and human evidence should the public world-class claim move to review.

下一步 2Next 2

有选择地借鉴一个成熟模式Borrow one proven pattern on purpose

已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.

  • 阅读 `obra/superpowers` 中最值得借鉴的模式。Read the strongest pattern from obra/superpowers.
  • 先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。Decide whether to borrow method, structure, execution, or portability, but only one of them first.
  • 记录本轮不借鉴的内容，避免包体过重。Record what you will not borrow so the package stays light.

包体形态更清晰，也减少偶然过度设计。A cleaner package shape with less accidental over-design.

下一步 3Next 3

加固跨环境语义Harden portability semantics

这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。The skill already signals reuse across environments, so contract clarity matters early.

  • 确认激活模式、执行上下文和信任假设。Confirm activation mode, execution context, and trust assumptions.
  • 补充或复核非原生目标端的降级策略。Add or review degradation strategy for non-native targets.
  • 至少打包一次 Skill，用来验证 adapter 预期。Package the skill once to verify adapter expectations.

跨环境复用更安全，目标漂移更少。Safer cross-environment reuse with less target drift.

+
+
+
+
+ + + diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json new file mode 100644 index 00000000..a2392d37 --- /dev/null +++ b/reports/skill-interpretation.json @@ -0,0 +1,2295 @@ +{ + "skill_summary": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "display_name": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "maturity": "governed", + "updated_at": "2026-03-31", + "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", + "audience": "Skill 作者、复用团队和后续 reviewer。", + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "skill-ir/examples/yao-meta-skill.json", + "reports/compiled_targets.md", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_adjudication.md", + "reports/benchmark_reproducibility.md", + "reports/output_blind_answer_key.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_ledger.md", + "reports/review_annotations.md", + "reports/review-studio.html", + "reports/skill-interpretation.html", + "reports/skill-overview.html", + "reports/skill-interpretation.json" + ], + "flow": [ + "输入材料", + "Skill 包体", + "可复用能力" + ] + }, + "scorecard": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 100, + "reasons": [ + "已生成 20 / 20 类报告证据。", + "skill-ir.json 已存在。", + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 339 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 339 个词/字，references 约 16934 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "capability_profile": { + "archetype": "governed", + "task_family": "Execution operation", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "principle_model": { + "nodes": [ + { + "title": "意图澄清", + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "contract_boundary": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "outputs": [ + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "quality_review": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" + ], + "gaps": [ + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." + ] + }, + "prompt_quality": { + "overall_quality_score": 89.0, + "highlights": [ + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." + ] + }, + "system_model": { + "stability": { + "score": 100, + "band": "system-ready" + }, + "highlights": [ + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." + ] + } + }, + "risk_governance": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 1, + "signal": "已生成 20 / 20 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 339 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "world_class_readiness": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 19, + "source_pass_count": 11, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete", + "Reviewer metadata" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, + "package_assets": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 35 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 153 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 29 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 235 + } + ], + "file_count": 456, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 35 + }, + { + "label": "scripts", + "value": 153 + }, + { + "label": "evals", + "value": 29 + }, + { + "label": "reports", + "value": 235 + } + ] + }, + "iteration_roadmap": { + "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + } + ] + }, + "report_contract": { + "schema_version": "2.0", + "html_report": "reports/skill-interpretation.html", + "language": "zh-CN", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "layout": "kami-white-audit-v2", + "nav_labels": [ + "技能概述", + "总览指标", + "能力画像", + "原理结构", + "契约边界", + "质量评估", + "风险治理", + "包体资产", + "迭代路线" + ], + "nav_labels_en": [ + "Overview", + "Metrics", + "Profile", + "Principle", + "Contract", + "Quality", + "Risk", + "Assets", + "Roadmap" + ], + "report_kind": "skill-interpretation", + "canonical_overview_report": "reports/skill-overview.html", + "json_report": "reports/skill-interpretation.json", + "purpose": "Explain the generated skill's role, principles, usage scenarios, trigger contract, inputs, outputs, quality evidence, risks, assets, highlights, and next upgrade directions." + }, + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "display_name": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "logic_steps": [ + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." + ], + "usage_steps": [ + "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." + ], + "package_map": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 35 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 153 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 29 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 235 + } + ], + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" + ], + "scenarios": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." + ], + "trigger_contract": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "io_contract": { + "inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "outputs": [ + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", + "结构化 Skill 目录，共 8 类关键资产。" + ] + }, + "principles": [ + { + "title": "意图澄清", + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." + } + ], + "roadmap": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + } + ], + "cards": [], + "introduction": [ + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" + ], + "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "source_path": "skill-ir/examples/yao-meta-skill.json", + "target_count": 5, + "trigger_samples": 8, + "output_eval_cases": 3 + }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 5, + "pass_count": 5, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, + "output_quality": { + "case_count": 5, + "file_backed_case_count": 1, + "near_neighbor_case_count": 1, + "boundary_case_count": 1, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "gate_pass": true, + "failure_taxonomy": [], + "blind_pair_count": 5 + }, + "output_execution": { + "ok": true, + "summary": { + "case_count": 5, + "variant_run_count": 10, + "command_executed_count": 10, + "model_executed_count": 10, + "recorded_fixture_count": 0, + "timing_observed_count": 10, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, + "regression_count": 0, + "failure_count": 0, + "gate_pass": true + }, + "runner": { + "mode": "command", + "command": [ + "python3", + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" + ], + "timeout_seconds": 180.0 + }, + "failures": [] + }, + "output_blind_review": { + "summary": { + "pair_count": 5, + "answer_key_separate": true, + "with_skill_hidden_count": 5 + }, + "seed": "yao-output-eval-blind-v1", + "pair_count": 5, + "answer_key_separate": true + }, + "output_review_kit": { + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "failures": [] + }, + "output_review_adjudication": { + "ok": true, + "summary": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "agreement_count": 0, + "disagreement_count": 0, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, + "agreement_rate": null, + "needs_review": true, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "failure_count": 0, + "reviewer_checklist_count": 5, + "reviewer_checklist_pending_count": 5, + "reviewer_checklist_invalid_count": 0, + "reviewer_checklist_ready_count": 0 + }, + "reviewer": "", + "reviewed_at": "", + "failures": [] + }, + "benchmark_reproducibility": { + "ok": true, + "summary": { + "reproducibility_ready": true, + "release_lock_ready": true, + "methodology_complete": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "evidence_bundle_sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0", + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 23, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 10, + "token_observed_count": 10, + "human_review_complete": false, + "provider_evidence_complete": true, + "world_class_ready": false, + "world_class_open_gap_count": 3, + "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3, + "working_tree_dirty": true, + "changed_file_count": 29, + "source_tree_dirty": false, + "source_changed_file_count": 0, + "generated_tree_dirty": true, + "generated_changed_file_count": 29 + }, + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "missing_artifacts": [], + "limitations": [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims." + ] + }, + "runtime_conformance": { + "target_count": 5, + "pass_count": 5, + "fail_count": 0 + }, + "runtime_permissions": { + "ok": true, + "summary": { + "target_count": 4, + "pass_count": 4, + "fail_count": 0, + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "required_capability_count": 3, + "failure_count": 0, + "installer_enforcement_source_status": "present", + "installer_enforcement_target_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "installer_permission_capability_count": 3, + "world_class_native_evidence_ready": false, + "installer_enforcement_ready": true + }, + "expected_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "targets": [ + { + "target": "openai", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "claude", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "generic", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "vscode", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + } + ], + "failures": [] + }, + "trust_security": { + "scanned_files": 245, + "script_count": 153, + "internal_module_count": 67, + "secret_findings": 0, + "dependency_files": [ + "requirements-ci.txt" + ], + "network_script_count": 3, + "network_policy_covered_count": 3, + "network_policy_missing_count": 0, + "file_write_script_count": 73, + "permission_required_count": 3, + "permission_approved_count": 3, + "permission_missing_count": 0, + "permission_invalid_count": 0, + "permission_expired_count": 0, + "help_smoke_checked_count": 86, + "help_smoke_failed_count": 0, + "interactive_script_count": 0, + "package_hash_scope": "source-contract-without-generated-reports", + "package_hash_file_count": 245, + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" + }, + "skill_atlas": { + "skill_count": 12, + "actionable_skill_count": 1, + "route_collision_count": 5, + "actionable_route_collision_count": 0, + "owner_gap_count": 9, + "actionable_owner_gap_count": 0, + "stale_count": 10, + "actionable_stale_count": 0, + "shared_resource_count": 0, + "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, + "non_actionable_issue_count": 24 + }, + "registry_distribution": { + "ok": true, + "package": { + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible", + "vscode" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" + }, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": true, + "summary": { + "target_count": 4, + "adapter_count": 4, + "archive_present": true, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "archive_entry_count": 689, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": true, + "summary": { + "archive_present": true, + "archive_entry_count": 696, + "archive_extracted": true, + "entrypoint_loaded": true, + "manifest_loaded": true, + "interface_loaded": true, + "adapter_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, + "install_root_is_temp": false, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": true, + "summary": { + "declared_bump": "minor", + "recommended_bump": "minor", + "breaking_change_count": 0, + "added_target_count": 2, + "metadata_change_count": 1, + "checksum_change_count": 2, + "failure_count": 0, + "warning_count": 0 + }, + "upgrade_diff": { + "name_changed": false, + "added_targets": [ + "agent-skills-compatible", + "vscode" + ], + "removed_targets": [], + "compatibility_changes": [ + { + "target": "agent-skills", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "agent-skills-compatible", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "vscode", + "from": "missing", + "to": "pass", + "direction": "improved" + } + ], + "metadata_changes": [ + { + "field": "description", + "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." + } + ], + "checksum_changes": [ + { + "field": "archive_sha256", + "from": "", + "to": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" + }, + { + "field": "package_sha256", + "from": "0000000000000000000000000000000000000000000000000000000000000000", + "to": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" + } + ] + }, + "release_notes": [ + "Recommended version bump: minor.", + "Added targets: agent-skills-compatible, vscode.", + "Package or archive checksum changed; reviewers should verify package artifacts before release.", + "Package verification evidence: reports/package_verification.md." + ], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "low", + "event_types": { + "review_event": 1 + }, + "failure_types": {}, + "source_types": { + "manual": 1 + }, + "command_counts": {} + }, + "next_iteration_candidates": [], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [], + "waiver_candidate_count": 2, + "waiverable_open_count": 1, + "non_waivable_count": 1 + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "review_studio_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "known_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "waiverable_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "non_waivable_gate_keys": [ + "review-waivers", + "world-class-evidence" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] + }, + "world_class_evidence_plan": { + "ok": true, + "summary": { + "audit_decision": "continue-iteration", + "world_class_ready": false, + "audit_world_class_ready": false, + "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, + "task_count": 4, + "human_task_count": 1, + "external_task_count": 3, + "review_task_count": 0, + "decision": "collect-external-evidence" + }, + "tasks": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_audit": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "open_gap_count": 3 + } + }, + "world_class_evidence_ledger": { + "ok": true, + "summary": { + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "source_accepted": false, + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4, + "evidence_requirement_count": 4 + } + }, + "synthesis_highlights": [ + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "artifact_design": { + "design_system": "metric editorial", + "highlights": [ + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." + ] + }, + "prompt_quality": { + "overall_quality_score": 89.0, + "highlights": [ + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." + ] + }, + "system_model": { + "stability": { + "score": 100, + "band": "system-ready" + }, + "highlights": [ + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." + ] + }, + "metadata": { + "canonical_format": "agent-skills", + "targets": [ + "openai", + "claude", + "generic", + "vscode" + ], + "maturity_tier": "governed", + "skill_archetype": "governed", + "updated_at": "2026-03-31" + }, + "interpretation_contract": { + "schema_version": "2.0", + "source_model": "skill-overview-v2", + "source_model_reused": true, + "overview_report": "reports/skill-overview.html", + "html_report": "reports/skill-interpretation.html", + "json_report": "reports/skill-interpretation.json", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "includes": [ + "skill role", + "principles", + "usage scenarios", + "trigger contract", + "inputs and outputs", + "quality evidence", + "risk governance", + "package assets", + "highlights", + "upgrade roadmap" + ] + } +} \ No newline at end of file diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index ae003a98..3223a857 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -8,84 +8,123 @@ Scope: Yao Meta Skill against the user-provided Skill OS 2.0 upgrade plan. Yao Meta Skill is no longer only a Meta Skill factory. The current working tree now has the first verifiable Skill OS foundation: - Skill IR v0 for platform-neutral contract capture. -- Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, and reviewer adjudication reporting. +- Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, reviewer decision import, and adjudication reporting. - Runtime Conformance v0 for target-consumption checks. - Trust/Security v0 for secret, dependency, script, bounded network host policy, execution-level CLI help smoke checks, trust metadata, and stable source-contract integrity checks. -- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities. +- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, aggregate drift signals, and no-route opportunities. - Bilingual Skill Overview v2 that includes these evidence surfaces. +- Skill Overview Asset Split v0 so long report CSS and JavaScript live in `assets/skill-overview.css` and `assets/skill-overview.js`, while the Python layout module keeps the generated report single-file by inlining those assets at render time. +- Review Studio Source Anchor Excerpts v0 so every non-pass action source ref carries a matched pattern and short line excerpt, letting reviewers understand the anchor before opening the full artifact. +- Review Studio World-Class Collection Contract v0 so each pending world-class evidence action card shows provenance requirements, success checks, evidence artifacts, and privacy boundaries alongside commands and blocked checks. +- Review Studio Asset Split v0 so long Review Studio CSS lives in `assets/review-studio.css`, while the Python layout module still inlines it into the generated static HTML report. +- Skill OS 2.0 Audit v0 for requirement-by-requirement evidence mapping, with explicit separation between local evidence, human-required review, and external-required provider/client integrations. +- World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts. +- World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. +- World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. +- World-Class Human Evidence Guard v0 so human-adjudication decision artifacts recursively reject raw content, credential, secret, token, and answer-key fields before blind A/B review evidence can be accepted. +- Output Review Privacy Guard v0 so the blind-review importer and world-class human evidence validator share one recursive blocked-field contract for raw content, credential, secret, token, and answer-key leakage. +- World-Class Submission Privacy Guard v0 so real evidence submission packets reuse the same blocked-field contract and reject nested answer-key leakage before ledger review. +- World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance. +- World-Class Native Permission Evidence Guard v0 so native-permission submissions must reconcile runtime probe summary counts with target rows, require at least one native-enforced target row, and keep installer permission checks failure-free. +- World-Class Native Telemetry Evidence Guard v0 so native-client-telemetry submissions must reconcile adoption summary counts with external metadata event rows and keep hook recipes metadata-only without claiming native auto-capture. +- World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. +- World-Class Repair Checklist v0 so the submission kit turns every draft, artifact, and source-check blocker into machine-readable repair rows without counting repair guidance as completion evidence. +- World-Class Preflight Repair Rows v0 so the collection preflight exposes required precheck and source-check blockers as machine-readable operator actions before evidence submission. +- World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence. +- World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission. +- Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence. - Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. - Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers. -- Target Compiler v0 so Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts before packaging, including target-specific permission contracts and native behavior contracts. -- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, compiler contract, parity checks, and IR provenance. +- Target Compiler v0 so Skill IR compiles into OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot target contracts before packaging, including target-specific permission contracts and native behavior contracts. +- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, compiler contract, parity checks, IR provenance, and VS Code / Copilot install-scope notes where applicable. - Registry & Distribution v0 for package metadata, checksum, owner, license, compatibility matrix, and audit reporting. - Package Verification v0 for generated manifest, target adapters, zip archive safety, archive checksum, and registry parity. -- Install Simulation v0 for local extraction, entrypoint loading, interface loading, report presence, and adapter readability. +- Install Simulation v0 for local extraction, entrypoint loading, interface loading, report presence, adapter readability, and installer-level permission approval/enforcement checks. - Upgrade Check v0 for registry package diffs, semver bump recommendations, breaking-change notes, and release evidence. -- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, and iteration candidate generation. +- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, iteration candidate generation, and Skill Atlas drift-signal input. +- CLI Telemetry Capture v0 for opt-in `yao.py` command-run metadata that records command name, source, outcome, failure class, and timestamp without command arguments or raw user/model content. +- External Telemetry Import v0 for whole-file validated metadata-only JSONL ingestion from non-CLI clients, with dry-run support and all-or-nothing rejection on raw content fields or schema violations. +- Telemetry Emit Hook v0 for Browser/Chrome/IDE/wrapper integrations to append one validated metadata-only external event into a local spool before import, with dry-run validation and raw-content blocking. +- Telemetry Hook Recipes v0 for Browser, Chrome, VS Code, CLI wrapper, and provider-adapter recipe generation with dry-run commands, metadata-only privacy checks, and explicit non-claiming of native host integration. +- Telemetry Native Host v0 for Browser/Chrome Native Messaging length-prefixed stdio, metadata-only event validation, local launcher generation, and Chrome host manifest generation. - Review Waivers v0 for human warning acceptance with reviewer, reason, scope, expiry, and blocker-safe policy. - Governed Permission Gates v0 for reviewer-approved network, file-write, and subprocess capabilities with scope, reason, expiry, evidence, and target-enforcement mapping. - Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks. +- Local Install Sync Preflight v0 so source-to-local and source-to-active install syncs rebuild the package first, run install simulation, enforce installer permission coverage, and refuse to copy files before any destructive sync when the package is not install-ready. - Atlas Scope Policy v0 so examples, evolution snapshots, embedded generated skills, and validator fixtures remain visible in the full portfolio report without polluting release-actionable gates. - Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision. -- Review Studio now reaches a clean `ready` decision for the root Meta Skill: `13 / 13` gates pass, world-class score `100`, no blockers, no warnings, and no review actions required. +- Review Studio now avoids over-claiming release readiness when blind A/B adjudication, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `91`, no blockers, `3` warnings, and explicit actions for Output Lab reviewer adjudication, waiver handling, and world-class evidence completion. +- Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. +- Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. +- Output Review Import v0 now accepts reviewer JSON, JSONL, or CSV decision sources, recursively rejects raw prompt/output/transcript/message, credential, secret, token, and answer-key fields case-insensitively, writes canonical `reports/output_review_decisions.json`, and can run adjudication immediately without opening the answer key before valid decisions exist. +- Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. +- Weekly SkillOps Curator v0 so daily SkillOps opportunities, Skill Atlas portfolio signals, release-lock state, and world-class evidence gaps roll up into a proposal-only weekly maintenance queue without source-file writes. -This is still not the final world-class state. Target-native behavior contracts are now explicit and local output-eval command execution is wired, but deeper provider-native execution transforms, real client telemetry capture, installer integration, provider-backed model-executed output eval, real human adjudication decisions, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, provider-backed output-eval execution is wired through DeepSeek evidence, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, external clients can now emit one validated metadata-only event into a local spool, hook recipes now make Browser/Chrome/VS Code/wrapper/provider-adapter integration commands auditable, a Browser/Chrome Native Messaging host can receive length-prefixed metadata events and generate a local launcher plus manifest, validated external JSONL imports can bring those non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. World-class intake now rejects non-canonical submission filenames and nested raw content or credential-like fields before ledger review. The preflight and submission kit now give operators repair rows, matrix state, artifact readiness, source blockers, and next actions for each pending world-class evidence key. Review Studio keeps pending human adjudication, waiver handling, and pending world-class evidence visible as warnings instead of treating them as a clean pass. Accepted external ledger evidence, real human adjudication decisions, installed platform-native client telemetry implementations, provider-native installer integration, and native runtime permission enforcement remain open. ## Coverage Matrix | 2.0 Area | Current Evidence | Status | | --- | --- | --- | | Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed | -| Target Compiler | `scripts/compile_skill.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed | -| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_adjudication.md` | v0 landed | +| Target Compiler | `scripts/compile_skill.py`, `scripts/compile_skill_targets.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed | +| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/import_output_review_decisions.py`, `scripts/output_review_privacy.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed | | Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | +| Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed | +| Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | +| World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | +| World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | +| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_human_evidence.py`, `scripts/output_review_privacy.py`, `scripts/world_class_provider_evidence.py`, `scripts/world_class_native_permission_evidence.py`, `scripts/world_class_native_telemetry_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, recursive human decision privacy and answer-key validation, provider run-row validation, native permission target-row validation, native telemetry event-row validation, nested raw-field rejection, and real-submission answer-key leakage rejection | v0 landed | +| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_repair_checklist.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, machine-readable repair rows, and separated Markdown/HTML rendering | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | | Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed | | Review Studio Source Refs | `scripts/render_review_studio.py`, `tests/verify_review_studio.py`, `references/review-studio-method.md` | v0 landed | | Review Annotations | `scripts/render_review_annotations.py`, `reports/review_annotations.md`, `tests/verify_review_annotations.py`, Review Studio annotation panel and blocker decision hook | v0 landed | -| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed | +| Weekly SkillOps Curator | `scripts/render_weekly_curator_report.py`, `reports/skillops/weekly/2026-W25.md`, `tests/verify_weekly_curator.py`, proposal-only maintenance queue with no source writes | v0 landed | +| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `skill_atlas/drift_signals.json`, `reports/skill_atlas.html` | v0 landed | | Registry & Distribution | `registry/*.schema.json`, `scripts/registry_audit.py`, `reports/registry_audit.md`, `registry/packages/yao-meta-skill.json` | v0 landed | | Package Verification | `scripts/verify_package.py`, `reports/package_verification.md`, `tests/verify_package_verification.py` | v0 landed | -| Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` | v0 landed | +| Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` with installer permission coverage checks | v0 landed | +| Local Install Sync Preflight | `scripts/sync_local_install.py`, `tests/verify_local_install_sync.py`, `Makefile` package-check prerequisites | v0 landed | | Upgrade Check | `scripts/upgrade_check.py`, `reports/upgrade_check.md`, `tests/verify_upgrade_check.py` | v0 landed | -| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `references/telemetry-drift-method.md` | v0 landed | +| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `scripts/yao_cli_telemetry.py`, `scripts/emit_telemetry_event.py`, `scripts/render_telemetry_hook_recipes.py`, `scripts/telemetry_native_host.py`, `scripts/import_telemetry_events.py`, `reports/adoption_drift_report.md`, `reports/telemetry_hook_recipes.md`, `tests/verify_adoption_drift.py`, `tests/verify_telemetry_emit.py`, `tests/verify_telemetry_hooks.py`, `tests/verify_telemetry_native_host.py`, `tests/verify_telemetry_import.py`, `tests/verify_yao_cli.py`, `references/telemetry-drift-method.md` | v0 landed | | Review Waivers | `scripts/render_review_waivers.py`, `reports/review_waivers.md`, `tests/verify_review_waivers.py`, `references/review-waiver-method.md` | v0 landed | | Governed Permission Gates | `security/permission_policy.json`, `scripts/trust_check.py`, `scripts/render_review_studio.py`, `tests/verify_trust_check.py`, `tests/verify_review_studio.py` | v0 landed | | Runtime Permission Probes | `scripts/probe_runtime_permissions.py`, `reports/runtime_permission_probes.md`, `tests/verify_runtime_permission_probes.py`, `tests/verify_review_studio.py` | v0 landed | | Atlas Scope Policy | `skill_atlas/policy.json`, `scripts/build_skill_atlas.py`, `tests/verify_skill_atlas.py` | v0 landed | | Compiler from IR | Packager consumes compiled target contracts for compiler provenance, generated files, adapter modes, permissions, preserved semantics, warnings, and unsupported features | v0 landed | -| Target Native Contracts | `scripts/compile_skill.py`, `scripts/cross_packager.py`, `reports/compiled_targets.md`, adapter snapshots, and generated target outputs carry native surface, activation policy, resource/script strategy, permission enforcement, install scope, review artifacts, fallback behavior, and unsupported native features | v0 landed | +| Target Native Contracts | `scripts/compile_skill.py`, `scripts/compile_skill_targets.py`, `scripts/cross_packager.py`, `reports/compiled_targets.md`, adapter snapshots, and generated target outputs carry native surface, activation policy, resource/script strategy, permission enforcement, install scope, review artifacts, fallback behavior, and unsupported native features | v0 landed | ## Top Findings ### 1. Target-native behavior contracts are landed, but provider execution is still shallow -`scripts/compile_skill.py` now reads Skill IR and emits target-specific contracts for OpenAI, Claude, generic, and Agent Skills compatible outputs. The packager embeds `compiler`, `compiled_contract`, `permission_contract`, `target_permission_contract`, `target_native_contract`, `target_transform`, warnings, and unsupported-feature notes in each adapter. The native contract makes the target surface, activation policy, resource strategy, script strategy, permission enforcement mode, install scope, review artifacts, fallback behavior, and unsupported native features reviewable instead of implicit. +`scripts/compile_skill.py` now reads Skill IR and emits target-specific contracts for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, with target platform models and contract builders isolated in `scripts/compile_skill_targets.py`. The packager embeds `compiler`, `compiled_contract`, `permission_contract`, `target_permission_contract`, `target_native_contract`, `target_transform`, warnings, and unsupported-feature notes in each adapter. The native contract makes the target surface, activation policy, resource strategy, script strategy, permission enforcement mode, install scope, review artifacts, fallback behavior, and unsupported native features reviewable instead of implicit. Next move: deepen provider-native execution transforms so OpenAI, Claude, VS Code/Copilot, and generic packages can express and verify behavior through real client or installer capabilities, not only auditable contracts and fallback notes. -### 2. Output eval now has command-runner evidence, but provider model execution is still open +### 2. Output eval now has provider-backed model evidence, but human review is still open -The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. Each run emits assertion grading, execution provenance, a blind A/B review pack that hides baseline vs with-skill labels, a separate answer key, and an adjudication report from reviewer choices. The current root execution report now runs all `10` variants through `scripts/local_output_eval_runner.py`, so timing and command-runner behavior are observed instead of relying only on static recorded fixtures. It still intentionally reports `0` provider-backed model-executed runs and `10` estimated token counts, because the local runner verifies the harness contract without pretending to be a model provider. The current root adjudication report intentionally records `0 / 5` human judgments pending instead of pretending human review happened. The next gap is provider-backed model-executed holdout runs with real provider timing/tokens and real reviewer decision records. +The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. Each run emits assertion grading, execution provenance, a blind A/B review pack that hides baseline vs with-skill labels, a separate answer key, an importer for reviewer JSON/JSONL/CSV decision sources, and an adjudication report from reviewer choices. The current root execution report runs all `10` variants through `scripts/provider_output_eval_runner.py` against DeepSeek `deepseek-v4-flash`, so model execution, timing, and token usage are observed rather than estimated. The provider source checks now pass with `10` model-executed runs, `10` observed timing rows, `10` observed token rows, `0` estimated token rows, and no regressions, while the world-class ledger still requires an independent reviewer-approved submission before it counts as accepted evidence. The current root adjudication report intentionally records `0 / 5` human judgments pending instead of pretending human review happened. -Next move: add provider-backed model execution for holdout cases and one real multi-file fixture with actual reviewer decisions. +Next move: complete independent provider evidence ledger review, then record actual blind A/B reviewer decisions. ### 3. Review Studio is unified and now has reviewer actions plus annotations -The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. After adding root intent context, Atlas scope policy, metadata-only telemetry evidence, runtime permission probe evidence, action source refs, and an empty annotation ledger, the current root report is clean: decision `ready`, score `100`, `13` gates, `0` gate blockers, `0` gate warnings, `0` actions, and `0` open annotation blockers. +The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, world-class evidence, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. The current root report is intentionally not a clean pass: decision `review`, score `91`, `16` gates, `0` blockers, `3` warnings, `3` actions, and `0` open annotation blockers. The warnings are useful evidence: the automated Output Eval Lab has provider-backed execution but blind A/B adjudication still has `5` pending reviewer decisions, warning waivers still need reviewer handling, and the world-class ledger still has `4` pending external or human evidence entries, so the package should not claim fully reviewed or world-class status. -Next move: add richer source-line anchors inside generated reports and record real reviewer annotations during the next human review pass. +Next move: add richer source-line anchors inside generated reports, record real reviewer annotations during the next human review pass, and close the provider ledger-review, native-permission, real-client telemetry, and human review evidence entries in the world-class ledger. -### 4. Multi-skill operation now has v0 coverage, but needs real telemetry +### 4. Multi-skill operation now links Atlas with drift, but needs platform-native capture -The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, and feed next iteration candidates into Review Studio. It still needs real client-side capture instead of manual CLI event recording. +The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, aggregate drift signals, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, feed next iteration candidates into Review Studio, and publish aggregate drift input for Atlas. `yao.py` now adds opt-in automatic CLI run capture through `YAO_CLI_TELEMETRY=1` or `--record-cli-telemetry`, recording only `source=yao_cli`, normalized subcommand name, outcome, failure class, and timestamp. `telemetry-emit` gives Browser/Chrome/IDE/wrapper integrations a stable local hook for appending one normalized external event into `.yao/telemetry_spool/external_events.jsonl`, with dry-run validation and no raw content fields. `telemetry-hooks` renders Browser, Chrome, VS Code, CLI wrapper, and provider-adapter recipes with dry-run commands, import commands, trigger points, and explicit `native_auto_capture=false` caveats. `telemetry_native_host.py` now provides a tested Browser/Chrome Native Messaging host: it reads length-prefixed stdio JSON, rejects raw content fields, writes metadata-only events, and generates an operator-installable launcher plus Chrome host manifest. `telemetry-import` accepts already-sanitized external client JSONL or the emitted spool, validates the whole file before appending, supports dry-run, defaults `source=external`, and rejects any raw content or unknown metadata fields without mutating the local event stream. It still needs an installed real extension or provider client that sends production events to the native host. -Next move: connect client activations and failure history so Atlas can rank stale or conflicting skills by real usage impact. +Next move: install a real Browser/Chrome extension or provider client against the native host and record production metadata events, then let Atlas rank stale, drifting, or conflicting skills by real usage impact. ### 5. Trust report is structural, not full security review -Trust v0 blocks obvious secrets and remote inline execution, and now makes checksum scope explicit: `package_sha256` is a stable source-contract digest that excludes generated reports, packages, and raw local telemetry; archive integrity is carried by Package Verification and Registry as `archive_sha256`. Registry v0 carries both checksums into distributable metadata. Package Verification v0 now checks generated manifests, target adapters, archive path safety, required source entries, and registry parity. Install Simulation v0 now extracts the generated archive into a temporary local skill root and verifies entrypoint, manifest, interface, reports, and adapters. Upgrade Check v0 now compares package baselines, recommends semver bumps, and blocks breaking changes without an adequate declared version. Trust Report now distinguishes CLI scripts from declared internal modules, so help-surface warnings focus on executable entrypoints. Network-capable scripts are now covered by `security/network_policy.json`, with `allowed_hosts` checked against HTTPS URL literals. Trust Report also executes `python3 scripts/name.py --help` for CLI scripts with `argparse`, recording pass/fail smoke evidence without executing scripts that lack a help surface. Target adapters now carry `permission_contract` and `target_permission_contract` fields, including network, file-write, subprocess, and interactive capability counts. Governed permission gates now require reviewer-approved scope, reason, expiry, evidence, and target-enforcement mapping in `security/permission_policy.json`. Runtime Permission Probes now verify that packaged adapters expose the target permission contract and explicitly report native-enforcement limits. The remaining trust gap is native runtime permission enforcement in real clients/installers. +Trust v0 blocks obvious secrets and remote inline execution, and now makes checksum scope explicit: `package_sha256` is a stable source-contract digest that excludes generated reports, packages, and raw local telemetry; archive integrity is carried by Package Verification and Registry as `archive_sha256`. Registry v0 carries both checksums into distributable metadata. Package Verification v0 now checks generated manifests, target adapters, archive path safety, required source entries, and registry parity. Install Simulation v0 now extracts the generated archive into a temporary local skill root, verifies entrypoint, manifest, interface, reports, and adapters, then checks every packaged target/capability pair against the installed `security/permission_policy.json` approval and `target_enforcement` mapping. Local Install Sync Preflight now runs the same install simulation before syncing source into the disabled or active local install path, so a broken package or permission enforcement gap fails before stale files are removed or new files are copied. Upgrade Check v0 now compares package baselines, recommends semver bumps, and blocks breaking changes without an adequate declared version. Trust Report now distinguishes CLI scripts from declared internal modules, so help-surface warnings focus on executable entrypoints. Network-capable scripts are now covered by `security/network_policy.json`, with `allowed_hosts` checked against HTTPS URL literals. Trust Report also executes `python3 scripts/name.py --help` for CLI scripts with `argparse`, recording pass/fail smoke evidence without executing scripts that lack a help surface. Target adapters now carry `permission_contract` and `target_permission_contract` fields, including network, file-write, subprocess, and interactive capability counts. Governed permission gates now require reviewer-approved scope, reason, expiry, evidence, and target-enforcement mapping in `security/permission_policy.json`. Runtime Permission Probes now verify that packaged adapters expose the target permission contract and explicitly report native-enforcement limits. The remaining trust gap is native runtime permission enforcement in real clients/installers. Next move: add real client or installer permission enforcement integration. @@ -93,29 +132,34 @@ Next move: add real client or installer permission enforcement integration. | Gate | Current Result | | --- | --- | -| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs, `10` estimated token counts, `5` blind A/B review pairs, and `0 / 5` reviewer decisions pending | +| Output Eval | `5` cases, DeepSeek provider-backed execution for `10 / 10` variants, with-skill pass rate `43.67`, baseline pass rate `23.67`, delta `20.0`, with file-backed, near-neighbor, boundary coverage, `10` command execution runs, `0` recorded fixture runs, `10` provider model-executed runs in root release evidence, `10` observed token rows, `0` estimated token rows, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | -| Target Compiler | compiled target contracts generated for OpenAI, Claude, generic, and Agent Skills compatible outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `3` declared internal modules, `2 / 2` network-capable scripts covered by bounded host policy, `59 / 59` CLI help smoke checks passing across `62` scripts, source-contract hash scope explicit | +| Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | +| Trust | `0` secret findings, `1` pinned dependency file, `67` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `153` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | -| Runtime Permission Probes | `3 / 3` target adapters probed, `0` native-enforcement adapters, `3` explicit metadata fallbacks, `3` residual risks retained for reviewer visibility | -| Skill Atlas | `12` scanned skills, `1` actionable root skill, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `24` scoped non-actionable issue signals retained for visibility | +| Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | +| Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `3 / 3` target adapters present, archive verified, `483` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `483` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `3` adapters readable, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `689` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `689` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | -| Adoption Drift | `1` metadata-only activation event, adoption `100`, risk band `low`; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | -| Review Waivers | ledger generated; current release has no warning gates to waive, blockers remain non-waivable in v0 | +| Adoption Drift | `1` metadata-only review event, `1` adoption sample, adoption `100`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | +| Review Waivers | ledger generated; current release has `2` warning gates that still need reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0, and world-class evidence gaps must be closed with accepted evidence rather than waived as completed | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | -| Review Studio | decision `ready`, world-class score `100`, `13` gates, `0` blockers, `0` warnings, `0` review actions, `0` open annotation blockers | -| IR-first Packaging | `openai`, `claude`, and `generic` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, and semantic parity checks | -| Context Budget | initial load `987/1000`, under the production budget | -| CI | `make ci-test` target count is now `56` after adding Review Annotations | +| Review Studio | decision `review`, world-class score `91`, `16` gates, `0` blockers, `3` warnings, `3` review actions, `0` open annotation blockers | +| Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps | +| World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts | +| World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion | +| Benchmark Reproducibility | local reproducibility ready with `25` required artifacts, `0` missing artifacts, `23` reproduction commands, and `3` disclosed failure cases; human review, native permission enforcement, native telemetry, and provider ledger approval remain explicit limitations | +| IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable | +| Context Budget | initial load `998/1000`; deferred `512782/120000`; resource governance governed | +| CI | `make ci-test` target count is `82` after the dedicated CLI world-class verifier split | ## Next Highest-Leverage Moves -1. Deepen target-native behavior contracts into provider-native execution and installer integrations. -2. Add native client or installer enforcement for approved high-permission capabilities. -3. Expand Output Eval Lab from local command-runner evidence to provider-backed model holdout and real reviewer-adjudicated cases. +1. Deepen target-native behavior contracts into provider-native execution and provider-native installer integrations. +2. Add native client or installer runtime enforcement for approved high-permission capabilities. +3. Complete independent provider ledger review, then record the current blind A/B decisions before claiming fully ready status. 4. Add real reviewer annotation records during the next human review pass. -5. Connect Skill Atlas with real client telemetry and drift history. +5. Install a real Browser/Chrome extension or provider client against `telemetry_native_host.py` and record production metadata events. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 6e3e2091..7448c1c8 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -307,8 +307,8 @@ } .section-head { display: grid; - grid-template-columns: minmax(0, 246px) minmax(0, 1fr); - gap: 36px; + grid-template-columns: minmax(0, 1fr); + gap: 22px; align-items: start; } h2 { @@ -330,12 +330,15 @@ line-height: 1.3; margin-bottom: 10px; } - .section-head p { + .section-head > div:first-child p { margin: 12px 0 0; color: var(--muted); - max-width: 43ch; + max-width: 68ch; text-wrap: pretty; } + .section-body { + margin-top: 24px; + } .two-col, .metric-grid, .chart-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); @@ -345,20 +348,28 @@ .quality-panels { margin-top: 16px; } - .metrics-stack { + .metrics-flow { display: grid; - gap: 18px; + gap: 30px; } - .metrics-lead { + .metrics-primary { display: grid; - grid-template-columns: minmax(420px, 1.15fr) minmax(300px, 0.85fr); - gap: 18px; - align-items: stretch; + grid-template-columns: minmax(360px, 0.92fr) minmax(0, 1.08fr); + gap: 22px; + align-items: start; + } + .metrics-primary .chart-figure { + min-height: 0; + display: grid; + align-content: center; + } + .metrics-primary .chart-figure svg { + max-height: 520px; } .metrics-note { display: grid; align-content: start; - gap: 16px; + gap: 14px; } .metrics-note p { margin: 0; @@ -395,6 +406,22 @@ grid-column: 2 / 4; color: var(--muted); line-height: 1.5; + overflow-wrap: break-word; + } + .metric-detail-section { + display: grid; + gap: 14px; + } + .detail-section-kicker { + display: inline-flex; + width: fit-content; + min-height: 28px; + align-items: center; + padding: 3px 8px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 12px; } .metric-status { display: inline-flex; @@ -406,9 +433,9 @@ color: var(--brand); font-size: 12px; } - .metric-grid { - grid-template-columns: repeat(2, minmax(0, 1fr)); - margin-top: 2px; + .metric-detail-grid { + grid-template-columns: repeat(2, minmax(min(100%, 420px), 1fr)); + align-items: stretch; } .list, .compact-list, .step-list { margin: 0; @@ -419,8 +446,13 @@ .list li::marker, .compact-list li::marker, .step-list li::marker { color: var(--brand); } .compact-list { gap: 6px; - font-size: 13px; + font-size: 13.5px; color: var(--muted); + line-height: 1.55; + } + .compact-list li, .list li, .step-list li { + min-width: 0; + overflow-wrap: break-word; } .panel, .metric-card, .roadmap-item { background: #ffffff; @@ -428,21 +460,23 @@ border-radius: 8px; padding: 22px; box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04); + min-width: 0; } .metric-card { display: grid; - grid-template-columns: 92px minmax(0, 1fr); + grid-template-columns: minmax(112px, 0.28fr) minmax(0, 1fr); gap: 18px; align-content: start; min-height: 0; } .metric-card-head { display: grid; + gap: 10px; align-content: start; } .metric-card strong { display: block; - margin: 8px 0 10px; + margin: 0; color: var(--brand); font-family: var(--mono); font-size: 2rem; @@ -453,10 +487,20 @@ .metric-label { color: var(--muted); font-size: 13px; + line-height: 1.45; + overflow-wrap: break-word; } .metric-card-body { min-width: 0; } + .metric-card-body .compact-list { + gap: 7px; + max-width: 76ch; + } + .metric-card-body .compact-list li { + overflow-wrap: break-word; + word-break: normal; + } .chart-figure { margin: 0; padding: 18px; @@ -512,6 +556,120 @@ font-size: 12px; } td:first-child, th:first-child { width: 96px; } + .world-readiness { + margin-top: 16px; + display: grid; + gap: 16px; + } + .section-head > .world-readiness { grid-column: 1; } + .world-readiness-head { + display: grid; + grid-template-columns: minmax(0, 1fr) auto; + gap: 16px; + align-items: start; + } + .world-readiness-head p { + margin-top: 8px; + max-width: 72ch; + } + .world-status { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 4px 10px; + border-radius: 6px; + background: var(--brand-soft); + color: var(--brand); + font-size: 13px; + white-space: nowrap; + } + .evidence-kpis { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 10px; + } + .evidence-kpi { + padding: 14px; + border: 1px solid var(--line-soft); + border-radius: 8px; + background: var(--wash); + min-width: 0; + } + .evidence-kpi span { + display: block; + color: var(--muted); + font-size: 12px; + } + .evidence-kpi strong { + display: block; + margin-top: 4px; + color: var(--brand); + font-family: var(--mono); + font-size: 1.5rem; + font-weight: 500; + line-height: 1; + font-variant-numeric: tabular-nums; + } + .evidence-kpi small { + display: block; + margin-top: 8px; + color: var(--muted); + line-height: 1.45; + overflow-wrap: break-word; + } + .evidence-list { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 10px; + } + .evidence-item { + padding: 14px; + border: 1px solid var(--line-soft); + border-radius: 8px; + background: #ffffff; + min-width: 0; + } + .evidence-item span { + color: var(--brand); + font-family: var(--mono); + font-size: 11px; + text-transform: uppercase; + } + .evidence-item h4 { + margin: 4px 0 8px; + color: var(--ink); + font-size: 1rem; + font-weight: 500; + line-height: 1.3; + } + .evidence-item h5 { + margin: 12px 0 6px; + color: var(--brand); + font-size: 12px; + font-weight: 500; + } + .evidence-item p { + margin: 0; + max-width: none; + color: var(--muted); + overflow-wrap: break-word; + } + .blocked-checks { + margin: 0; + padding-left: 1.05em; + color: var(--muted); + font-size: 13px; + line-height: 1.45; + } + .blocked-checks li { + overflow-wrap: break-word; + } + .blocked-checks li::marker { + color: var(--brand); + } + .blocked-checks-empty { + font-size: 13px; + } .roadmap { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); @@ -542,10 +700,12 @@ .report-mark { display: none; } .report-nav a { min-width: 72px; } .hero-grid, .section-head { grid-template-columns: 1fr; } + .section-head > .world-readiness { grid-column: auto; } .hero-grid { gap: 24px; } .score-strip { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .metrics-lead, .two-col, .metric-grid, .chart-grid, .roadmap { grid-template-columns: 1fr; } - .metric-card { grid-template-columns: 86px minmax(0, 1fr); } + .metrics-primary, .metric-detail-grid, .two-col, .metric-grid, .chart-grid, .roadmap { grid-template-columns: 1fr; } + .evidence-list { grid-template-columns: 1fr; } + .metric-card { grid-template-columns: 1fr; } h1 { font-size: 2.75rem; } } @media (max-width: 540px) { @@ -569,6 +729,7 @@ .score-strip { grid-template-columns: 1fr; } section { margin-top: 34px; padding-top: 34px; } .panel, .metric-card, .roadmap-item, .chart-figure, .hero-card { padding: 18px; } + .world-readiness-head, .evidence-kpis { grid-template-columns: 1fr; } .metric-card { grid-template-columns: 1fr; gap: 10px; } .metric-card strong { margin-bottom: 2px; } .metric-summary-list li { grid-template-columns: auto minmax(0, 1fr) auto; } @@ -600,16 +761,16 @@

YAO Skill 生成审计报告YAO Skill Generation Audit

yao-meta-skill

技能审计报告Yao Meta Skill Audit Report

-

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

技能名称：yao-meta-skillSkill name: yao-meta-skill成熟度：治理Maturity: governed格式：Agent SkillsFormat: agent-skills更新时间：2026-03-31Updated: 2026-03-31
-
openaiopenaiclaudeclaudegenericgeneric
+
openaiopenaiclaudeclaudegenericgenericvscodevscode
-
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
+
完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
@@ -621,9 +782,9 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。Turn one-off experience into a reusable, evaluable, and portable skill package.
  • 交付结果：SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.htmlDeliverables: SKILL.md, agents/interface.yaml, skill-ir/examples/yao-meta-skill.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_review_kit.md, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/output_blind_answer_key.json, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-interpretation.html, reports/skill-overview.html
-
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
+
交付流程Delivery Flow输入材料Input materialSkill 包体Skill package可复用能力Reusable capability
交付流程把用户输入、生成的包体和可复用能力放在一条线上。The delivery flow places user input, generated package, and reusable capability on one path.
@@ -634,17 +795,24 @@

总览指标Metrics

分数来自本地文件和 reports 证据，缺失时明确标为证据不足。Scores are derived from package files and reports; missing inputs are shown as evidence gaps.

-
-
-
-
评分雷达完整度触发清晰证据充分可维护性可迁移性
评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。
-
-

指标判读Reading

-

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
-
-
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
+

指标判读Reading

+

先看雷达图判断能力短板，再看每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

+
+
+
+
+
+
评分雷达Rating Radar完整度Completeness触发清晰Trigger clarity证据充分Evidence depth可维护性Maintainability可迁移性Portability
评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。The radar chart compares completeness, trigger clarity, evidence, maintainability, and portability.
+
+

成熟度条Maturity Bar

+

这里把每个指标压缩为状态、分数和第一条证据，先给出整体判断，再进入下方明细。This compresses each metric into status, score, and the first evidence item before the detailed cards below.

+
  1. 稳定Stable完整度Completeness100SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  2. 稳定Stable触发清晰Trigger clarity100frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  3. 稳定Stable证据充分Evidence depth100已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  4. 稳定Stable可维护性Maintainability100SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  5. 稳定Stable可迁移性Portability100agents/interface.yaml 已存在。agents/interface.yaml exists.
  6. 关注Watch上下文成本Context cost42入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
+
+
+
+
指标明细Metric Details
+
完整度Completeness100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md exists and acts as the skill entrypoint.
  • README.md 已存在，便于人工阅读。README.md exists for human-readable usage.
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml exists for cross-platform adaptation.
触发清晰Trigger clarity100
  • frontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.
  • description 有足够长度说明任务边界。The description is long enough to explain the task boundary.
  • description 已包含使用场景或排除边界信号。The description includes usage-scenario or exclusion-boundary signals.
证据充分Evidence depth100
  • 已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.
  • skill-ir.json 已存在。skill-ir.json exists.
  • compiled_targets.json 已存在。compiled_targets.json exists.
可维护性Maintainability100
  • SKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.
  • 入口文件保持克制，可维护性较好。The entrypoint stays restrained, which supports maintainability.
  • references/ 已承载扩展指导。references/ carries extended guidance.
可迁移性Portability100
  • agents/interface.yaml 已存在。agents/interface.yaml exists.
  • manifest.json 已存在。manifest.json exists.
  • 目标平台或 adapter target 已声明。Target platforms or adapter targets are declared.
上下文成本Context cost42
  • 入口约 339 个词/字，references 约 16934 个词/字。Entrypoint is about 339 words/characters; references are about 16934.
  • 分数越高代表上下文成本越低。A higher score means lower context cost.
  • 上下文成本偏高，建议压缩入口或拆分 references。Context cost is high; compress the entrypoint or split references further.
@@ -657,10 +825,10 @@

判断这个 Skill 在能力地图中的位置和复用范围。Places this skill on a capability map and clarifies reuse scope.

-
能力矩阵执行确定性知识密度Execution operation
能力矩阵说明这个 Skill 更偏知识密集还是执行确定。
+
能力矩阵Capability Matrix执行确定性Execution certainty知识密度Knowledge densityExecution operationExecution operation
能力矩阵说明这个 Skill 更偏知识密集还是执行确定。The capability matrix shows whether the skill leans toward knowledge density or execution certainty.

画像摘要Profile

-
  • 能力类型：Execution operation能力类型：Execution operation
  • 成熟度：governed成熟度：governed
  • 触发强度：手动触发 + description 路由触发强度：手动触发 + description 路由
  • 复用范围：跨平台复用范围：跨平台
+
  • 能力类型：Execution operationCapability type: Execution operation
  • 成熟度：governedMaturity: governed
  • 触发强度：手动触发 + description 路由Trigger strength: Manual activation plus description-based routing
  • 复用范围：跨平台Reuse scope: Cross-platform
@@ -673,15 +841,15 @@

说明入口、参考、脚本、评估和报告如何组成一个稳定闭环。Explains how entrypoint, references, scripts, evals, and reports form a stable loop.

-
分层结构入口层参考层脚本层评估层报告层
分层结构展示入口、参考、脚本、评估和报告如何各司其职。
+
分层结构Layered Structure入口层Entrypoint layer参考层Reference layer脚本层Script layer评估层Evaluation layer报告层Report layer
分层结构展示入口、参考、脚本、评估和报告如何各司其职。The layered structure shows how entrypoint, references, scripts, evals, and reports each carry a distinct role.

执行流程Execution Flow

-
  1. 判断请求是否应该沉淀为 Skill，并选择最轻量可靠的模式。Decide whether the request should become a skill and choose the lightest fit.
  2. 捕捉任务、输出、排除项、约束和质量标准。Capture job, output, exclusions, constraints, and standards.
  3. 运行参考扫描：先看外部 benchmark，再看用户材料，最后校验本地适配；只暴露不确定性或冲突。Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
  4. 尽早写出 `description`，先测试路由质量，再扩展包体。Write the `description` early and test route quality before expanding the package.
  5. 只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.
+
  1. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.
  2. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Capture job, output, exclusions, constraints, standards, and the lightest fit.
  3. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.
  4. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Write `description` early, test route quality, then add only earned folders and gates.
  5. 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful.

调用方式How To Use

-
  1. 当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  2. 当用户请求与该 Skill 的触发描述匹配时使用。Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
+
  1. 当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  2. 当用户请求与该 Skill 的触发描述匹配时使用。Use this skill when the request matches the frontmatter description.
@@ -697,15 +865,15 @@

触发描述Trigger

-

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.

输入材料Inputs

  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。an existing skill directory that needs refactor, evaluation, packaging, or governance hardening
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。benchmark references, local constraints, desired maturity tier, and review standards

输出结果Outputs

-
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.
  • 结构化 Skill 目录，共 8 类关键资产。结构化 Skill 目录，共 8 类关键资产。
+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.
  • 结构化 Skill 目录，共 8 类关键资产。Structured skill directory with 8 key asset groups.

不应触发Should Not Trigger

-
  • 只需要一次性回答、没有复用价值的临时请求。只需要一次性回答、没有复用价值的临时请求。
  • 要求直接执行相邻任务，而不是沉淀或使用这个 Skill。要求直接执行相邻任务，而不是沉淀或使用这个 Skill。
  • 缺少必要事实且用户不允许澄清的场景。缺少必要事实且用户不允许澄清的场景。
+
  • 只需要一次性回答、没有复用价值的临时请求。One-off requests that do not need reusable skill behavior.
  • 要求直接执行相邻任务，而不是沉淀或使用这个 Skill。Requests to perform an adjacent task directly rather than create or use this skill.
  • 缺少必要事实且用户不允许澄清的场景。Cases that lack required facts and do not allow clarification.
@@ -720,16 +888,16 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。Context cost needs improvement: Entrypoint is about 339 words/characters; references are about 16934.纳入下一轮修复Fix next

执行证据Execution Evidence

-
  • 变体运行：10变体运行：10
  • 模型执行：0模型执行：0
  • 记录样本：0记录样本：0
  • Token 估算：10Token 估算：10
+
  • 变体运行：10Variant runs: 10
  • 模型执行：10Model executions: 10
  • 记录样本：0Recorded fixtures: 0
  • Token 估算：0Token estimates: 0

盲评审定Blind Adjudication

-
  • 评审进度：0 / 5评审进度：0 / 5
  • 待评审：5待评审：5
  • 一致率：暂无一致率：暂无
  • 非法决策：0非法决策：0
+
  • 评审进度：0 / 5Review progress: 0 / 5
  • 待评审：5Pending review: 5
  • 一致率：暂无Agreement rate: not available yet
  • 非法决策：0Invalid decisions: 0
@@ -753,14 +921,15 @@

提前暴露误触发、漂移、证据不足和迁移风险。Surfaces trigger, drift, evidence, and portability risks before the package grows.

-
风险热力000200300发生概率影响程度
风险热力图用影响程度和发生概率标出当前治理重点。
+
风险热力Risk Heatmap000200300发生概率Probability影响程度Impact
风险热力图用影响程度和发生概率标出当前治理重点。The risk heatmap marks governance priorities by impact and probability.
- +
风险Risk信号Signal应对Response
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
误触发风险Trigger riskfrontmatter description 已存在，具备基础路由面。The frontmatter description exists, giving the skill a basic routing surface.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
输出漂移风险Output drift risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
证据不足风险Evidence gap risk已生成 20 / 20 类报告证据。Generated 20 / 20 evidence report types.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
包体膨胀风险Package bloat riskSKILL.md 约 339 个词/字。SKILL.md is about 339 words/characters.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
跨平台迁移风险Portability riskagents/interface.yaml 已存在。agents/interface.yaml exists.先补证据和边界，再增加包体复杂度。Improve evidence and boundaries before adding package complexity.
+

世界证据World Evidence

世界级证据尚未完成：4 项待补，0 项已接受。World-class evidence is not complete: 4 pending, 0 accepted.

证据待补Evidence pending
待补证据Pending4仍需外部或人工证据接受。External or human evidence still needs acceptance.
已接受Accepted0已通过 source check 与提交契约。Passed source checks and submission contract.
源检查Source Checks11 / 19通过数 / 总检查数。Passed checks / total checks.
外部证据External evidence

提供商留出Provider Holdout

缺少真实 provider 模型运行和 token metadata。Missing a real provider model run and token metadata.

阻塞检查Blocked Checks

当前没有阻塞检查。No blocked checks right now.

人工证据Human evidence

人工盲评Human Adjudication

盲评 pair 仍待真实 reviewer 决策。Blind-review pairs still need real reviewer decisions.

阻塞检查Blocked Checks
  • 无待判定No pending decisions
  • 盲评完成Judgments complete
  • Reviewer metadataReviewer metadata
外部证据External evidence

原生权限Native Permission

原生 runtime enforcement 仍待目标客户端或外部安装器证明。Native runtime enforcement still needs target-client or external-installer proof.

阻塞检查Blocked Checks
  • 原生执行Native enforcement
外部证据External evidence

原生遥测Native Telemetry

真实外部客户端 metadata-only 事件仍未导入。Real external-client metadata-only events have not been imported yet.

阻塞检查Blocked Checks
  • 外部事件External events
  • 采用样本Adoption sample
@@ -771,7 +940,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布300项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布Asset Distribution456项456 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
@@ -787,8 +956,8 @@

把下一步升级收束为少数高价值动作。Keeps next iteration moves focused and actionable.

-
迭代时间下一步 1Borrow one proven…下一步 2Harden portabilit…下一步 3Create an iterati…
迭代时间线把下一步升级收束成少数可执行动作。
-
下一步 1Next 1

有选择地借鉴一个成熟模式Borrow one proven pattern on purpose

已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.

  • 阅读 `obra/superpowers` 中最值得借鉴的模式。Read the strongest pattern from obra/superpowers.
  • 先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。Decide whether to borrow method, structure, execution, or portability, but only one of them first.
  • 记录本轮不借鉴的内容，避免包体过重。Record what you will not borrow so the package stays light.

包体形态更清晰，也减少偶然过度设计。A cleaner package shape with less accidental over-design.

下一步 2Next 2

加固跨环境语义Harden portability semantics

这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。The skill already signals reuse across environments, so contract clarity matters early.

  • 确认激活模式、执行上下文和信任假设。Confirm activation mode, execution context, and trust assumptions.
  • 补充或复核非原生目标端的降级策略。Add or review degradation strategy for non-native targets.
  • 至少打包一次 Skill，用来验证 adapter 预期。Package the skill once to verify adapter expectations.

跨环境复用更安全，目标漂移更少。Safer cross-environment reuse with less target drift.

下一步 3Next 3

建立迭代证据回路Create an iteration evidence loop

第一版之后，包体应该能说明改了什么以及为什么改。The package should show what changed and why after the first draft.

  • 生成 HTML Skill 报告，并保持它与包体内容一致。Generate the HTML skill report and keep it aligned with the package.
  • 记录参考扫描的取舍和非目标。Record reference scan choices and non-goals.
  • 在继续增加文件前，明确记录下一轮迭代选择。Capture the next iteration choice explicitly before adding more files.

让下一位作者或评审者更容易接手。A clearer path for the next author or reviewer.

+
迭代时间Iteration Timeline下一步 1Next 1补齐世界证据Close world-class evide...下一步 2Next 2Borrow one proven…Borrow one proven…下一步 3Next 3Harden portabilit…Harden portabilit…
迭代时间线把下一步升级收束成少数可执行动作。The iteration timeline narrows the next upgrade into a few executable moves.
+
下一步 1Next 1

补齐世界证据Close world-class evidence

世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。World-class evidence still has 4 pending item(s); public completion claims must stay blocked.

  • 补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。Close provider holdout evidence: Missing a real provider model run and token metadata.
  • 补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。Close human adjudication evidence: Blind-review pairs still need real reviewer decisions.
  • 按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。Close all 4 pending evidence item(s) in the ledger (external 3, human 1; 2 not shown here) and keep the claim guard pending.

全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。Only after the ledger accepts all external and human evidence should the public world-class claim move to review.

下一步 2Next 2

有选择地借鉴一个成熟模式Borrow one proven pattern on purpose

已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.

  • 阅读 `obra/superpowers` 中最值得借鉴的模式。Read the strongest pattern from obra/superpowers.
  • 先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。Decide whether to borrow method, structure, execution, or portability, but only one of them first.
  • 记录本轮不借鉴的内容，避免包体过重。Record what you will not borrow so the package stays light.

包体形态更清晰，也减少偶然过度设计。A cleaner package shape with less accidental over-design.

下一步 3Next 3

加固跨环境语义Harden portability semantics

这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。The skill already signals reuse across environments, so contract clarity matters early.

  • 确认激活模式、执行上下文和信任假设。Confirm activation mode, execution context, and trust assumptions.
  • 补充或复核非原生目标端的降级策略。Add or review degradation strategy for non-native targets.
  • 至少打包一次 Skill，用来验证 adapter 预期。Package the skill once to verify adapter expectations.

跨环境复用更安全，目标漂移更少。Safer cross-environment reuse with less target drift.

diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 41de1c7a..a6034e28 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -3,7 +3,7 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "maturity": "governed", "updated_at": "2026-03-31", "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", @@ -11,13 +11,15 @@ "deliverables": [ "SKILL.md", "agents/interface.yaml", - "reports/skill-ir.json", + "skill-ir/examples/yao-meta-skill.json", "reports/compiled_targets.md", "reports/output_quality_scorecard.md", "reports/output_execution_runs.md", "reports/output_blind_review_pack.md", - "reports/output_blind_answer_key.json", + "reports/output_review_kit.md", "reports/output_review_adjudication.md", + "reports/benchmark_reproducibility.md", + "reports/output_blind_answer_key.json", "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/runtime_permission_probes.md", @@ -28,8 +30,11 @@ "reports/upgrade_check.md", "reports/adoption_drift_report.md", "reports/review_waivers.md", + "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_ledger.md", "reports/review_annotations.md", "reports/review-studio.html", + "reports/skill-interpretation.html", "reports/skill-overview.html" ], "flow": [ @@ -75,7 +80,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 365 个词/字。", + "SKILL.md 约 339 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -96,7 +101,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 365 个词/字，references 约 14247 个词/字。", + "入口约 339 个词/字，references 约 16934 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -150,7 +155,7 @@ }, "contract_boundary": { "trigger": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -178,7 +183,7 @@ "boundary_cards": [ { "label": "Owned", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." }, { "label": "Adjacent", @@ -200,7 +205,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。" + "上下文成本需要补强：入口约 339 个词/字，references 约 16934 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -210,10 +215,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -265,7 +270,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 365 个词/字。", + "signal": "SKILL.md 约 339 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -397,6 +402,78 @@ "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." ] }, + "world_class_readiness": { + "ready": false, + "decision": "evidence-pending", + "entry_count": 4, + "pending_count": 4, + "accepted_count": 0, + "external_pending_count": 3, + "human_pending_count": 1, + "source_check_count": 19, + "source_pass_count": 11, + "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", + "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", + "entries": [ + { + "key": "provider-holdout", + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + "blocked_checks": [] + }, + { + "key": "human-adjudication", + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "category": "human", + "category_zh": "人工证据", + "category_en": "Human evidence", + "status": "pending", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + "blocked_checks": [ + "No pending decisions", + "Judgments complete", + "Reviewer metadata" + ] + }, + { + "key": "native-permission-enforcement", + "label_zh": "原生权限", + "label_en": "Native Permission", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + "blocked_checks": [ + "Native enforcement" + ] + }, + { + "key": "native-client-telemetry", + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "category": "external", + "category_zh": "外部证据", + "category_en": "External evidence", + "status": "pending", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + "blocked_checks": [ + "External events", + "Adoption sample" + ] + } + ] + }, "package_assets": { "entries": [ { @@ -427,28 +504,28 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 32 + "file_count": 35 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 66 + "file_count": 153 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 28 + "file_count": 29 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 170 + "file_count": 235 } ], - "file_count": 300, + "file_count": 456, "folder_count": 4, "distribution": [ { @@ -469,24 +546,35 @@ }, { "label": "references", - "value": 32 + "value": 35 }, { "label": "scripts", - "value": 66 + "value": 153 }, { "label": "evals", - "value": 28 + "value": 29 }, { "label": "reports", - "value": 170 + "value": 235 } ] }, "iteration_roadmap": { "items": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, { "title": "Borrow one proven pattern on purpose", "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", @@ -506,16 +594,6 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift." - }, - { - "title": "Create an iteration evidence loop", - "why": "The package should show what changed and why after the first draft.", - "actions": [ - "Generate the HTML skill report and keep it aligned with the package.", - "Record reference scan choices and non-goals.", - "Capture the next iteration choice explicitly before adding more files." - ], - "unlocks": "A clearer path for the next author or reviewer." } ] }, @@ -555,17 +633,17 @@ "name": "yao-meta-skill", "title": "Yao Meta Skill", "display_name": "Yao Meta Skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "logic_steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "usage_steps": [ "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", - "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." ], "package_map": [ { @@ -596,25 +674,25 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 32 + "file_count": 35 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 66 + "file_count": 153 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 28 + "file_count": 29 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 170 + "file_count": 235 } ], "strengths": [ @@ -632,7 +710,7 @@ "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." ], "trigger_contract": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "activation": "manual", "execution": "inline", "shell": "bash" @@ -676,6 +754,17 @@ } ], "roadmap": [ + { + "title": "补齐世界证据", + "why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": [ + "补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。", + "补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。", + "按 ledger 总量继续补齐 4 项待补证据（外部 3 项、人工 1 项；未展开 2 项），并保持 claim guard 为 pending 状态。" + ], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger" + }, { "title": "Borrow one proven pattern on purpose", "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", @@ -695,16 +784,6 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift." - }, - { - "title": "Create an iteration evidence loop", - "why": "The package should show what changed and why after the first draft.", - "actions": [ - "Generate the HTML skill report and keep it aligned with the package.", - "Record reference scan choices and non-goals.", - "Capture the next iteration choice explicitly before adding more files." - ], - "unlocks": "A clearer path for the next author or reviewer." } ], "cards": [], @@ -716,7 +795,8 @@ "benchmark_highlights": [], "skill_ir": { "schema_version": "2.0.0", - "target_count": 4, + "source_path": "skill-ir/examples/yao-meta-skill.json", + "target_count": 5, "trigger_samples": 8, "output_eval_cases": 3 }, @@ -724,8 +804,8 @@ "ok": true, "schema_version": "1.0", "summary": { - "target_count": 4, - "pass_count": 4, + "target_count": 5, + "pass_count": 5, "warn_count": 0, "block_count": 0, "failure_count": 0, @@ -790,6 +870,22 @@ ], "unsupported_features": [], "warnings": [] + }, + { + "target": "vscode", + "status": "pass", + "adapter_mode": "vscode-agent-skills-adapter", + "degradation_strategy": "agent-skills-source-with-vscode-notes", + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "permission_enforcement": "client-or-workspace-trust", + "generated_files": [ + "targets/vscode/adapter.json", + "targets/vscode/README.md" + ], + "unsupported_features": [ + "VS Code installation scope is documented but not installed by this compiler" + ], + "warnings": [] } ], "failures": [], @@ -814,14 +910,14 @@ "case_count": 5, "variant_run_count": 10, "command_executed_count": 10, - "model_executed_count": 0, + "model_executed_count": 10, "recorded_fixture_count": 0, "timing_observed_count": 10, - "token_observed_count": 0, - "token_estimated_count": 10, - "baseline_pass_rate": 0.0, - "with_skill_pass_rate": 100.0, - "delta": 100.0, + "token_observed_count": 10, + "token_estimated_count": 0, + "baseline_pass_rate": 23.67, + "with_skill_pass_rate": 43.67, + "delta": 20.0, "regression_count": 0, "failure_count": 0, "gate_pass": true @@ -830,9 +926,19 @@ "mode": "command", "command": [ "python3", - "scripts/local_output_eval_runner.py" + "scripts/provider_output_eval_runner.py", + "--provider", + "deepseek", + "--model", + "deepseek-v4-flash", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--api-key-env", + "DEEPSEEK_API_KEY" ], - "timeout_seconds": 30.0 + "timeout_seconds": 180.0 }, "failures": [] }, @@ -846,6 +952,31 @@ "pair_count": 5, "answer_key_separate": true }, + "output_review_kit": { + "ok": true, + "summary": { + "case_count": 5, + "ready_for_adjudication_count": 0, + "pending_decision_count": 5, + "invalid_decision_count": 0, + "reviewer_metadata_present": false, + "answer_key_hidden": true, + "answer_key_path_exposed": false, + "ready_to_run_adjudication": false, + "failure_count": 0 + }, + "artifacts": { + "reviewer_kit_json": "reports/output_review_kit.json", + "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", + "blind_pack_json": "reports/output_blind_review_pack.json", + "blind_pack_markdown": "reports/output_blind_review_pack.md", + "decisions": "reports/output_review_decisions.json", + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md" + }, + "failures": [] + }, "output_review_adjudication": { "ok": true, "summary": { @@ -855,14 +986,77 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, - "failure_count": 0 + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "blind_review_completed_before_answer_key": false, + "answer_key_not_opened_before_decisions": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "failure_count": 0, + "reviewer_checklist_count": 5, + "reviewer_checklist_pending_count": 5, + "reviewer_checklist_invalid_count": 0, + "reviewer_checklist_ready_count": 0 }, "reviewer": "", "reviewed_at": "", "failures": [] }, + "benchmark_reproducibility": { + "ok": true, + "summary": { + "reproducibility_ready": true, + "release_lock_ready": true, + "methodology_complete": true, + "required_artifact_count": 25, + "missing_artifact_count": 0, + "evidence_bundle_sha256": "55a6c33c12e14d6507d4a5d4d4cf1268db35fb323d523e8b23ce9c73aedf8db0", + "source_contract_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 23, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 10, + "token_observed_count": 10, + "human_review_complete": false, + "provider_evidence_complete": true, + "world_class_ready": false, + "world_class_open_gap_count": 3, + "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, + "world_class_source_check_count": 19, + "world_class_source_pass_count": 11, + "world_class_source_blocked_count": 8, + "beta_test_ready": true, + "beta_test_blocker_count": 0, + "beta_test_deferred_evidence_count": 4, + "public_claim_ready": false, + "public_claim_blocker_count": 3, + "working_tree_dirty": true, + "changed_file_count": 29, + "source_tree_dirty": false, + "source_changed_file_count": 0, + "generated_tree_dirty": true, + "generated_changed_file_count": 29 + }, + "commit": "e79050ed12d057811d7d13ab7d8bcc5a06af76d4", + "missing_artifacts": [], + "limitations": [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims." + ] + }, "runtime_conformance": { "target_count": 5, "pass_count": 5, @@ -871,14 +1065,22 @@ "runtime_permissions": { "ok": true, "summary": { - "target_count": 3, - "pass_count": 3, + "target_count": 4, + "pass_count": 4, "fail_count": 0, "native_enforcement_count": 0, - "metadata_fallback_count": 3, - "residual_risk_count": 3, + "metadata_fallback_count": 4, + "residual_risk_count": 4, "required_capability_count": 3, - "failure_count": 0 + "failure_count": 0, + "installer_enforcement_source_status": "present", + "installer_enforcement_target_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "installer_permission_capability_count": 3, + "world_class_native_evidence_ready": false, + "installer_enforcement_ready": true }, "expected_capabilities": [ "file_write", @@ -915,33 +1117,43 @@ "residual_risks": [ "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." ] + }, + { + "target": "vscode", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] } ], "failures": [] }, "trust_security": { - "scanned_files": 150, - "script_count": 66, - "internal_module_count": 7, + "scanned_files": 245, + "script_count": 153, + "internal_module_count": 67, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], - "network_script_count": 2, - "network_policy_covered_count": 2, + "network_script_count": 3, + "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 47, + "file_write_script_count": 73, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 59, + "help_smoke_checked_count": 86, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 150, - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "package_hash_file_count": 245, + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" }, "skill_atlas": { "skill_count": 12, @@ -954,6 +1166,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "registry_distribution": { @@ -962,12 +1177,13 @@ "schema_version": "2.0", "name": "yao-meta-skill", "version": "1.1.0", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "targets": [ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "maturity": "governed", "owner": "Yao Team", @@ -975,8 +1191,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f", - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "package_sha256": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, "compatibility": { "openai": "pass", @@ -1007,7 +1223,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1020,11 +1236,11 @@ "package_verification": { "ok": true, "summary": { - "target_count": 3, - "adapter_count": 3, + "target_count": 4, + "adapter_count": 4, "archive_present": true, - "archive_sha256": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9", - "archive_entry_count": 488, + "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", + "archive_entry_count": 689, "failure_count": 0, "warning_count": 0 }, @@ -1035,12 +1251,16 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 488, + "archive_entry_count": 696, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, "interface_loaded": true, - "adapter_count": 3, + "adapter_count": 4, + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 4, + "permission_capability_count": 3, "install_root_is_temp": false, "failure_count": 0, "warning_count": 0 @@ -1054,7 +1274,7 @@ "declared_bump": "minor", "recommended_bump": "minor", "breaking_change_count": 0, - "added_target_count": 1, + "added_target_count": 2, "metadata_change_count": 1, "checksum_change_count": 2, "failure_count": 0, @@ -1063,7 +1283,8 @@ "upgrade_diff": { "name_changed": false, "added_targets": [ - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "removed_targets": [], "compatibility_changes": [ @@ -1090,25 +1311,25 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ { "field": "archive_sha256", "from": "", - "to": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "to": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "to": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] }, "release_notes": [ "Recommended version bump: minor.", - "Added targets: agent-skills-compatible.", + "Added targets: agent-skills-compatible, vscode.", "Package or archive checksum changed; reviewers should verify package artifacts before release.", "Package verification evidence: reports/package_verification.md." ], @@ -1119,13 +1340,14 @@ "ok": true, "summary": { "event_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1134,9 +1356,13 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, - "failure_types": {} + "failure_types": {}, + "source_types": { + "manual": 1 + }, + "command_counts": {} }, "next_iteration_candidates": [], "privacy_contract": { @@ -1169,25 +1395,68 @@ "expired_count": 0, "invalid_count": 0, "covered_gate_count": 0, - "covered_gate_keys": [] + "covered_gate_keys": [], + "waiver_candidate_count": 2, + "waiverable_open_count": 1, + "non_waivable_count": 1 }, "policy": { "blocker_waivers_allowed": false, "minimum_reason_chars": 20, "expires_required": true, - "known_gate_keys": [ + "review_studio_gate_keys": [ + "architecture-maintainability", "context-budget", "intent-canvas", "operations-loop", "output-lab", "permission-gates", "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "world-class-evidence" + ], + "known_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", "registry-audit", "release-notes", "runtime-matrix", "skill-atlas", "trigger-lab", "trust-report" + ], + "waiverable_gate_keys": [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ], + "non_waivable_gate_keys": [ + "review-waivers", + "world-class-evidence" ] }, "failures": [], @@ -1209,6 +1478,743 @@ "annotations": [], "failures": [] }, + "world_class_evidence_plan": { + "ok": true, + "summary": { + "audit_decision": "continue-iteration", + "world_class_ready": false, + "audit_world_class_ready": false, + "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, + "task_count": 4, + "human_task_count": 1, + "external_task_count": 3, + "review_task_count": 0, + "decision": "collect-external-evidence" + }, + "tasks": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_audit": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "open_gap_count": 3 + } + }, + "world_class_evidence_ledger": { + "ok": true, + "summary": { + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "source_accepted": false, + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4, + "evidence_requirement_count": 4 + } + }, "synthesis_highlights": [ "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", "Borrow a review checkpoint wherever trust matters more than raw speed.", @@ -1217,10 +2223,10 @@ "artifact_design": { "design_system": "metric editorial", "highlights": [ - "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.", + "Avoid paragraph-heavy table cells.", + "Keep charts tied to one analytical question each.", + "Preserve stable color meaning across metrics and entities." ] }, "prompt_quality": { @@ -1249,7 +2255,8 @@ "targets": [ "openai", "claude", - "generic" + "generic", + "vscode" ], "maturity_tier": "governed", "skill_archetype": "governed", diff --git a/reports/skill_atlas.html b/reports/skill_atlas.html index 0c14b4a4..196b1c40 100644 --- a/reports/skill_atlas.html +++ b/reports/skill_atlas.html @@ -22,22 +22,28 @@

Skill Atlas

-

Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

+

Portfolio-level review for route overlap, stale ownership, shared resources, telemetry drift, and no-route opportunities.

Skills12
Actionable1
Route Collisions0
Owner Gaps0
Stale Skills0
+
Drift Signals0
No-Route Opportunities0

Actionable Issues

  • No blocking portfolio issues detected.
+
+

No-Route Opportunities

+

Missed-trigger telemetry and explicit failure cases become candidate routing work without storing raw prompts or outputs.

+
  • No no-route opportunities detected.
+

Full Portfolio Counts

-

All scanned skills remain visible: 5 total route collisions, 9 total owner gaps, and 10 total stale signals.

+

All scanned skills remain visible: 5 total route collisions, 9 total owner gaps, 10 total stale signals, and 0 telemetry drift signals.

Catalog

diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 9e4ab393..c8d8ed7e 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -12,6 +12,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "scope_policy": { @@ -46,7 +49,7 @@ { "name": "yao-meta-skill", "path": ".", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "owner": "Yao Team", "version": "1.1.0", "status": "active", @@ -57,37 +60,68 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "resources": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -102,32 +136,90 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", + "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py", "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -155,35 +247,48 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", "references/trust-security-method.md", + "references/user-memory-policy.md", + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], "token_set": [ - "add", - "asked", + "creation", + "distribution", "docs", "evals", "evaluate", - "existing", - "improve", + "improvement", "notes", "or", "package", - "process", + "packaging", "prompts", "refactor", - "repeated", - "reuse", - "team", + "team-ready", "transcripts", + "workflow", "workflows" ], "atlas_scope": "release", "actionable": true, - "scope_reason": "default release-actionable skill" + "scope_reason": "default release-actionable skill", + "telemetry": { + "report_present": true, + "report": "reports/adoption_drift_report.json", + "risk_band": "low", + "event_count": 1, + "adoption_sample_count": 1, + "adoption_rate": 100.0, + "candidate_count": 0 + } }, { "name": "release-orchestrator", @@ -241,7 +346,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/complex-release-orchestrator/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -280,7 +394,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/final/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -303,7 +426,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v0/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -341,7 +473,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v1/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "incident-command-governor", @@ -382,7 +523,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/governed-incident-command/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "note-cleanup", @@ -419,7 +569,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/simple-note-cleanup/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -459,7 +618,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/team-frontend-review/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "geo-ranking-article-generator", @@ -519,7 +687,16 @@ ], "atlas_scope": "embedded-generated-skill", "actionable": false, - "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release.", + "telemetry": { + "report_present": false, + "report": "geo-ranking-article-generator/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "invalid-governance-skill", @@ -541,7 +718,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/governance_invalid_manifest/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-yaml-skill", @@ -565,7 +751,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_invalid_yaml/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-skill", @@ -588,7 +783,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_missing_interface_field/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } } ], "summary": { @@ -602,6 +806,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 } }, @@ -892,6 +1099,8 @@ } ], "actionable_owner_review_gaps": [], + "drift_signals": [], + "actionable_drift_signals": [], "no_route_opportunities": [], "artifacts": { "catalog": "skill_atlas/catalog.json", @@ -899,6 +1108,7 @@ "dependency_graph": "skill_atlas/dependency_graph.json", "stale_skills": "skill_atlas/stale_skills.json", "owner_review_gaps": "skill_atlas/owner_review_gaps.json", + "drift_signals": "skill_atlas/drift_signals.json", "no_route_opportunities": "skill_atlas/no_route_opportunities.json", "report_json": "reports/skill_atlas.json", "report_html": "reports/skill_atlas.html" diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json new file mode 100644 index 00000000..96ff4372 --- /dev/null +++ b/reports/skill_os2_audit.json @@ -0,0 +1,438 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "item_count": 15, + "pass_count": 12, + "warn_count": 0, + "human_required_count": 1, + "external_required_count": 2, + "missing_count": 0, + "open_gap_count": 3, + "local_foundation_ready": true, + "world_class_ready": false, + "decision": "continue-iteration" + }, + "status_counts": { + "pass": 12, + "warn": 0, + "human_required": 1, + "external_required": 2, + "missing": 0 + }, + "items": [ + { + "key": "skill-ir", + "label": "Skill IR", + "status": "pass", + "current": "schema 2.0.0; targets 5", + "target": "2.0 schema, root export, and target-neutral contract evidence", + "evidence": [ + { + "path": "skill-ir/schema.json", + "exists": true + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "exists": true + }, + { + "path": "references/skill-ir-method.md", + "exists": true + } + ], + "next_action": "Keep IR as the source before target packaging." + }, + { + "key": "target-compiler", + "label": "Target Compiler", + "status": "pass", + "current": "5/5 targets pass", + "target": "OpenAI, Claude, generic, Agent Skills compatible, and VS Code contracts generated from IR", + "evidence": [ + { + "path": "scripts/compile_skill.py", + "exists": true + }, + { + "path": "reports/compiled_targets.json", + "exists": true + }, + { + "path": "tests/verify_compile_skill.py", + "exists": true + } + ], + "next_action": "Deepen target-native transforms when provider clients expose stronger runtime APIs." + }, + { + "key": "output-eval-lab", + "label": "Output Eval Lab", + "status": "pass", + "current": "5 cases; delta 100.0; exec 10; blind 5", + "target": "with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy", + "evidence": [ + { + "path": "evals/output/cases.jsonl", + "exists": true + }, + { + "path": "scripts/run_output_eval.py", + "exists": true + }, + { + "path": "scripts/run_output_execution.py", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "reports/output_execution_runs.json", + "exists": true + }, + { + "path": "reports/output_blind_review_pack.json", + "exists": true + } + ], + "next_action": "Add more real-file and adversarial holdout cases as usage grows." + }, + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "current": "model-executed 10; token-observed 10", + "target": "At least one real provider-backed holdout run with observed model/timing/token metadata", + "evidence": [ + { + "path": "scripts/provider_output_eval_runner.py", + "exists": true + }, + { + "path": "reports/output_execution_runs.json", + "exists": true + } + ], + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "current": "0/5 decisions; pending 5", + "target": "Real reviewer decisions, blind-review attestation, and integrity fingerprints recorded before claiming output review completion", + "evidence": [ + { + "path": "reports/output_review_decisions.json", + "exists": true + }, + { + "path": "reports/output_review_adjudication.json", + "exists": true + }, + { + "path": "scripts/adjudicate_output_review.py", + "exists": true + } + ], + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "benchmark-reproducibility", + "label": "Benchmark Reproducibility", + "status": "pass", + "current": "artifacts 25; missing 0; failures 3", + "target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable", + "evidence": [ + { + "path": "reports/benchmark_methodology.md", + "exists": true + }, + { + "path": "reports/benchmark_reproducibility.json", + "exists": true + }, + { + "path": "reports/benchmark_reproducibility.md", + "exists": true + }, + { + "path": "evals/failure-cases.md", + "exists": true + }, + { + "path": "tests/verify_benchmark_reproducibility.py", + "exists": true + } + ], + "next_action": "Keep the manifest current with every benchmark, package, and release evidence change." + }, + { + "key": "runtime-conformance", + "label": "Runtime Conformance", + "status": "pass", + "current": "5/5 targets pass", + "target": "Target package structure, metadata, relative paths, and degradation notes pass", + "evidence": [ + { + "path": "runtime/conformance/schema.json", + "exists": true + }, + { + "path": "scripts/run_conformance_suite.py", + "exists": true + }, + { + "path": "reports/conformance_matrix.json", + "exists": true + } + ], + "next_action": "Keep target conformance fixtures updated as platform contracts change." + }, + { + "key": "trust-security", + "label": "Trust Security", + "status": "pass", + "current": "secrets 0; scripts 153; help failures 0", + "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", + "evidence": [ + { + "path": "scripts/trust_check.py", + "exists": true + }, + { + "path": "reports/security_trust_report.json", + "exists": true + }, + { + "path": "security/permission_policy.json", + "exists": true + } + ], + "next_action": "Keep high-permission approvals scoped, expiring, and target-mapped." + }, + { + "key": "runtime-permission-metadata", + "label": "Permission Metadata", + "status": "pass", + "current": "4/4 target probes pass; metadata fallback 4; installer enforcement 4", + "target": "Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available", + "evidence": [ + { + "path": "scripts/probe_runtime_permissions.py", + "exists": true + }, + { + "path": "reports/runtime_permission_probes.json", + "exists": true + } + ], + "next_action": "Preserve residual-risk notes until real native enforcement exists." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "current": "native-enforced targets 0; installer-enforced targets 4", + "target": "At least one target/client enforces approved permissions at runtime", + "evidence": [ + { + "path": "reports/runtime_permission_probes.json", + "exists": true + }, + { + "path": "reports/install_simulation.json", + "exists": true + }, + { + "path": "security/permission_policy.json", + "exists": true + } + ], + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "skill-atlas", + "label": "Skill Atlas", + "status": "pass", + "current": "12 skills; actionable collisions 0", + "target": "Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities", + "evidence": [ + { + "path": "scripts/build_skill_atlas.py", + "exists": true + }, + { + "path": "skill_atlas/catalog.json", + "exists": true + }, + { + "path": "reports/skill_atlas.json", + "exists": true + }, + { + "path": "skill_atlas/policy.json", + "exists": true + } + ], + "next_action": "Feed real drift data into Atlas once client telemetry is installed." + }, + { + "key": "registry-distribution", + "label": "Registry Distribution", + "status": "pass", + "current": "zip entries 689; install failures 0; permission failures 0", + "target": "Package metadata, archive checksum, package verification, and install simulation pass", + "evidence": [ + { + "path": "registry/packages/yao-meta-skill.json", + "exists": true + }, + { + "path": "reports/package_verification.json", + "exists": true + }, + { + "path": "reports/install_simulation.json", + "exists": true + } + ], + "next_action": "Regenerate registry after package verification so checksums stay aligned." + }, + { + "key": "review-studio", + "label": "Review Studio", + "status": "pass", + "current": "decision review; warnings 3; score 91", + "target": "One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations", + "evidence": [ + { + "path": "scripts/render_review_studio.py", + "exists": true + }, + { + "path": "reports/review-studio.json", + "exists": true + }, + { + "path": "reports/review-studio.html", + "exists": true + } + ], + "next_action": "Resolve human/external warning gates before claiming full release readiness." + }, + { + "key": "telemetry-drift", + "label": "Telemetry Drift", + "status": "pass", + "current": "events 1; risk low; recipes 5", + "target": "Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path", + "evidence": [ + { + "path": "reports/adoption_drift_report.json", + "exists": true + }, + { + "path": "reports/telemetry_hook_recipes.json", + "exists": true + }, + { + "path": "scripts/import_telemetry_events.py", + "exists": true + } + ], + "next_action": "Keep raw JSONL out of distributed packages and use aggregate reports for Atlas." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "current": "external source events 0; adoption samples 0", + "target": "A real Browser/Chrome/provider client sends production metadata events", + "evidence": [ + { + "path": "scripts/telemetry_native_host.py", + "exists": true + }, + { + "path": "reports/adoption_drift_report.json", + "exists": true + } + ], + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "next_highest_leverage": [ + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "current": "0/5 decisions; pending 5", + "target": "Real reviewer decisions, blind-review attestation, and integrity fingerprints recorded before claiming output review completion", + "evidence": [ + { + "path": "reports/output_review_decisions.json", + "exists": true + }, + { + "path": "reports/output_review_adjudication.json", + "exists": true + }, + { + "path": "scripts/adjudicate_output_review.py", + "exists": true + } + ], + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "current": "native-enforced targets 0; installer-enforced targets 4", + "target": "At least one target/client enforces approved permissions at runtime", + "evidence": [ + { + "path": "reports/runtime_permission_probes.json", + "exists": true + }, + { + "path": "reports/install_simulation.json", + "exists": true + }, + { + "path": "security/permission_policy.json", + "exists": true + } + ], + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "current": "external source events 0; adoption samples 0", + "target": "A real Browser/Chrome/provider client sends production metadata events", + "evidence": [ + { + "path": "scripts/telemetry_native_host.py", + "exists": true + }, + { + "path": "reports/adoption_drift_report.json", + "exists": true + } + ], + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "artifacts": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "world_class_evidence_plan": "reports/world_class_evidence_plan.md" + } +} diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md new file mode 100644 index 00000000..60878f68 --- /dev/null +++ b/reports/skill_os2_audit.md @@ -0,0 +1,101 @@ +# Skill OS 2.0 Audit + +Generated at: `2026-06-17` + +## Summary + +- decision: `continue-iteration` +- pass: `12` / `15` +- human required: `1` +- external required: `2` +- missing: `0` +- world-class ready: `false` +- evidence plan: `reports/world_class_evidence_plan.md` + +## Audit Items + +| Area | Status | Current | Target | Next action | +| --- | --- | --- | --- | --- | +| Skill IR | pass | schema 2.0.0; targets 5 | 2.0 schema, root export, and target-neutral contract evidence | Keep IR as the source before target packaging. | +| Target Compiler | pass | 5/5 targets pass | OpenAI, Claude, generic, Agent Skills compatible, and VS Code contracts generated from IR | Deepen target-native transforms when provider clients expose stronger runtime APIs. | +| Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. | +| Provider Holdout | pass | model-executed 10; token-observed 10 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions, blind-review attestation, and integrity fingerprints recorded before claiming output review completion | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | +| Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | +| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | +| Trust Security | pass | secrets 0; scripts 153; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | +| Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | +| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | +| Registry Distribution | pass | zip entries 689; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | +| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | +| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | + +## Open Highest-Leverage Gaps + +- `human-adjudication` (human-required): Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. +- `native-permission-enforcement` (external-required): Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. +- `native-client-telemetry` (external-required): Install a real client against the native host and import production metadata-only events. + +## Evidence + +### Skill IR + +- existing evidence: `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `references/skill-ir-method.md` + +### Target Compiler + +- existing evidence: `scripts/compile_skill.py`, `reports/compiled_targets.json`, `tests/verify_compile_skill.py` + +### Output Eval Lab + +- existing evidence: `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `reports/output_quality_scorecard.json`, `reports/output_execution_runs.json`, `reports/output_blind_review_pack.json` + +### Provider Holdout + +- existing evidence: `scripts/provider_output_eval_runner.py`, `reports/output_execution_runs.json` + +### Human Adjudication + +- existing evidence: `reports/output_review_decisions.json`, `reports/output_review_adjudication.json`, `scripts/adjudicate_output_review.py` + +### Benchmark Reproducibility + +- existing evidence: `reports/benchmark_methodology.md`, `reports/benchmark_reproducibility.json`, `reports/benchmark_reproducibility.md`, `evals/failure-cases.md`, `tests/verify_benchmark_reproducibility.py` + +### Runtime Conformance + +- existing evidence: `runtime/conformance/schema.json`, `scripts/run_conformance_suite.py`, `reports/conformance_matrix.json` + +### Trust Security + +- existing evidence: `scripts/trust_check.py`, `reports/security_trust_report.json`, `security/permission_policy.json` + +### Permission Metadata + +- existing evidence: `scripts/probe_runtime_permissions.py`, `reports/runtime_permission_probes.json` + +### Native Permission Enforcement + +- existing evidence: `reports/runtime_permission_probes.json`, `reports/install_simulation.json`, `security/permission_policy.json` + +### Skill Atlas + +- existing evidence: `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `reports/skill_atlas.json`, `skill_atlas/policy.json` + +### Registry Distribution + +- existing evidence: `registry/packages/yao-meta-skill.json`, `reports/package_verification.json`, `reports/install_simulation.json` + +### Review Studio + +- existing evidence: `scripts/render_review_studio.py`, `reports/review-studio.json`, `reports/review-studio.html` + +### Telemetry Drift + +- existing evidence: `reports/adoption_drift_report.json`, `reports/telemetry_hook_recipes.json`, `scripts/import_telemetry_events.py` + +### Native Client Telemetry + +- existing evidence: `scripts/telemetry_native_host.py`, `reports/adoption_drift_report.json` diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json new file mode 100644 index 00000000..8adc38bc --- /dev/null +++ b/reports/skill_os2_coverage.json @@ -0,0 +1,811 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "item_count": 21, + "module_count": 8, + "recommended_pr_count": 13, + "pass_count": 21, + "warn_count": 0, + "missing_count": 0, + "extension_track_count": 4, + "extension_partial_count": 0, + "extension_planned_count": 0, + "extension_covered_count": 4, + "adaptive_extension_ready": true, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_evidence_pending_count": 4, + "decision": "local-blueprint-covered-evidence-pending" + }, + "status_counts": { + "pass": 21, + "warn": 0, + "missing": 0 + }, + "extension_status_counts": { + "covered": 4 + }, + "modules": [ + { + "key": "skill-ir", + "category": "core-module", + "label": "Skill IR", + "status": "pass", + "objective": "Platform-neutral capability contract exists before platform-specific packaging.", + "current": "schema 2.0.0; targets 5", + "command": "python3 scripts/yao.py skill-ir .", + "test": "python3 tests/verify_skill_ir.py", + "evidence": [ + { + "path": "skill-ir/schema.json", + "exists": true + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "exists": true + }, + { + "path": "scripts/export_skill_ir.py", + "exists": true + }, + { + "path": "tests/verify_skill_ir.py", + "exists": true + } + ], + "next_action": "Keep all target packages compiled from IR rather than hand-maintained per target." + }, + { + "key": "output-eval-lab", + "category": "core-module", + "label": "Output Eval Lab", + "status": "pass", + "objective": "with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs.", + "current": "5 cases; delta 100.0; execution 10", + "command": "python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .", + "test": "python3 tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "evals/output/schema.json", + "exists": true + }, + { + "path": "evals/output/cases.jsonl", + "exists": true + }, + { + "path": "scripts/run_output_eval.py", + "exists": true + }, + { + "path": "scripts/run_output_execution.py", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Add more real-file and adversarial holdout cases as adoption data grows." + }, + { + "key": "runtime-conformance", + "category": "core-module", + "label": "Runtime Conformance", + "status": "pass", + "objective": "Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets.", + "current": "5/5 targets pass", + "command": "python3 scripts/yao.py conformance .", + "test": "python3 tests/verify_conformance_suite.py", + "evidence": [ + { + "path": "runtime/conformance/schema.json", + "exists": true + }, + { + "path": "scripts/run_conformance_suite.py", + "exists": true + }, + { + "path": "reports/conformance_matrix.json", + "exists": true + }, + { + "path": "tests/verify_conformance_suite.py", + "exists": true + } + ], + "next_action": "Keep conformance cases aligned with current platform metadata rules." + }, + { + "key": "trust-security", + "category": "core-module", + "label": "Trust Security", + "status": "pass", + "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", + "current": "153 scripts; secrets 0; help failures 0", + "command": "python3 scripts/yao.py trust .", + "test": "python3 tests/verify_trust_check.py", + "evidence": [ + { + "path": "scripts/trust_check.py", + "exists": true + }, + { + "path": "security/trust_policy.md", + "exists": true + }, + { + "path": "security/script_policy.md", + "exists": true + }, + { + "path": "security/permission_policy.json", + "exists": true + }, + { + "path": "reports/security_trust_report.json", + "exists": true + }, + { + "path": "tests/verify_trust_check.py", + "exists": true + } + ], + "next_action": "Keep high-permission approvals scoped, expiring, and mapped to target enforcement." + }, + { + "key": "skill-atlas", + "category": "core-module", + "label": "Skill Atlas", + "status": "pass", + "objective": "Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities.", + "current": "12 scanned skills; actionable collisions 0", + "command": "python3 scripts/yao.py skill-atlas --workspace-root .", + "test": "python3 tests/verify_skill_atlas.py", + "evidence": [ + { + "path": "scripts/build_skill_atlas.py", + "exists": true + }, + { + "path": "skill_atlas/catalog.json", + "exists": true + }, + { + "path": "skill_atlas/route_overlap_matrix.csv", + "exists": true + }, + { + "path": "skill_atlas/dependency_graph.json", + "exists": true + }, + { + "path": "reports/skill_atlas.json", + "exists": true + }, + { + "path": "tests/verify_skill_atlas.py", + "exists": true + } + ], + "next_action": "Use real telemetry to rank stale or drifting skills by impact, not only by static metadata." + }, + { + "key": "registry-distribution", + "category": "core-module", + "label": "Registry Distribution", + "status": "pass", + "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", + "current": "archive entries 689; install failures 0", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", + "test": "python3 tests/verify_registry_audit.py", + "evidence": [ + { + "path": "registry/index.schema.json", + "exists": true + }, + { + "path": "registry/package.schema.json", + "exists": true + }, + { + "path": "registry/packages/yao-meta-skill.json", + "exists": true + }, + { + "path": "scripts/registry_audit.py", + "exists": true + }, + { + "path": "reports/package_verification.json", + "exists": true + }, + { + "path": "reports/install_simulation.json", + "exists": true + }, + { + "path": "tests/verify_registry_audit.py", + "exists": true + } + ], + "next_action": "Regenerate registry metadata after package verification so source and archive checksums stay aligned." + }, + { + "key": "review-studio", + "category": "core-module", + "label": "Review Studio", + "status": "pass", + "objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.", + "current": "16 gates; decision review; warnings 3", + "command": "python3 scripts/yao.py review-studio .", + "test": "python3 tests/verify_review_studio.py", + "evidence": [ + { + "path": "scripts/render_review_studio.py", + "exists": true + }, + { + "path": "reports/review-studio.html", + "exists": true + }, + { + "path": "reports/review-studio.json", + "exists": true + }, + { + "path": "tests/verify_review_studio.py", + "exists": true + } + ], + "next_action": "Close pending human and external evidence before claiming full release readiness." + }, + { + "key": "telemetry-drift", + "category": "core-module", + "label": "Telemetry Drift", + "status": "pass", + "objective": "Real usage feedback is captured as metadata-only local-first drift signals.", + "current": "events 1; recipes 5; risk low", + "command": "python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .", + "test": "python3 tests/verify_telemetry_hooks.py", + "evidence": [ + { + "path": "scripts/emit_telemetry_event.py", + "exists": true + }, + { + "path": "scripts/import_telemetry_events.py", + "exists": true + }, + { + "path": "scripts/telemetry_native_host.py", + "exists": true + }, + { + "path": "reports/adoption_drift_report.json", + "exists": true + }, + { + "path": "reports/telemetry_hook_recipes.json", + "exists": true + }, + { + "path": "tests/verify_telemetry_hooks.py", + "exists": true + } + ], + "next_action": "Install a real client and import production metadata-only events into the local drift loop." + } + ], + "recommended_prs": [ + { + "key": "benchmark-methodology", + "category": "recommended-pr", + "label": "Benchmark Methodology", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "25 required artifacts checked", + "command": "make ci-test", + "test": "tests/verify_benchmark_reproducibility.py", + "evidence": [ + { + "path": "reports/benchmark_methodology.md", + "exists": true + }, + { + "path": "reports/benchmark_reproducibility.json", + "exists": true + }, + { + "path": "tests/verify_benchmark_reproducibility.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "output-eval-schema", + "category": "recommended-pr", + "label": "Output Eval Schema", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "5 output cases", + "command": "make ci-test", + "test": "tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "evals/output/schema.json", + "exists": true + }, + { + "path": "evals/output/cases.jsonl", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "output-eval-runner", + "category": "recommended-pr", + "label": "Output Eval Runner", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "delta 100.0", + "command": "make ci-test", + "test": "tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "scripts/run_output_eval.py", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "output-quality-scorecard", + "category": "recommended-pr", + "label": "Output Quality Scorecard", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "gate pass True", + "command": "make ci-test", + "test": "tests/verify_output_eval_lab.py", + "evidence": [ + { + "path": "reports/output_quality_scorecard.md", + "exists": true + }, + { + "path": "reports/output_quality_scorecard.json", + "exists": true + }, + { + "path": "tests/verify_output_eval_lab.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "skill-ir-v0", + "category": "recommended-pr", + "label": "Skill IR V0", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "schema 2.0.0", + "command": "make ci-test", + "test": "tests/verify_skill_ir.py", + "evidence": [ + { + "path": "skill-ir/schema.json", + "exists": true + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "exists": true + }, + { + "path": "tests/verify_skill_ir.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "compiler-refactor", + "category": "recommended-pr", + "label": "Compiler Refactor", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "5/5 compiled targets", + "command": "make ci-test", + "test": "tests/verify_compile_skill.py", + "evidence": [ + { + "path": "scripts/compile_skill.py", + "exists": true + }, + { + "path": "reports/compiled_targets.json", + "exists": true + }, + { + "path": "tests/verify_compile_skill.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "agent-skills-conformance", + "category": "recommended-pr", + "label": "Agent Skills Conformance", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "agent-skills target present", + "command": "make ci-test", + "test": "tests/verify_conformance_suite.py", + "evidence": [ + { + "path": "runtime/conformance/schema.json", + "exists": true + }, + { + "path": "reports/conformance_matrix.json", + "exists": true + }, + { + "path": "tests/verify_conformance_suite.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "trust-check", + "category": "recommended-pr", + "label": "Trust Check", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "secret findings 0", + "command": "make ci-test", + "test": "tests/verify_trust_check.py", + "evidence": [ + { + "path": "scripts/trust_check.py", + "exists": true + }, + { + "path": "reports/security_trust_report.json", + "exists": true + }, + { + "path": "tests/verify_trust_check.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "skill-atlas-generator", + "category": "recommended-pr", + "label": "Skill Atlas Generator", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "12 scanned skills", + "command": "make ci-test", + "test": "tests/verify_skill_atlas.py", + "evidence": [ + { + "path": "scripts/build_skill_atlas.py", + "exists": true + }, + { + "path": "reports/skill_atlas.json", + "exists": true + }, + { + "path": "tests/verify_skill_atlas.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "registry-package-format", + "category": "recommended-pr", + "label": "Registry Package Format", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "registry ok True", + "command": "make ci-test", + "test": "tests/verify_registry_audit.py", + "evidence": [ + { + "path": "registry/package.schema.json", + "exists": true + }, + { + "path": "reports/registry_audit.json", + "exists": true + }, + { + "path": "tests/verify_registry_audit.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "review-studio-2", + "category": "recommended-pr", + "label": "Review Studio 2.0", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "16 review gates", + "command": "make ci-test", + "test": "tests/verify_review_studio.py", + "evidence": [ + { + "path": "scripts/render_review_studio.py", + "exists": true + }, + { + "path": "reports/review-studio.html", + "exists": true + }, + { + "path": "tests/verify_review_studio.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "migration-v2-docs", + "category": "recommended-pr", + "label": "Migration V2 Docs", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "migration guide present", + "command": "make ci-test", + "test": "docs review", + "evidence": [ + { + "path": "docs/migration-v2.md", + "exists": true + }, + { + "path": "reports/skill-os-2-review.md", + "exists": true + }, + { + "path": "README.md", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + }, + { + "key": "evidence-consistency", + "category": "recommended-pr", + "label": "Evidence Consistency", + "status": "pass", + "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + "current": "39 consistency checks", + "command": "make ci-test", + "test": "tests/verify_evidence_consistency.py", + "evidence": [ + { + "path": "scripts/render_evidence_consistency.py", + "exists": true + }, + { + "path": "reports/evidence_consistency.json", + "exists": true + }, + { + "path": "tests/verify_evidence_consistency.py", + "exists": true + } + ], + "next_action": "Keep this item covered as the implementation evolves." + } + ], + "reference_extension_tracks": [ + { + "key": "skill-interpretation-report", + "label": "Skill Interpretation Report", + "status": "covered", + "objective": "User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations.", + "current": "Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests.", + "target": "Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests.", + "evidence": [ + { + "path": "reports/skill-overview.html", + "exists": true + }, + { + "path": "reports/skill-overview.json", + "exists": true + }, + { + "path": "scripts/render_skill_overview.py", + "exists": true + }, + { + "path": "scripts/render_skill_interpretation.py", + "exists": true + }, + { + "path": "schemas/skill-interpretation.schema.json", + "exists": true + }, + { + "path": "tests/verify_skill_interpretation.py", + "exists": true + } + ], + "next_action": "Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change." + }, + { + "key": "adaptive-self-iteration", + "label": "Adaptive Self-Iteration", + "status": "covered", + "objective": "Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback.", + "current": "Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata.", + "target": "Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan.", + "evidence": [ + { + "path": "references/autonomous-adaptation.md", + "exists": true + }, + { + "path": "references/user-memory-policy.md", + "exists": true + }, + { + "path": "schemas/adaptation-proposal.schema.json", + "exists": true + }, + { + "path": "scripts/summarize_user_signals.py", + "exists": true + }, + { + "path": "scripts/propose_adaptation.py", + "exists": true + }, + { + "path": "tests/verify_adaptation_safety.py", + "exists": true + }, + { + "path": "scripts/apply_adaptation.py", + "exists": true + }, + { + "path": "reports/adaptation_approval_ledger.json", + "exists": true + }, + { + "path": "reports/adaptation_regression_report.json", + "exists": true + }, + { + "path": "reports/user_patterns.json", + "exists": true + }, + { + "path": "reports/adaptation_proposals.json", + "exists": true + }, + { + "path": "reports/iteration-directions.md", + "exists": true + }, + { + "path": "reports/adoption_drift_report.md", + "exists": true + } + ], + "next_action": "Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path." + }, + { + "key": "daily-skillops-report", + "label": "Daily SkillOps Report", + "status": "covered", + "objective": "Daily operations layer summarizes explicit-source conversation patterns, proposal-only adaptation work, approval state, release locks, and world-class evidence gaps.", + "current": "Daily SkillOps report is a CLI-backed, explicit-source operations cockpit with tests and committed dated evidence.", + "target": "Generate reports/skillops/daily/YYYY-MM-DD.* from explicit sources without private log scanning, source writes, auto-patching, or world-class overclaiming.", + "evidence": [ + { + "path": "scripts/render_daily_skillops_report.py", + "exists": true + }, + { + "path": "tests/verify_daily_skillops.py", + "exists": true + }, + { + "path": "reports/skillops/daily/2026-06-17.json", + "exists": true + }, + { + "path": "reports/skillops/daily/2026-06-17.md", + "exists": true + } + ], + "next_action": "Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves." + }, + { + "key": "weekly-curator-report", + "label": "Weekly Curator Report", + "status": "covered", + "objective": "Weekly curator layer aggregates Daily SkillOps opportunities, Skill Atlas portfolio signals, release locks, and world-class evidence gaps into a maintenance queue.", + "current": "Weekly curator report is CLI-backed, proposal-only, and backed by committed weekly JSON/Markdown evidence.", + "target": "Generate reports/skillops/weekly/YYYY-WNN.* without private log scanning, source writes, auto-patching, or world-class overclaiming.", + "evidence": [ + { + "path": "scripts/render_weekly_curator_report.py", + "exists": true + }, + { + "path": "tests/verify_weekly_curator.py", + "exists": true + }, + { + "path": "reports/skillops/weekly/2026-W25.json", + "exists": true + }, + { + "path": "reports/skillops/weekly/2026-W25.md", + "exists": true + } + ], + "next_action": "Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes." + } + ], + "next_highest_leverage": [ + "Close the four world-class evidence ledger entries with accepted human or external evidence.", + "Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves.", + "Use Daily SkillOps with explicit curated sources to review adaptation proposals without scanning private logs or auto-patching.", + "Keep the blueprint coverage report in CI so 2.0 plan drift is visible." + ], + "source_blueprint": { + "title": "Skill Overview / Skill OS 2.0 upgrade plan", + "core_module_count": 8, + "recommended_pr_count": 13, + "reference_extension_count": 4, + "reference_extensions": [ + "Skill interpretation report", + "Adaptive self-iteration with local preference memory and approval gates", + "Daily SkillOps operations report" + ] + }, + "artifacts": { + "json": "reports/skill_os2_coverage.json", + "markdown": "reports/skill_os2_coverage.md" + } +} diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md new file mode 100644 index 00000000..e857c756 --- /dev/null +++ b/reports/skill_os2_coverage.md @@ -0,0 +1,246 @@ +# Skill OS 2.0 Blueprint Coverage + +Generated at: `2026-06-17` + +## Summary + +- decision: `local-blueprint-covered-evidence-pending` +- local blueprint ready: `true` +- public world-class ready: `false` +- pass: `21` / `21` +- missing: `0` +- warn: `0` +- reference extensions: `4` +- extension covered: `4` +- extension partial: `0` +- extension planned: `0` +- adaptive extension ready: `true` +- world-class evidence pending: `4` + +This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, commands, and tests. It does not count pending human review, provider runs, metadata fallbacks, or planned work as public world-class evidence. + +## Core Modules + +| Item | Status | Current | Command | Test | +| --- | --- | --- | --- | --- | +| Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | +| Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | +| Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | +| Trust Security | `pass` | 153 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | +| Registry Distribution | `pass` | archive entries 689; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | +| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | + +## Recommended PR Coverage + +| Item | Status | Current | Command | Test | +| --- | --- | --- | --- | --- | +| Benchmark Methodology | `pass` | 25 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` | +| Output Eval Schema | `pass` | 5 output cases | `make ci-test` | `tests/verify_output_eval_lab.py` | +| Output Eval Runner | `pass` | delta 100.0 | `make ci-test` | `tests/verify_output_eval_lab.py` | +| Output Quality Scorecard | `pass` | gate pass True | `make ci-test` | `tests/verify_output_eval_lab.py` | +| Skill IR V0 | `pass` | schema 2.0.0 | `make ci-test` | `tests/verify_skill_ir.py` | +| Compiler Refactor | `pass` | 5/5 compiled targets | `make ci-test` | `tests/verify_compile_skill.py` | +| Agent Skills Conformance | `pass` | agent-skills target present | `make ci-test` | `tests/verify_conformance_suite.py` | +| Trust Check | `pass` | secret findings 0 | `make ci-test` | `tests/verify_trust_check.py` | +| Skill Atlas Generator | `pass` | 12 scanned skills | `make ci-test` | `tests/verify_skill_atlas.py` | +| Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` | +| Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` | +| Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` | +| Evidence Consistency | `pass` | 39 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | + +## Reference Extension Tracks + +| Track | Status | Current | Target | Next action | +| --- | --- | --- | --- | --- | +| Skill Interpretation Report | `covered` | Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests. | Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests. | Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change. | +| Adaptive Self-Iteration | `covered` | Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata. | Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan. | Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path. | +| Daily SkillOps Report | `covered` | Daily SkillOps report is a CLI-backed, explicit-source operations cockpit with tests and committed dated evidence. | Generate reports/skillops/daily/YYYY-MM-DD.* from explicit sources without private log scanning, source writes, auto-patching, or world-class overclaiming. | Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves. | +| Weekly Curator Report | `covered` | Weekly curator report is CLI-backed, proposal-only, and backed by committed weekly JSON/Markdown evidence. | Generate reports/skillops/weekly/YYYY-WNN.* without private log scanning, source writes, auto-patching, or world-class overclaiming. | Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes. | + +These extension tracks come from the user-supplied 2.0 reference plan. They are tracked separately from the formal Skill OS blueprint so the report can distinguish landed local architecture from planned explainer/adaptor evolution. + +## Next Highest-Leverage Moves + +- Close the four world-class evidence ledger entries with accepted human or external evidence. +- Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves. +- Use Daily SkillOps with explicit curated sources to review adaptation proposals without scanning private logs or auto-patching. +- Keep the blueprint coverage report in CI so 2.0 plan drift is visible. + +## Evidence Detail + +### Skill IR + +- objective: Platform-neutral capability contract exists before platform-specific packaging. +- status: `pass` +- existing evidence: `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py`, `tests/verify_skill_ir.py` +- next action: Keep all target packages compiled from IR rather than hand-maintained per target. + +### Output Eval Lab + +- objective: with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs. +- status: `pass` +- existing evidence: `evals/output/schema.json`, `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `reports/output_quality_scorecard.json`, `tests/verify_output_eval_lab.py` +- next action: Add more real-file and adversarial holdout cases as adoption data grows. + +### Runtime Conformance + +- objective: Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets. +- status: `pass` +- existing evidence: `runtime/conformance/schema.json`, `scripts/run_conformance_suite.py`, `reports/conformance_matrix.json`, `tests/verify_conformance_suite.py` +- next action: Keep conformance cases aligned with current platform metadata rules. + +### Trust Security + +- objective: Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution. +- status: `pass` +- existing evidence: `scripts/trust_check.py`, `security/trust_policy.md`, `security/script_policy.md`, `security/permission_policy.json`, `reports/security_trust_report.json`, `tests/verify_trust_check.py` +- next action: Keep high-permission approvals scoped, expiring, and mapped to target enforcement. + +### Skill Atlas + +- objective: Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities. +- status: `pass` +- existing evidence: `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `skill_atlas/dependency_graph.json`, `reports/skill_atlas.json`, `tests/verify_skill_atlas.py` +- next action: Use real telemetry to rank stale or drifting skills by impact, not only by static metadata. + +### Registry Distribution + +- objective: Skill packages are installable, versioned, checksumed, and upgrade-reviewable. +- status: `pass` +- existing evidence: `registry/index.schema.json`, `registry/package.schema.json`, `registry/packages/yao-meta-skill.json`, `scripts/registry_audit.py`, `reports/package_verification.json`, `reports/install_simulation.json`, `tests/verify_registry_audit.py` +- next action: Regenerate registry metadata after package verification so source and archive checksums stay aligned. + +### Review Studio + +- objective: One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions. +- status: `pass` +- existing evidence: `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json`, `tests/verify_review_studio.py` +- next action: Close pending human and external evidence before claiming full release readiness. + +### Telemetry Drift + +- objective: Real usage feedback is captured as metadata-only local-first drift signals. +- status: `pass` +- existing evidence: `scripts/emit_telemetry_event.py`, `scripts/import_telemetry_events.py`, `scripts/telemetry_native_host.py`, `reports/adoption_drift_report.json`, `reports/telemetry_hook_recipes.json`, `tests/verify_telemetry_hooks.py` +- next action: Install a real client and import production metadata-only events into the local drift loop. + +### Benchmark Methodology + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `reports/benchmark_methodology.md`, `reports/benchmark_reproducibility.json`, `tests/verify_benchmark_reproducibility.py` +- next action: Keep this item covered as the implementation evolves. + +### Output Eval Schema + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `evals/output/schema.json`, `evals/output/cases.jsonl`, `tests/verify_output_eval_lab.py` +- next action: Keep this item covered as the implementation evolves. + +### Output Eval Runner + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/run_output_eval.py`, `reports/output_quality_scorecard.json`, `tests/verify_output_eval_lab.py` +- next action: Keep this item covered as the implementation evolves. + +### Output Quality Scorecard + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `reports/output_quality_scorecard.md`, `reports/output_quality_scorecard.json`, `tests/verify_output_eval_lab.py` +- next action: Keep this item covered as the implementation evolves. + +### Skill IR V0 + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `tests/verify_skill_ir.py` +- next action: Keep this item covered as the implementation evolves. + +### Compiler Refactor + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/compile_skill.py`, `reports/compiled_targets.json`, `tests/verify_compile_skill.py` +- next action: Keep this item covered as the implementation evolves. + +### Agent Skills Conformance + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `runtime/conformance/schema.json`, `reports/conformance_matrix.json`, `tests/verify_conformance_suite.py` +- next action: Keep this item covered as the implementation evolves. + +### Trust Check + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/trust_check.py`, `reports/security_trust_report.json`, `tests/verify_trust_check.py` +- next action: Keep this item covered as the implementation evolves. + +### Skill Atlas Generator + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/build_skill_atlas.py`, `reports/skill_atlas.json`, `tests/verify_skill_atlas.py` +- next action: Keep this item covered as the implementation evolves. + +### Registry Package Format + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `registry/package.schema.json`, `reports/registry_audit.json`, `tests/verify_registry_audit.py` +- next action: Keep this item covered as the implementation evolves. + +### Review Studio 2.0 + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/render_review_studio.py`, `reports/review-studio.html`, `tests/verify_review_studio.py` +- next action: Keep this item covered as the implementation evolves. + +### Migration V2 Docs + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `docs/migration-v2.md`, `reports/skill-os-2-review.md`, `README.md` +- next action: Keep this item covered as the implementation evolves. + +### Evidence Consistency + +- objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. +- status: `pass` +- existing evidence: `scripts/render_evidence_consistency.py`, `reports/evidence_consistency.json`, `tests/verify_evidence_consistency.py` +- next action: Keep this item covered as the implementation evolves. + +### Skill Interpretation Report + +- objective: User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations. +- status: `covered` +- existing evidence: `reports/skill-overview.html`, `reports/skill-overview.json`, `scripts/render_skill_overview.py`, `scripts/render_skill_interpretation.py`, `schemas/skill-interpretation.schema.json`, `tests/verify_skill_interpretation.py` +- next action: Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change. + +### Adaptive Self-Iteration + +- objective: Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback. +- status: `covered` +- existing evidence: `references/autonomous-adaptation.md`, `references/user-memory-policy.md`, `schemas/adaptation-proposal.schema.json`, `scripts/summarize_user_signals.py`, `scripts/propose_adaptation.py`, `tests/verify_adaptation_safety.py`, `scripts/apply_adaptation.py`, `reports/adaptation_approval_ledger.json`, `reports/adaptation_regression_report.json`, `reports/user_patterns.json`, `reports/adaptation_proposals.json`, `reports/iteration-directions.md`, `reports/adoption_drift_report.md` +- next action: Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path. + +### Daily SkillOps Report + +- objective: Daily operations layer summarizes explicit-source conversation patterns, proposal-only adaptation work, approval state, release locks, and world-class evidence gaps. +- status: `covered` +- existing evidence: `scripts/render_daily_skillops_report.py`, `tests/verify_daily_skillops.py`, `reports/skillops/daily/2026-06-17.json`, `reports/skillops/daily/2026-06-17.md` +- next action: Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves. + +### Weekly Curator Report + +- objective: Weekly curator layer aggregates Daily SkillOps opportunities, Skill Atlas portfolio signals, release locks, and world-class evidence gaps into a maintenance queue. +- status: `covered` +- existing evidence: `scripts/render_weekly_curator_report.py`, `tests/verify_weekly_curator.py`, `reports/skillops/weekly/2026-W25.json`, `reports/skillops/weekly/2026-W25.md` +- next action: Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes. diff --git a/reports/skillops/daily/2026-06-16.json b/reports/skillops/daily/2026-06-16.json new file mode 100644 index 00000000..331b7837 --- /dev/null +++ b/reports/skillops/daily/2026-06-16.json @@ -0,0 +1,513 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-16T21:24:27Z", + "skill_dir": ".", + "decision": "proposal-review", + "source_supplied": false, + "pattern_count": 5, + "proposal_count": 5, + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0, + "summary": { + "decision": "proposal-review", + "source_supplied": false, + "pattern_count": 5, + "proposal_count": 5, + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0 + }, + "operations_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "explicit_source_required_for_scan": true, + "implicit_private_log_scan": false, + "raw_content_stored": false, + "redacted_excerpts_only": true, + "proposal_only": true, + "approval_required_for_writes": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "daily_report_counts_as_world_class_evidence": false + }, + "report_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "source_supplied", + "pattern_count", + "proposal_count", + "approval_count", + "pending_review_count", + "applied_count", + "rollback_count", + "local_blueprint_ready", + "public_world_class_ready", + "world_class_pending_count", + "release_lock_ready", + "evidence_consistency_ok", + "writes_source_files", + "auto_patch_enabled", + "failure_count" + ], + "source_of_truth": [ + "summary", + "operations_contract" + ] + }, + "source": { + "path": "evals/adaptation/user_signals.example.jsonl", + "fingerprint_sha256": "48f94a4f3b82f90db29921c13d09c765ec10aabfde1adce65710c137cc6c5abf", + "record_count": 8, + "explicit_source": true, + "raw_content_stored": false, + "redacted_excerpts_only": true + }, + "patterns": [ + { + "pattern_id": "language_default", + "label": "Default language preference", + "signal_type": "report-language", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated report-language signals.", + "recommended_action": "Keep generated reports Chinese-first with an English switch where user-facing.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + } + ] + }, + { + "pattern_id": "report_ui", + "label": "Report UI and visualization preference", + "signal_type": "artifact-design", + "support_count": 5, + "confidence": 0.95, + "reason": "5 redacted records matched repeated artifact-design signals.", + "recommended_action": "Prioritize white-background Kami-style reports with readable charts and stable navigation.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-3", + "excerpt": "报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。" + } + ] + }, + { + "pattern_id": "approval_safety", + "label": "Approval and privacy boundary", + "signal_type": "governance", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated governance signals.", + "recommended_action": "Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path.", + "evidence": [ + { + "record_id": "line-5", + "excerpt": "自适应升级必须先生成提案，不能直接自动修改源文件。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "delivery_format", + "label": "Delivery format preference", + "signal_type": "artifact-format", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated artifact-format signals.", + "recommended_action": "Surface stable artifact paths and formats in CLI output and generated summaries.", + "evidence": [ + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "evidence_testing", + "label": "Evidence and testing preference", + "signal_type": "quality-gate", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated quality-gate signals.", + "recommended_action": "Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade.", + "evidence": [ + { + "record_id": "line-7", + "excerpt": "每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。" + }, + { + "record_id": "line-8", + "excerpt": "涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。" + } + ] + } + ], + "proposals": [ + { + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ] + }, + { + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "risk_level": "medium", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ] + }, + { + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ] + }, + { + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ] + }, + { + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "risk_level": "medium", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ] + } + ], + "opportunity_summary": { + "opportunity_count": 5, + "top_score": 88, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "action_type_counts": { + "patch_existing_skill": 3, + "agents_update": 1, + "add_eval": 1 + }, + "decision_counts": { + "ready_for_approval_review": 1, + "proposal_review": 4 + } + }, + "opportunities": [ + { + "opportunity_id": "skillops-4ad91c18dd", + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "opportunity_type": "artifact-quality", + "action_type": "patch_existing_skill", + "decision": "ready_for_approval_review", + "priority": "medium", + "score": 88, + "score_reasons": [ + "support_count=5", + "confidence=0.95", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ] + }, + { + "opportunity_id": "skillops-531e13c1c6", + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "opportunity_type": "governance", + "action_type": "agents_update", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated privacy and approval signals should tighten durable operating guidance.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ] + }, + { + "opportunity_id": "skillops-709ef884f3", + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "opportunity_type": "report-experience", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated language preferences can improve existing report templates after approval.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ] + }, + { + "opportunity_id": "skillops-f9e43237aa", + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "opportunity_type": "artifact-discoverability", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ] + }, + { + "opportunity_id": "skillops-0a3a2de7c3", + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "opportunity_type": "quality-gate", + "action_type": "add_eval", + "decision": "proposal_review", + "priority": "medium", + "score": 74, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated verification and evidence requests should become focused checks.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ] + } + ], + "decision_policy": { + "schema_version": "1.0", + "contract": "skillops-opportunity-scoring", + "score_range": [ + 0, + 100 + ], + "score_bands": { + "85-100": "ready_for_approval_review", + "70-84": "proposal_review", + "50-69": "observe_more_evidence", + "0-49": "report_only_or_no_action" + }, + "action_types": [ + "add_eval", + "agents_update", + "patch_existing_skill", + "report_only" + ], + "writes_source_files": false, + "auto_patch_enabled": false, + "approval_required_for_writes": true + }, + "approval": { + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0 + }, + "release_state": { + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true + }, + "actions": [ + { + "key": "review-adaptation-proposals", + "priority": "high", + "action": "Review proposal-only adaptation items before preparing any approval ledger entry." + }, + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for the pending world-class ledger entries." + } + ], + "failures": [], + "source_reports": { + "patterns": "reports/user_patterns.json", + "proposals": "reports/adaptation_proposals.json", + "approval_ledger": "reports/adaptation_approval_ledger.json", + "regression": "reports/adaptation_regression_report.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + "evidence_consistency": "reports/evidence_consistency.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json" + }, + "artifacts": { + "json": "reports/skillops/daily/2026-06-16.json", + "markdown": "reports/skillops/daily/2026-06-16.md" + } +} diff --git a/reports/skillops/daily/2026-06-16.md b/reports/skillops/daily/2026-06-16.md new file mode 100644 index 00000000..5d1c83f2 --- /dev/null +++ b/reports/skillops/daily/2026-06-16.md @@ -0,0 +1,174 @@ +# Daily SkillOps Report + +Generated at: `2026-06-16T21:24:27Z` + +## Summary + +- decision: `proposal-review` +- source supplied: `false` +- patterns: `5` +- proposals: `5` +- pending approvals: `0` +- applied patches: `0` +- rollbacks: `0` +- local blueprint ready: `true` +- public world-class ready: `false` +- world-class pending: `4` +- release lock ready: `false` +- evidence consistency ok: `true` + +This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. + +## Privacy Boundary + +- explicit_source_required_for_scan: `true` +- implicit_private_log_scan: `false` +- raw_content_stored: `false` +- redacted_excerpts_only: `true` +- proposal_only: `true` +- approval_required_for_writes: `true` +- writes_source_files: `false` +- auto_patch_enabled: `false` +- daily_report_counts_as_world_class_evidence: `false` + +## Actions + +- `high` Review proposal-only adaptation items before preparing any approval ledger entry. +- `high` Collect accepted external or human evidence for the pending world-class ledger entries. + +## Opportunities + +- count: `5` +- top score: `88` +- ready for approval review: `1` +### Improve report layout, visual hierarchy, and chart readability + +- ID: `skillops-4ad91c18dd` +- Action: `patch_existing_skill` +- Decision: `ready_for_approval_review` +- Score: `88` +- Risk: `medium` +- Policy: Repeated report layout feedback maps to existing renderer and design doctrine changes. +### Keep adaptive iteration approval-gated + +- ID: `skillops-531e13c1c6` +- Action: `agents_update` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Policy: Repeated privacy and approval signals should tighten durable operating guidance. +### Keep reports Chinese-first with optional English + +- ID: `skillops-709ef884f3` +- Action: `patch_existing_skill` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Policy: Repeated language preferences can improve existing report templates after approval. +### Make generated artifact paths explicit in CLI output + +- ID: `skillops-f9e43237aa` +- Action: `patch_existing_skill` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Policy: Repeated delivery-path requests can improve CLI and README artifact discoverability. +### Attach tests and evidence refresh to each upgrade + +- ID: `skillops-0a3a2de7c3` +- Action: `add_eval` +- Decision: `proposal_review` +- Score: `74` +- Risk: `medium` +- Policy: Repeated verification and evidence requests should become focused checks. + +## Patterns + +- `language_default`: Default language preference (support `2`, confidence `0.79`) +- `report_ui`: Report UI and visualization preference (support `5`, confidence `0.95`) +- `approval_safety`: Approval and privacy boundary (support `2`, confidence `0.79`) +- `delivery_format`: Delivery format preference (support `2`, confidence `0.79`) +- `evidence_testing`: Evidence and testing preference (support `2`, confidence `0.79`) + +## Proposals + +### Keep reports Chinese-first with optional English + +- ID: `adapt-18c7517f3d` +- Pattern: `language_default` +- Status: `proposal-only` +- Risk: `low` +- Requires approval: `true` +- Target files: + - `scripts/render_skill_overview.py` + - `references/artifact-design-doctrine.md` +- Verification: + - `python3 tests/verify_skill_overview.py` + +### Improve report layout, visual hierarchy, and chart readability + +- ID: `adapt-fbfe921ba5` +- Pattern: `report_ui` +- Status: `proposal-only` +- Risk: `medium` +- Requires approval: `true` +- Target files: + - `scripts/render_skill_overview.py` + - `references/artifact-design-doctrine.md` + - `tests/verify_skill_overview.py` +- Verification: + - `python3 tests/verify_skill_overview.py` + - `python3 tests/verify_skill_report_charts.py` + +### Keep adaptive iteration approval-gated + +- ID: `adapt-59d219a1fb` +- Pattern: `approval_safety` +- Status: `proposal-only` +- Risk: `low` +- Requires approval: `true` +- Target files: + - `references/user-memory-policy.md` + - `references/autonomous-adaptation.md` + - `schemas/adaptation-proposal.schema.json` +- Verification: + - `python3 tests/verify_adaptation_safety.py` + +### Make generated artifact paths explicit in CLI output + +- ID: `adapt-457baca160` +- Pattern: `delivery_format` +- Status: `proposal-only` +- Risk: `low` +- Requires approval: `true` +- Target files: + - `scripts/yao.py` + - `README.md` +- Verification: + - `python3 tests/verify_yao_cli.py` + +### Attach tests and evidence refresh to each upgrade + +- ID: `adapt-abfee25d3a` +- Pattern: `evidence_testing` +- Status: `proposal-only` +- Risk: `medium` +- Requires approval: `true` +- Target files: + - `tests/verify_adaptation_safety.py` + - `scripts/render_skill_os2_coverage.py` + - `reports/skill_os2_coverage.json` +- Verification: + - `python3 tests/verify_adaptation_safety.py` + - `python3 tests/verify_skill_os2_coverage.py` + +## Evidence + +- patterns: `reports/user_patterns.json` +- proposals: `reports/adaptation_proposals.json` +- approval_ledger: `reports/adaptation_approval_ledger.json` +- regression: `reports/adaptation_regression_report.json` +- skill_os2_coverage: `reports/skill_os2_coverage.json` +- world_class_ledger: `reports/world_class_evidence_ledger.json` +- evidence_consistency: `reports/evidence_consistency.json` +- benchmark_reproducibility: `reports/benchmark_reproducibility.json` diff --git a/reports/skillops/daily/2026-06-17.json b/reports/skillops/daily/2026-06-17.json new file mode 100644 index 00000000..0e4493b7 --- /dev/null +++ b/reports/skillops/daily/2026-06-17.json @@ -0,0 +1,513 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17T08:58:54Z", + "skill_dir": ".", + "decision": "proposal-review", + "source_supplied": false, + "pattern_count": 5, + "proposal_count": 5, + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0, + "summary": { + "decision": "proposal-review", + "source_supplied": false, + "pattern_count": 5, + "proposal_count": 5, + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0 + }, + "operations_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "explicit_source_required_for_scan": true, + "implicit_private_log_scan": false, + "raw_content_stored": false, + "redacted_excerpts_only": true, + "proposal_only": true, + "approval_required_for_writes": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "daily_report_counts_as_world_class_evidence": false + }, + "report_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "source_supplied", + "pattern_count", + "proposal_count", + "approval_count", + "pending_review_count", + "applied_count", + "rollback_count", + "local_blueprint_ready", + "public_world_class_ready", + "world_class_pending_count", + "release_lock_ready", + "evidence_consistency_ok", + "writes_source_files", + "auto_patch_enabled", + "failure_count" + ], + "source_of_truth": [ + "summary", + "operations_contract" + ] + }, + "source": { + "path": "evals/adaptation/user_signals.example.jsonl", + "fingerprint_sha256": "48f94a4f3b82f90db29921c13d09c765ec10aabfde1adce65710c137cc6c5abf", + "record_count": 8, + "explicit_source": true, + "raw_content_stored": false, + "redacted_excerpts_only": true + }, + "patterns": [ + { + "pattern_id": "language_default", + "label": "Default language preference", + "signal_type": "report-language", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated report-language signals.", + "recommended_action": "Keep generated reports Chinese-first with an English switch where user-facing.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + } + ] + }, + { + "pattern_id": "report_ui", + "label": "Report UI and visualization preference", + "signal_type": "artifact-design", + "support_count": 5, + "confidence": 0.95, + "reason": "5 redacted records matched repeated artifact-design signals.", + "recommended_action": "Prioritize white-background Kami-style reports with readable charts and stable navigation.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-3", + "excerpt": "报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。" + } + ] + }, + { + "pattern_id": "approval_safety", + "label": "Approval and privacy boundary", + "signal_type": "governance", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated governance signals.", + "recommended_action": "Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path.", + "evidence": [ + { + "record_id": "line-5", + "excerpt": "自适应升级必须先生成提案，不能直接自动修改源文件。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "delivery_format", + "label": "Delivery format preference", + "signal_type": "artifact-format", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated artifact-format signals.", + "recommended_action": "Surface stable artifact paths and formats in CLI output and generated summaries.", + "evidence": [ + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "evidence_testing", + "label": "Evidence and testing preference", + "signal_type": "quality-gate", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated quality-gate signals.", + "recommended_action": "Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade.", + "evidence": [ + { + "record_id": "line-7", + "excerpt": "每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。" + }, + { + "record_id": "line-8", + "excerpt": "涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。" + } + ] + } + ], + "proposals": [ + { + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ] + }, + { + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "risk_level": "medium", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ] + }, + { + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ] + }, + { + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "risk_level": "low", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ] + }, + { + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "risk_level": "medium", + "status": "proposal-only", + "requires_approval": true, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ] + } + ], + "opportunity_summary": { + "opportunity_count": 5, + "top_score": 88, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "action_type_counts": { + "patch_existing_skill": 3, + "agents_update": 1, + "add_eval": 1 + }, + "decision_counts": { + "ready_for_approval_review": 1, + "proposal_review": 4 + } + }, + "opportunities": [ + { + "opportunity_id": "skillops-4ad91c18dd", + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "opportunity_type": "artifact-quality", + "action_type": "patch_existing_skill", + "decision": "ready_for_approval_review", + "priority": "medium", + "score": 88, + "score_reasons": [ + "support_count=5", + "confidence=0.95", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ] + }, + { + "opportunity_id": "skillops-531e13c1c6", + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "opportunity_type": "governance", + "action_type": "agents_update", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated privacy and approval signals should tighten durable operating guidance.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ] + }, + { + "opportunity_id": "skillops-709ef884f3", + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "opportunity_type": "report-experience", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated language preferences can improve existing report templates after approval.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ] + }, + { + "opportunity_id": "skillops-f9e43237aa", + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "opportunity_type": "artifact-discoverability", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ] + }, + { + "opportunity_id": "skillops-0a3a2de7c3", + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "opportunity_type": "quality-gate", + "action_type": "add_eval", + "decision": "proposal_review", + "priority": "medium", + "score": 74, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated verification and evidence requests should become focused checks.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ] + } + ], + "decision_policy": { + "schema_version": "1.0", + "contract": "skillops-opportunity-scoring", + "score_range": [ + 0, + 100 + ], + "score_bands": { + "85-100": "ready_for_approval_review", + "70-84": "proposal_review", + "50-69": "observe_more_evidence", + "0-49": "report_only_or_no_action" + }, + "action_types": [ + "add_eval", + "agents_update", + "patch_existing_skill", + "report_only" + ], + "writes_source_files": false, + "auto_patch_enabled": false, + "approval_required_for_writes": true + }, + "approval": { + "approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0 + }, + "release_state": { + "local_blueprint_ready": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "release_lock_ready": false, + "evidence_consistency_ok": true + }, + "actions": [ + { + "key": "review-adaptation-proposals", + "priority": "high", + "action": "Review proposal-only adaptation items before preparing any approval ledger entry." + }, + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for the pending world-class ledger entries." + } + ], + "failures": [], + "source_reports": { + "patterns": "reports/user_patterns.json", + "proposals": "reports/adaptation_proposals.json", + "approval_ledger": "reports/adaptation_approval_ledger.json", + "regression": "reports/adaptation_regression_report.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + "evidence_consistency": "reports/evidence_consistency.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json" + }, + "artifacts": { + "json": "reports/skillops/daily/2026-06-17.json", + "markdown": "reports/skillops/daily/2026-06-17.md" + } +} diff --git a/reports/skillops/daily/2026-06-17.md b/reports/skillops/daily/2026-06-17.md new file mode 100644 index 00000000..628fd7c9 --- /dev/null +++ b/reports/skillops/daily/2026-06-17.md @@ -0,0 +1,174 @@ +# Daily SkillOps Report + +Generated at: `2026-06-17T08:58:54Z` + +## Summary + +- decision: `proposal-review` +- source supplied: `false` +- patterns: `5` +- proposals: `5` +- pending approvals: `0` +- applied patches: `0` +- rollbacks: `0` +- local blueprint ready: `true` +- public world-class ready: `false` +- world-class pending: `4` +- release lock ready: `false` +- evidence consistency ok: `true` + +This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. + +## Privacy Boundary + +- explicit_source_required_for_scan: `true` +- implicit_private_log_scan: `false` +- raw_content_stored: `false` +- redacted_excerpts_only: `true` +- proposal_only: `true` +- approval_required_for_writes: `true` +- writes_source_files: `false` +- auto_patch_enabled: `false` +- daily_report_counts_as_world_class_evidence: `false` + +## Actions + +- `high` Review proposal-only adaptation items before preparing any approval ledger entry. +- `high` Collect accepted external or human evidence for the pending world-class ledger entries. + +## Opportunities + +- count: `5` +- top score: `88` +- ready for approval review: `1` +### Improve report layout, visual hierarchy, and chart readability + +- ID: `skillops-4ad91c18dd` +- Action: `patch_existing_skill` +- Decision: `ready_for_approval_review` +- Score: `88` +- Risk: `medium` +- Policy: Repeated report layout feedback maps to existing renderer and design doctrine changes. +### Keep adaptive iteration approval-gated + +- ID: `skillops-531e13c1c6` +- Action: `agents_update` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Policy: Repeated privacy and approval signals should tighten durable operating guidance. +### Keep reports Chinese-first with optional English + +- ID: `skillops-709ef884f3` +- Action: `patch_existing_skill` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Policy: Repeated language preferences can improve existing report templates after approval. +### Make generated artifact paths explicit in CLI output + +- ID: `skillops-f9e43237aa` +- Action: `patch_existing_skill` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Policy: Repeated delivery-path requests can improve CLI and README artifact discoverability. +### Attach tests and evidence refresh to each upgrade + +- ID: `skillops-0a3a2de7c3` +- Action: `add_eval` +- Decision: `proposal_review` +- Score: `74` +- Risk: `medium` +- Policy: Repeated verification and evidence requests should become focused checks. + +## Patterns + +- `language_default`: Default language preference (support `2`, confidence `0.79`) +- `report_ui`: Report UI and visualization preference (support `5`, confidence `0.95`) +- `approval_safety`: Approval and privacy boundary (support `2`, confidence `0.79`) +- `delivery_format`: Delivery format preference (support `2`, confidence `0.79`) +- `evidence_testing`: Evidence and testing preference (support `2`, confidence `0.79`) + +## Proposals + +### Keep reports Chinese-first with optional English + +- ID: `adapt-18c7517f3d` +- Pattern: `language_default` +- Status: `proposal-only` +- Risk: `low` +- Requires approval: `true` +- Target files: + - `scripts/render_skill_overview.py` + - `references/artifact-design-doctrine.md` +- Verification: + - `python3 tests/verify_skill_overview.py` + +### Improve report layout, visual hierarchy, and chart readability + +- ID: `adapt-fbfe921ba5` +- Pattern: `report_ui` +- Status: `proposal-only` +- Risk: `medium` +- Requires approval: `true` +- Target files: + - `scripts/render_skill_overview.py` + - `references/artifact-design-doctrine.md` + - `tests/verify_skill_overview.py` +- Verification: + - `python3 tests/verify_skill_overview.py` + - `python3 tests/verify_skill_report_charts.py` + +### Keep adaptive iteration approval-gated + +- ID: `adapt-59d219a1fb` +- Pattern: `approval_safety` +- Status: `proposal-only` +- Risk: `low` +- Requires approval: `true` +- Target files: + - `references/user-memory-policy.md` + - `references/autonomous-adaptation.md` + - `schemas/adaptation-proposal.schema.json` +- Verification: + - `python3 tests/verify_adaptation_safety.py` + +### Make generated artifact paths explicit in CLI output + +- ID: `adapt-457baca160` +- Pattern: `delivery_format` +- Status: `proposal-only` +- Risk: `low` +- Requires approval: `true` +- Target files: + - `scripts/yao.py` + - `README.md` +- Verification: + - `python3 tests/verify_yao_cli.py` + +### Attach tests and evidence refresh to each upgrade + +- ID: `adapt-abfee25d3a` +- Pattern: `evidence_testing` +- Status: `proposal-only` +- Risk: `medium` +- Requires approval: `true` +- Target files: + - `tests/verify_adaptation_safety.py` + - `scripts/render_skill_os2_coverage.py` + - `reports/skill_os2_coverage.json` +- Verification: + - `python3 tests/verify_adaptation_safety.py` + - `python3 tests/verify_skill_os2_coverage.py` + +## Evidence + +- patterns: `reports/user_patterns.json` +- proposals: `reports/adaptation_proposals.json` +- approval_ledger: `reports/adaptation_approval_ledger.json` +- regression: `reports/adaptation_regression_report.json` +- skill_os2_coverage: `reports/skill_os2_coverage.json` +- world_class_ledger: `reports/world_class_evidence_ledger.json` +- evidence_consistency: `reports/evidence_consistency.json` +- benchmark_reproducibility: `reports/benchmark_reproducibility.json` diff --git a/reports/skillops/weekly/2026-W25.json b/reports/skillops/weekly/2026-W25.json new file mode 100644 index 00000000..e946f5f8 --- /dev/null +++ b/reports/skillops/weekly/2026-W25.json @@ -0,0 +1,320 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17T08:58:54Z", + "skill_dir": ".", + "decision": "curator-review", + "week_id": "2026-W25", + "daily_report_count": 2, + "opportunity_count": 10, + "unique_opportunity_count": 5, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "top_score": 88, + "skill_count": 12, + "actionable_portfolio_issue_count": 0, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0, + "summary": { + "decision": "curator-review", + "week_id": "2026-W25", + "daily_report_count": 2, + "opportunity_count": 10, + "unique_opportunity_count": 5, + "ready_for_approval_review_count": 1, + "proposal_review_count": 4, + "observe_more_evidence_count": 0, + "report_only_count": 0, + "top_score": 88, + "skill_count": 12, + "actionable_portfolio_issue_count": 0, + "release_lock_ready": false, + "evidence_consistency_ok": true, + "public_world_class_ready": false, + "world_class_pending_count": 4, + "writes_source_files": false, + "auto_patch_enabled": false, + "failure_count": 0 + }, + "curator_contract": { + "schema_version": "1.0", + "contract": "weekly-skillops-curator-report", + "cadence": "weekly", + "source_of_truth": [ + "reports/skillops/daily/*.json", + "reports/skill_atlas.json", + "reports/benchmark_reproducibility.json", + "reports/evidence_consistency.json", + "reports/world_class_evidence_ledger.json" + ], + "raw_content_stored": false, + "redacted_or_generated_evidence_only": true, + "proposal_only": true, + "writes_source_files": false, + "auto_patch_enabled": false, + "approval_required_for_writes": true, + "counts_as_world_class_evidence": false + }, + "report_contract": { + "schema_version": "1.0", + "contract": "weekly-skillops-curator-report", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "week_id", + "daily_report_count", + "opportunity_count", + "unique_opportunity_count", + "ready_for_approval_review_count", + "proposal_review_count", + "observe_more_evidence_count", + "report_only_count", + "top_score", + "skill_count", + "actionable_portfolio_issue_count", + "release_lock_ready", + "evidence_consistency_ok", + "public_world_class_ready", + "world_class_pending_count", + "writes_source_files", + "auto_patch_enabled", + "failure_count" + ] + }, + "daily_reports": [ + "reports/skillops/daily/2026-06-16.json", + "reports/skillops/daily/2026-06-17.json" + ], + "opportunity_summary": { + "action_type_counts": { + "add_eval": 1, + "agents_update": 1, + "patch_existing_skill": 3 + }, + "decision_counts": { + "proposal_review": 4, + "ready_for_approval_review": 1 + }, + "risk_level_counts": { + "low": 3, + "medium": 2 + } + }, + "curator_queue": [ + { + "opportunity_id": "skillops-4ad91c18dd", + "proposal_id": "adapt-fbfe921ba5", + "pattern_id": "report_ui", + "title": "Improve report layout, visual hierarchy, and chart readability", + "opportunity_type": "artifact-quality", + "action_type": "patch_existing_skill", + "decision": "ready_for_approval_review", + "priority": "medium", + "score": 88, + "score_reasons": [ + "support_count=5", + "confidence=0.95", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 5, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md", + "tests/verify_skill_overview.py" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py", + "python3 tests/verify_skill_report_charts.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-531e13c1c6", + "proposal_id": "adapt-59d219a1fb", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "opportunity_type": "governance", + "action_type": "agents_update", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated privacy and approval signals should tighten durable operating guidance.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "references/user-memory-policy.md", + "references/autonomous-adaptation.md", + "schemas/adaptation-proposal.schema.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-709ef884f3", + "proposal_id": "adapt-18c7517f3d", + "pattern_id": "language_default", + "title": "Keep reports Chinese-first with optional English", + "opportunity_type": "report-experience", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated language preferences can improve existing report templates after approval.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/render_skill_overview.py", + "references/artifact-design-doctrine.md" + ], + "verification_commands": [ + "python3 tests/verify_skill_overview.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-f9e43237aa", + "proposal_id": "adapt-457baca160", + "pattern_id": "delivery_format", + "title": "Make generated artifact paths explicit in CLI output", + "opportunity_type": "artifact-discoverability", + "action_type": "patch_existing_skill", + "decision": "proposal_review", + "priority": "medium", + "score": 82, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=low", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.", + "risk_level": "low", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "scripts/yao.py", + "README.md" + ], + "verification_commands": [ + "python3 tests/verify_yao_cli.py" + ], + "daily_report_date": "2026-06-16" + }, + { + "opportunity_id": "skillops-0a3a2de7c3", + "proposal_id": "adapt-abfee25d3a", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "opportunity_type": "quality-gate", + "action_type": "add_eval", + "decision": "proposal_review", + "priority": "medium", + "score": 74, + "score_reasons": [ + "support_count=2", + "confidence=0.79", + "risk=medium", + "target_files_present", + "verification_present", + "approval_required" + ], + "policy_reason": "Repeated verification and evidence requests should become focused checks.", + "risk_level": "medium", + "requires_approval": true, + "write_allowed_without_approval": false, + "evidence_count": 2, + "target_files": [ + "tests/verify_adaptation_safety.py", + "scripts/render_skill_os2_coverage.py", + "reports/skill_os2_coverage.json" + ], + "verification_commands": [ + "python3 tests/verify_adaptation_safety.py", + "python3 tests/verify_skill_os2_coverage.py" + ], + "daily_report_date": "2026-06-16" + } + ], + "portfolio": { + "skill_count": 12, + "actionable_issue_count": 0, + "route_collision_count": 5, + "actionable_route_collision_count": 0, + "owner_gap_count": 9, + "actionable_owner_gap_count": 0, + "stale_count": 10, + "actionable_stale_count": 0, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, + "no_route_opportunity_count": 0 + }, + "release_state": { + "release_lock_ready": false, + "evidence_consistency_ok": true, + "public_world_class_ready": false, + "world_class_pending_count": 4 + }, + "actions": [ + { + "key": "review-ready-opportunities", + "priority": "high", + "action": "Review the top ready-for-approval SkillOps opportunities before preparing any approval ledger entry." + }, + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for pending world-class ledger entries before public claims." + } + ], + "failures": [], + "source_reports": { + "daily_dir": "reports/skillops/daily", + "skill_atlas": "reports/skill_atlas.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", + "evidence_consistency": "reports/evidence_consistency.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json" + }, + "artifacts": { + "json": "reports/skillops/weekly/2026-W25.json", + "markdown": "reports/skillops/weekly/2026-W25.md" + } +} diff --git a/reports/skillops/weekly/2026-W25.md b/reports/skillops/weekly/2026-W25.md new file mode 100644 index 00000000..dae0f676 --- /dev/null +++ b/reports/skillops/weekly/2026-W25.md @@ -0,0 +1,110 @@ +# Weekly SkillOps Curator Report + +Generated at: `2026-06-17T08:58:54Z` +Week: `2026-W25` + +## Summary + +- decision: `curator-review` +- daily_report_count: `2` +- unique_opportunity_count: `5` +- ready_for_approval_review_count: `1` +- proposal_review_count: `4` +- top_score: `88` +- skill_count: `12` +- actionable_portfolio_issue_count: `0` +- release_lock_ready: `False` +- evidence_consistency_ok: `True` +- public_world_class_ready: `False` +- world_class_pending_count: `4` + +This report is a weekly curator cockpit for generated SkillOps evidence. It does not scan private logs, write source files, apply patches, or count as world-class evidence. + +## Curator Boundary + +- raw_content_stored: `false` +- redacted_or_generated_evidence_only: `true` +- proposal_only: `true` +- writes_source_files: `false` +- auto_patch_enabled: `false` +- approval_required_for_writes: `true` +- counts_as_world_class_evidence: `false` + +## Actions + +- `high` Review the top ready-for-approval SkillOps opportunities before preparing any approval ledger entry. +- `high` Collect accepted external or human evidence for pending world-class ledger entries before public claims. + +## Curator Queue + +### Improve report layout, visual hierarchy, and chart readability + +- ID: `skillops-4ad91c18dd` +- Action: `patch_existing_skill` +- Decision: `ready_for_approval_review` +- Score: `88` +- Risk: `medium` +- Source day: `2026-06-16` +- Policy: Repeated report layout feedback maps to existing renderer and design doctrine changes. + +### Keep adaptive iteration approval-gated + +- ID: `skillops-531e13c1c6` +- Action: `agents_update` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Source day: `2026-06-16` +- Policy: Repeated privacy and approval signals should tighten durable operating guidance. + +### Keep reports Chinese-first with optional English + +- ID: `skillops-709ef884f3` +- Action: `patch_existing_skill` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Source day: `2026-06-16` +- Policy: Repeated language preferences can improve existing report templates after approval. + +### Make generated artifact paths explicit in CLI output + +- ID: `skillops-f9e43237aa` +- Action: `patch_existing_skill` +- Decision: `proposal_review` +- Score: `82` +- Risk: `low` +- Source day: `2026-06-16` +- Policy: Repeated delivery-path requests can improve CLI and README artifact discoverability. + +### Attach tests and evidence refresh to each upgrade + +- ID: `skillops-0a3a2de7c3` +- Action: `add_eval` +- Decision: `proposal_review` +- Score: `74` +- Risk: `medium` +- Source day: `2026-06-16` +- Policy: Repeated verification and evidence requests should become focused checks. + +## Portfolio Signals + +- skill_count: `12` +- actionable_issue_count: `0` +- route_collision_count: `5` +- actionable_route_collision_count: `0` +- owner_gap_count: `9` +- actionable_owner_gap_count: `0` +- stale_count: `10` +- actionable_stale_count: `0` +- drift_signal_count: `0` +- actionable_drift_signal_count: `0` +- no_route_opportunity_count: `0` + +## Evidence + +- daily_dir: `reports/skillops/daily` +- skill_atlas: `reports/skill_atlas.json` +- benchmark_reproducibility: `reports/benchmark_reproducibility.json` +- evidence_consistency: `reports/evidence_consistency.json` +- world_class_ledger: `reports/world_class_evidence_ledger.json` diff --git a/reports/system-model.json b/reports/system-model.json index bd11096e..0385a0ec 100644 --- a/reports/system-model.json +++ b/reports/system-model.json @@ -1,6 +1,6 @@ { "skill_name": "yao-meta-skill", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "systems_doctrine": "Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight.", "stability": { "score": 100, diff --git a/reports/telemetry_hook_recipes.json b/reports/telemetry_hook_recipes.json new file mode 100644 index 00000000..b9a6db70 --- /dev/null +++ b/reports/telemetry_hook_recipes.json @@ -0,0 +1,338 @@ +{ + "ok": true, + "schema_version": "1.0", + "skill_dir": ".", + "summary": { + "recipe_count": 5, + "native_auto_capture_count": 0, + "metadata_only_recipe_count": 5 + }, + "privacy_contract": { + "raw_content_allowed": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ], + "allowed_fields": [ + "event", + "skill", + "version", + "source", + "command", + "activation_type", + "outcome", + "failure_type", + "timestamp" + ] + }, + "recipes": [ + { + "id": "browser-extension", + "client": "Browser extension", + "command": "browser-extension", + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "trigger_points": [ + "Skill is explicitly selected by the user.", + "Skill activation is accepted by the host client." + ], + "source": "external", + "native_auto_capture": false, + "integration_status": "client-hook-recipe", + "metadata_only": true, + "emit_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "browser-extension" + ], + "emit_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_activation --activation-type explicit --outcome accepted --failure-type none --command browser-extension", + "dry_run_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "browser-extension", + "--dry-run" + ], + "dry_run_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_activation --activation-type explicit --outcome accepted --failure-type none --command browser-extension --dry-run" + }, + { + "id": "chrome-extension", + "client": "Chrome extension", + "command": "chrome-extension", + "event": "skill_output", + "activation_type": "manual", + "outcome": "edited", + "failure_type": "none", + "trigger_points": [ + "User keeps or edits a skill-generated artifact.", + "Extension records only the final outcome class, not page content." + ], + "source": "external", + "native_auto_capture": false, + "integration_status": "client-hook-recipe", + "metadata_only": true, + "emit_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_output", + "--activation-type", + "manual", + "--outcome", + "edited", + "--failure-type", + "none", + "--command", + "chrome-extension" + ], + "emit_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_output --activation-type manual --outcome edited --failure-type none --command chrome-extension", + "dry_run_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_output", + "--activation-type", + "manual", + "--outcome", + "edited", + "--failure-type", + "none", + "--command", + "chrome-extension", + "--dry-run" + ], + "dry_run_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_output --activation-type manual --outcome edited --failure-type none --command chrome-extension --dry-run" + }, + { + "id": "vscode-extension", + "client": "VS Code extension", + "command": "vscode-extension", + "event": "skill_activation", + "activation_type": "implicit", + "outcome": "accepted", + "failure_type": "none", + "trigger_points": [ + "Workspace skill route chooses this skill.", + "Extension records route outcome without file paths or code snippets." + ], + "source": "external", + "native_auto_capture": false, + "integration_status": "client-hook-recipe", + "metadata_only": true, + "emit_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_activation", + "--activation-type", + "implicit", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "vscode-extension" + ], + "emit_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_activation --activation-type implicit --outcome accepted --failure-type none --command vscode-extension", + "dry_run_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_activation", + "--activation-type", + "implicit", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "vscode-extension", + "--dry-run" + ], + "dry_run_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_activation --activation-type implicit --outcome accepted --failure-type none --command vscode-extension --dry-run" + }, + { + "id": "cli-wrapper", + "client": "CLI wrapper", + "command": "cli-wrapper", + "event": "script_run", + "activation_type": "manual", + "outcome": "unknown", + "failure_type": "none", + "trigger_points": [ + "Wrapper starts or finishes a known skill workflow command.", + "Wrapper records command family only, never command arguments." + ], + "source": "external", + "native_auto_capture": false, + "integration_status": "client-hook-recipe", + "metadata_only": true, + "emit_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "script_run", + "--activation-type", + "manual", + "--outcome", + "unknown", + "--failure-type", + "none", + "--command", + "cli-wrapper" + ], + "emit_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event script_run --activation-type manual --outcome unknown --failure-type none --command cli-wrapper", + "dry_run_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "script_run", + "--activation-type", + "manual", + "--outcome", + "unknown", + "--failure-type", + "none", + "--command", + "cli-wrapper", + "--dry-run" + ], + "dry_run_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event script_run --activation-type manual --outcome unknown --failure-type none --command cli-wrapper --dry-run" + }, + { + "id": "provider-adapter", + "client": "Provider adapter", + "command": "provider-adapter", + "event": "skill_output", + "activation_type": "manual", + "outcome": "accepted", + "failure_type": "none", + "trigger_points": [ + "Provider adapter receives an accepted or rejected skill output signal.", + "Adapter records only normalized outcome metadata." + ], + "source": "external", + "native_auto_capture": false, + "integration_status": "client-hook-recipe", + "metadata_only": true, + "emit_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_output", + "--activation-type", + "manual", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "provider-adapter" + ], + "emit_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_output --activation-type manual --outcome accepted --failure-type none --command provider-adapter", + "dry_run_argv": [ + "python3", + "scripts/yao.py", + "telemetry-emit", + ".", + "--output-jsonl", + ".yao/telemetry_spool/external_events.jsonl", + "--event", + "skill_output", + "--activation-type", + "manual", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "provider-adapter", + "--dry-run" + ], + "dry_run_command": "python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_output --activation-type manual --outcome accepted --failure-type none --command provider-adapter --dry-run" + } + ], + "artifacts": { + "json": "reports/telemetry_hook_recipes.json", + "markdown": "reports/telemetry_hook_recipes.md", + "spool_jsonl": ".yao/telemetry_spool/external_events.jsonl", + "import_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "import_argv": [ + "python3", + "scripts/yao.py", + "telemetry-import", + ".", + "--input-jsonl", + ".yao/telemetry_spool/external_events.jsonl" + ] + }, + "warnings": [ + "These recipes do not prove native host-client integration; they define the local metadata-only hook contract for those integrations." + ], + "failures": [] +} diff --git a/reports/telemetry_hook_recipes.md b/reports/telemetry_hook_recipes.md new file mode 100644 index 00000000..b32e5be4 --- /dev/null +++ b/reports/telemetry_hook_recipes.md @@ -0,0 +1,35 @@ +# Telemetry Hook Recipes + +These recipes show how Browser, Chrome, IDE, wrapper, or provider-side integrations can call `telemetry-emit` without collecting raw prompts, outputs, transcripts, messages, notes, arguments, or file content. + +They are client hook recipes, not proof that a host client is already natively integrated. + +## Summary + +- Recipe count: `5` +- Native auto-capture integrations claimed: `0` +- Default spool: `.yao/telemetry_spool/external_events.jsonl` + +## Recipes + +| Client | Command | Event | Outcome | Dry run | +| --- | --- | --- | --- | --- | +| Browser extension | `browser-extension` | `skill_activation` | `accepted` | `python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_activation --activation-type explicit --outcome accepted --failure-type none --command browser-extension --dry-run` | +| Chrome extension | `chrome-extension` | `skill_output` | `edited` | `python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_output --activation-type manual --outcome edited --failure-type none --command chrome-extension --dry-run` | +| VS Code extension | `vscode-extension` | `skill_activation` | `accepted` | `python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_activation --activation-type implicit --outcome accepted --failure-type none --command vscode-extension --dry-run` | +| CLI wrapper | `cli-wrapper` | `script_run` | `unknown` | `python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event script_run --activation-type manual --outcome unknown --failure-type none --command cli-wrapper --dry-run` | +| Provider adapter | `provider-adapter` | `skill_output` | `accepted` | `python3 scripts/yao.py telemetry-emit . --output-jsonl .yao/telemetry_spool/external_events.jsonl --event skill_output --activation-type manual --outcome accepted --failure-type none --command provider-adapter --dry-run` | + +## Import + +After a client finishes a batch, import the local spool: + +```bash +python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl +``` + +## Privacy Contract + +- Allowed fields: `event`, `skill`, `version`, `source`, `command`, `activation_type`, `outcome`, `failure_type`, `timestamp`. +- Blocked raw-content fields: `content, input, inputs, message, messages, note, output, outputs, prompt, raw, text, transcript`. +- Client integrations must map local state to normalized outcome classes before calling the hook. diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 0607c156..4d38e5d7 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -18,14 +18,15 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ] }, "summary": { "declared_bump": "minor", "recommended_bump": "minor", "breaking_change_count": 0, - "added_target_count": 1, + "added_target_count": 2, "metadata_change_count": 1, "checksum_change_count": 2, "failure_count": 0, @@ -34,7 +35,8 @@ "upgrade_diff": { "name_changed": false, "added_targets": [ - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "removed_targets": [], "compatibility_changes": [ @@ -61,25 +63,25 @@ { "field": "description", "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", - "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution." } ], "checksum_changes": [ { "field": "archive_sha256", "from": "", - "to": "7bc83805501d24cf8e5932ef76e0454100398d5a73652e7440be47984d0cf4a9" + "to": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "742dc905b0d221c4f336e741d168861edaf7b75e97fe1e89f7d454c1cabf928f" + "to": "5a99ef3133b0148cd5bbc1cd85fad4830bf34ca2bbac709f1e1fea2f810b1257" } ] }, "release_notes": [ "Recommended version bump: minor.", - "Added targets: agent-skills-compatible.", + "Added targets: agent-skills-compatible, vscode.", "Package or archive checksum changed; reviewers should verify package artifacts before release.", "Package verification evidence: reports/package_verification.md." ], @@ -88,7 +90,7 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } } diff --git a/reports/upgrade_check.md b/reports/upgrade_check.md index 2eb27d78..55f3cb20 100644 --- a/reports/upgrade_check.md +++ b/reports/upgrade_check.md @@ -9,7 +9,7 @@ ## Upgrade Diff -- Added targets: `agent-skills-compatible` +- Added targets: `agent-skills-compatible, vscode` - Removed targets: `none` - Compatibility changes: `3` - Metadata changes: `1` @@ -18,7 +18,7 @@ ## Release Notes - Recommended version bump: minor. -- Added targets: agent-skills-compatible. +- Added targets: agent-skills-compatible, vscode. - Package or archive checksum changed; reviewers should verify package artifacts before release. - Package verification evidence: reports/package_verification.md. diff --git a/reports/user_patterns.json b/reports/user_patterns.json new file mode 100644 index 00000000..d73a4f86 --- /dev/null +++ b/reports/user_patterns.json @@ -0,0 +1,136 @@ +{ + "ok": true, + "schema_version": "1.0", + "generated_at": "2026-06-15T00:00:00Z", + "skill_dir": ".", + "source": { + "label": "user_signals.example.jsonl", + "path": "evals/adaptation/user_signals.example.jsonl", + "fingerprint_sha256": "48f94a4f3b82f90db29921c13d09c765ec10aabfde1adce65710c137cc6c5abf", + "explicit_source": true, + "record_count": 8 + }, + "privacy_contract": { + "local_only": true, + "explicit_source_required": true, + "implicit_private_log_scan": false, + "raw_content_stored": false, + "redacted_excerpts_only": true, + "redacted_excerpt_limit": 240, + "writes_repository_files": false + }, + "summary": { + "record_count": 8, + "pattern_count": 5, + "discarded_signal_count": 0, + "min_support": 2, + "failure_count": 0 + }, + "patterns": [ + { + "pattern_id": "language_default", + "label": "Default language preference", + "signal_type": "report-language", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated report-language signals.", + "recommended_action": "Keep generated reports Chinese-first with an English switch where user-facing.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + } + ] + }, + { + "pattern_id": "report_ui", + "label": "Report UI and visualization preference", + "signal_type": "artifact-design", + "support_count": 5, + "confidence": 0.95, + "reason": "5 redacted records matched repeated artifact-design signals.", + "recommended_action": "Prioritize white-background Kami-style reports with readable charts and stable navigation.", + "evidence": [ + { + "record_id": "line-1", + "excerpt": "新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。" + }, + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-3", + "excerpt": "报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。" + } + ] + }, + { + "pattern_id": "approval_safety", + "label": "Approval and privacy boundary", + "signal_type": "governance", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated governance signals.", + "recommended_action": "Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path.", + "evidence": [ + { + "record_id": "line-5", + "excerpt": "自适应升级必须先生成提案，不能直接自动修改源文件。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "delivery_format", + "label": "Delivery format preference", + "signal_type": "artifact-format", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated artifact-format signals.", + "recommended_action": "Surface stable artifact paths and formats in CLI output and generated summaries.", + "evidence": [ + { + "record_id": "line-2", + "excerpt": "HTML 报告需要双语能力，但默认内容应该保持中文简体。" + }, + { + "record_id": "line-6", + "excerpt": "用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。" + } + ] + }, + { + "pattern_id": "evidence_testing", + "label": "Evidence and testing preference", + "signal_type": "quality-gate", + "support_count": 2, + "confidence": 0.79, + "reason": "2 redacted records matched repeated quality-gate signals.", + "recommended_action": "Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade.", + "evidence": [ + { + "record_id": "line-7", + "excerpt": "每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。" + }, + { + "record_id": "line-8", + "excerpt": "涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。" + } + ] + } + ], + "discarded_signals": [], + "failures": [], + "artifacts": { + "json": "reports/user_patterns.json", + "markdown": "reports/user_patterns.md" + } +} diff --git a/reports/user_patterns.md b/reports/user_patterns.md new file mode 100644 index 00000000..855d5ebb --- /dev/null +++ b/reports/user_patterns.md @@ -0,0 +1,72 @@ +# User Pattern Summary + +- Generated at: `2026-06-15T00:00:00Z` +- Local only: `true` +- Explicit source: `evals/adaptation/user_signals.example.jsonl` +- Records: `8` +- Patterns: `5` +- Discarded signals: `0` + +## Privacy Contract + +- No implicit private log scan. +- No unredacted raw content stored. +- Scan and proposal stages do not write source files. + +## Patterns + +### Default language preference + +- Pattern: `language_default` +- Support: `2` +- Confidence: `0.79` +- Reason: 2 redacted records matched repeated report-language signals. +- Recommended action: Keep generated reports Chinese-first with an English switch where user-facing. +- Redacted evidence: + - `line-1`: 新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。 + - `line-2`: HTML 报告需要双语能力，但默认内容应该保持中文简体。 + +### Report UI and visualization preference + +- Pattern: `report_ui` +- Support: `5` +- Confidence: `0.95` +- Reason: 5 redacted records matched repeated artifact-design signals. +- Recommended action: Prioritize white-background Kami-style reports with readable charts and stable navigation. +- Redacted evidence: + - `line-1`: 新生成的 Skill 报告默认使用中文简体，并在右上角提供英文切换。 + - `line-2`: HTML 报告需要双语能力，但默认内容应该保持中文简体。 + - `line-3`: 报告排版采用白底 Kami 风格，图表、模块和导航都要清晰。 + +### Approval and privacy boundary + +- Pattern: `approval_safety` +- Support: `2` +- Confidence: `0.79` +- Reason: 2 redacted records matched repeated governance signals. +- Recommended action: Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path. +- Redacted evidence: + - `line-5`: 自适应升级必须先生成提案，不能直接自动修改源文件。 + - `line-6`: 用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。 + +### Delivery format preference + +- Pattern: `delivery_format` +- Support: `2` +- Confidence: `0.79` +- Reason: 2 redacted records matched repeated artifact-format signals. +- Recommended action: Surface stable artifact paths and formats in CLI output and generated summaries. +- Redacted evidence: + - `line-2`: HTML 报告需要双语能力，但默认内容应该保持中文简体。 + - `line-6`: 用户偏好扫描必须由用户提供明确路径，不要默认扫描私人日志。 + +### Evidence and testing preference + +- Pattern: `evidence_testing` +- Support: `2` +- Confidence: `0.79` +- Reason: 2 redacted records matched repeated quality-gate signals. +- Recommended action: Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade. +- Redacted evidence: + - `line-7`: 每次升级都需要测试、覆盖报告和可审计证据，推送前要跑 CI。 + - `line-8`: 涉及 GitHub 推送时，要保留证据链，避免把计划当作完成证明。 diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json new file mode 100644 index 00000000..22da9491 --- /dev/null +++ b/reports/world_class_claim_guard.json @@ -0,0 +1,783 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "ledger_ready_to_claim_world_class": false, + "ledger_pending_count": 4, + "claim_surface_count": 182, + "json_claim_surface_count": 89, + "metadata_claim_surface_count": 90, + "package_claim_surface_count": 17, + "violation_count": 0, + "overclaim_guard_active": true, + "decision": "claim-guard-pass-evidence-pending" + }, + "rules": [ + { + "key": "ready-to-claim-true", + "reason": "ready_to_claim_world_class cannot be true while ledger evidence is pending.", + "pattern": "ready[\\s_-]*to[\\s_-]*claim[\\s_-]*world[\\s_-]*class\\s*[:=]\\s*`?true`?" + }, + { + "key": "world-class-ready-true", + "reason": "world-class readiness cannot be claimed before accepted external and human evidence exists.", + "pattern": "(public\\s+)?world[\\s_-]*class[\\s_-]*ready\\s*[:=]\\s*`?true`?" + }, + { + "key": "json-world-class-ready-true", + "reason": "machine-readable claim fields must remain false until the ledger is ready.", + "pattern": "\"(?:ready_to_claim_world_class|world_class_ready|public_world_class_ready)\"\\s*:\\s*true" + }, + { + "key": "completion-phrase", + "reason": "completion language is blocked until the world-class ledger is accepted.", + "pattern": "(public\\s+)?world[\\s_-]*class\\s+(?:complete|completed|achieved|certified)" + }, + { + "key": "zh-completion-phrase", + "reason": "中文完成态表述必须等到 ledger ready 后才能出现。", + "pattern": "世界级(?:已完成|完成|已达成|达成|准备就绪|可声明|认证通过)" + } + ], + "scanned_surfaces": [ + { + "path": "README.md", + "violation_count": 0 + }, + { + "path": "SKILL.md", + "violation_count": 0 + }, + { + "path": "agents/interface.yaml", + "violation_count": 0 + }, + { + "path": "dist/manifest.json", + "violation_count": 0 + }, + { + "path": "dist/targets/claude/README.md", + "violation_count": 0 + }, + { + "path": "dist/targets/claude/adapter.json", + "violation_count": 0 + }, + { + "path": "dist/targets/generic/adapter.json", + "violation_count": 0 + }, + { + "path": "dist/targets/openai/adapter.json", + "violation_count": 0 + }, + { + "path": "dist/targets/vscode/README.md", + "violation_count": 0 + }, + { + "path": "dist/targets/vscode/adapter.json", + "violation_count": 0 + }, + { + "path": "docs/README.fr-FR.md", + "violation_count": 0 + }, + { + "path": "docs/README.ja-JP.md", + "violation_count": 0 + }, + { + "path": "docs/README.ru-RU.md", + "violation_count": 0 + }, + { + "path": "docs/README.zh-CN.md", + "violation_count": 0 + }, + { + "path": "docs/migration-v2.md", + "violation_count": 0 + }, + { + "path": "evidence/world_class/README.md", + "violation_count": 0 + }, + { + "path": "evidence/world_class/intake.schema.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/human-adjudication.intake.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "violation_count": 0 + }, + { + "path": "evidence/world_class/templates/provider-holdout.intake.json", + "violation_count": 0 + }, + { + "path": "manifest.json", + "violation_count": 0 + }, + { + "path": "registry/examples/yao-meta-skill-1.0.0.json", + "violation_count": 0 + }, + { + "path": "registry/index.json", + "violation_count": 0 + }, + { + "path": "registry/index.schema.json", + "violation_count": 0 + }, + { + "path": "registry/package.schema.json", + "violation_count": 0 + }, + { + "path": "registry/packages/yao-meta-skill.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_approval_ledger.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_proposals.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_proposals.md", + "violation_count": 0 + }, + { + "path": "reports/adaptation_regression_report.json", + "violation_count": 0 + }, + { + "path": "reports/adaptation_regression_report.md", + "violation_count": 0 + }, + { + "path": "reports/adoption_drift_report.json", + "violation_count": 0 + }, + { + "path": "reports/adoption_drift_report.md", + "violation_count": 0 + }, + { + "path": "reports/architecture_maintainability.json", + "violation_count": 0 + }, + { + "path": "reports/architecture_maintainability.md", + "violation_count": 0 + }, + { + "path": "reports/artifact-design-profile.json", + "violation_count": 0 + }, + { + "path": "reports/artifact-design-profile.md", + "violation_count": 0 + }, + { + "path": "reports/baseline-compare.json", + "violation_count": 0 + }, + { + "path": "reports/baseline-compare.md", + "violation_count": 0 + }, + { + "path": "reports/benchmark_methodology.md", + "violation_count": 0 + }, + { + "path": "reports/benchmark_reproducibility.json", + "violation_count": 0 + }, + { + "path": "reports/benchmark_reproducibility.md", + "violation_count": 0 + }, + { + "path": "reports/candidate_registry.json", + "violation_count": 0 + }, + { + "path": "reports/candidate_registry.md", + "violation_count": 0 + }, + { + "path": "reports/compiled_targets.json", + "violation_count": 0 + }, + { + "path": "reports/compiled_targets.md", + "violation_count": 0 + }, + { + "path": "reports/conformance_matrix.json", + "violation_count": 0 + }, + { + "path": "reports/conformance_matrix.md", + "violation_count": 0 + }, + { + "path": "reports/context_budget.json", + "violation_count": 0 + }, + { + "path": "reports/context_budget.md", + "violation_count": 0 + }, + { + "path": "reports/context_budget_summary.json", + "violation_count": 0 + }, + { + "path": "reports/description_drift_history.md", + "violation_count": 0 + }, + { + "path": "reports/description_optimization.json", + "violation_count": 0 + }, + { + "path": "reports/description_optimization.md", + "violation_count": 0 + }, + { + "path": "reports/description_optimization_suite.json", + "violation_count": 0 + }, + { + "path": "reports/description_optimization_suite.md", + "violation_count": 0 + }, + { + "path": "reports/eval_suite.json", + "violation_count": 0 + }, + { + "path": "reports/evidence_consistency.json", + "violation_count": 0 + }, + { + "path": "reports/evidence_consistency.md", + "violation_count": 0 + }, + { + "path": "reports/family_summary.md", + "violation_count": 0 + }, + { + "path": "reports/feishu-ranking-pattern-analysis-2026-04-01.md", + "violation_count": 0 + }, + { + "path": "reports/github-benchmark-scan.json", + "violation_count": 0 + }, + { + "path": "reports/github-benchmark-scan.md", + "violation_count": 0 + }, + { + "path": "reports/governance_score.json", + "violation_count": 0 + }, + { + "path": "reports/install_simulation.json", + "violation_count": 0 + }, + { + "path": "reports/install_simulation.md", + "violation_count": 0 + }, + { + "path": "reports/intent-confidence.json", + "violation_count": 0 + }, + { + "path": "reports/intent-confidence.md", + "violation_count": 0 + }, + { + "path": "reports/intent-context.json", + "violation_count": 0 + }, + { + "path": "reports/intent-dialogue.json", + "violation_count": 0 + }, + { + "path": "reports/intent-dialogue.md", + "violation_count": 0 + }, + { + "path": "reports/iteration-directions.json", + "violation_count": 0 + }, + { + "path": "reports/iteration-directions.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/governed-incident-command/bundle.json", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/governed-incident-command/bundle.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/governed-incident-command/review.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/team-frontend-review/bundle.json", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/team-frontend-review/bundle.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/team-frontend-review/review.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/yao-meta-skill/bundle.json", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/yao-meta-skill/bundle.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_bundles/yao-meta-skill/review.md", + "violation_count": 0 + }, + { + "path": "reports/iteration_ledger.md", + "violation_count": 0 + }, + { + "path": "reports/output-risk-profile.json", + "violation_count": 0 + }, + { + "path": "reports/output-risk-profile.md", + "violation_count": 0 + }, + { + "path": "reports/output_blind_answer_key.json", + "violation_count": 0 + }, + { + "path": "reports/output_blind_review_pack.json", + "violation_count": 0 + }, + { + "path": "reports/output_blind_review_pack.md", + "violation_count": 0 + }, + { + "path": "reports/output_execution_runs.json", + "violation_count": 0 + }, + { + "path": "reports/output_execution_runs.md", + "violation_count": 0 + }, + { + "path": "reports/output_quality_scorecard.json", + "violation_count": 0 + }, + { + "path": "reports/output_quality_scorecard.md", + "violation_count": 0 + }, + { + "path": "reports/output_review_adjudication.json", + "violation_count": 0 + }, + { + "path": "reports/output_review_adjudication.md", + "violation_count": 0 + }, + { + "path": "reports/output_review_decisions.json", + "violation_count": 0 + }, + { + "path": "reports/output_review_kit.html", + "violation_count": 0 + }, + { + "path": "reports/output_review_kit.json", + "violation_count": 0 + }, + { + "path": "reports/output_review_kit.md", + "violation_count": 0 + }, + { + "path": "reports/package_verification.json", + "violation_count": 0 + }, + { + "path": "reports/package_verification.md", + "violation_count": 0 + }, + { + "path": "reports/portability_score.json", + "violation_count": 0 + }, + { + "path": "reports/portability_score.md", + "violation_count": 0 + }, + { + "path": "reports/promotion_decisions.json", + "violation_count": 0 + }, + { + "path": "reports/promotion_decisions.md", + "violation_count": 0 + }, + { + "path": "reports/prompt-quality-profile.json", + "violation_count": 0 + }, + { + "path": "reports/prompt-quality-profile.md", + "violation_count": 0 + }, + { + "path": "reports/python_compatibility.json", + "violation_count": 0 + }, + { + "path": "reports/python_compatibility.md", + "violation_count": 0 + }, + { + "path": "reports/reference-scan.json", + "violation_count": 0 + }, + { + "path": "reports/reference-scan.md", + "violation_count": 0 + }, + { + "path": "reports/reference-synthesis.json", + "violation_count": 0 + }, + { + "path": "reports/reference-synthesis.md", + "violation_count": 0 + }, + { + "path": "reports/registry_audit.json", + "violation_count": 0 + }, + { + "path": "reports/registry_audit.md", + "violation_count": 0 + }, + { + "path": "reports/regression_history.md", + "violation_count": 0 + }, + { + "path": "reports/review-studio.html", + "violation_count": 0 + }, + { + "path": "reports/review-studio.json", + "violation_count": 0 + }, + { + "path": "reports/review-viewer.html", + "violation_count": 0 + }, + { + "path": "reports/review-viewer.json", + "violation_count": 0 + }, + { + "path": "reports/review_annotations.json", + "violation_count": 0 + }, + { + "path": "reports/review_annotations.md", + "violation_count": 0 + }, + { + "path": "reports/review_waivers.json", + "violation_count": 0 + }, + { + "path": "reports/review_waivers.md", + "violation_count": 0 + }, + { + "path": "reports/route_scorecard.json", + "violation_count": 0 + }, + { + "path": "reports/route_scorecard.md", + "violation_count": 0 + }, + { + "path": "reports/runtime_permission_probes.json", + "violation_count": 0 + }, + { + "path": "reports/runtime_permission_probes.md", + "violation_count": 0 + }, + { + "path": "reports/security_trust_report.json", + "violation_count": 0 + }, + { + "path": "reports/security_trust_report.md", + "violation_count": 0 + }, + { + "path": "reports/skill-interpretation.html", + "violation_count": 0 + }, + { + "path": "reports/skill-interpretation.json", + "violation_count": 0 + }, + { + "path": "reports/skill-os-2-review.md", + "violation_count": 0 + }, + { + "path": "reports/skill-overview-v2-plan.md", + "violation_count": 0 + }, + { + "path": "reports/skill-overview.html", + "violation_count": 0 + }, + { + "path": "reports/skill-overview.json", + "violation_count": 0 + }, + { + "path": "reports/skill_atlas.html", + "violation_count": 0 + }, + { + "path": "reports/skill_atlas.json", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_audit.json", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_audit.md", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_coverage.json", + "violation_count": 0 + }, + { + "path": "reports/skill_os2_coverage.md", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-16.json", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-16.md", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-17.json", + "violation_count": 0 + }, + { + "path": "reports/skillops/daily/2026-06-17.md", + "violation_count": 0 + }, + { + "path": "reports/skillops/weekly/2026-W25.json", + "violation_count": 0 + }, + { + "path": "reports/skillops/weekly/2026-W25.md", + "violation_count": 0 + }, + { + "path": "reports/system-model.json", + "violation_count": 0 + }, + { + "path": "reports/system-model.md", + "violation_count": 0 + }, + { + "path": "reports/telemetry_hook_recipes.json", + "violation_count": 0 + }, + { + "path": "reports/telemetry_hook_recipes.md", + "violation_count": 0 + }, + { + "path": "reports/upgrade_check.json", + "violation_count": 0 + }, + { + "path": "reports/upgrade_check.md", + "violation_count": 0 + }, + { + "path": "reports/user_patterns.json", + "violation_count": 0 + }, + { + "path": "reports/user_patterns.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_claim_guard.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_intake.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_intake.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_ledger.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_ledger.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_plan.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_plan.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.html", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_evidence_preflight.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.html", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_operator_runbook.md", + "violation_count": 0 + }, + { + "path": "reports/world_class_submission_review.json", + "violation_count": 0 + }, + { + "path": "reports/world_class_submission_review.md", + "violation_count": 0 + }, + { + "path": "reports/yao-deploy-bridge-skill-research-plan.md", + "violation_count": 0 + }, + { + "path": "security/dependency_policy.md", + "violation_count": 0 + }, + { + "path": "security/network_policy.json", + "violation_count": 0 + }, + { + "path": "security/network_policy.md", + "violation_count": 0 + }, + { + "path": "security/permission_policy.json", + "violation_count": 0 + }, + { + "path": "security/permission_policy.md", + "violation_count": 0 + }, + { + "path": "security/script_policy.md", + "violation_count": 0 + }, + { + "path": "security/secret_policy.md", + "violation_count": 0 + }, + { + "path": "security/trust_policy.md", + "violation_count": 0 + }, + { + "path": "skill-ir/examples/yao-meta-skill.json", + "violation_count": 0 + }, + { + "path": "skill-ir/schema.json", + "violation_count": 0 + } + ], + "violations": [], + "source_ledger": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md" + }, + "artifacts": { + "json": "reports/world_class_claim_guard.json", + "markdown": "reports/world_class_claim_guard.md" + } +} diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md new file mode 100644 index 00000000..d2d4d668 --- /dev/null +++ b/reports/world_class_claim_guard.md @@ -0,0 +1,33 @@ +# World-Class Claim Guard + +Generated at: `2026-06-17` + +## Summary + +- decision: `claim-guard-pass-evidence-pending` +- ledger ready to claim world-class: `false` +- ledger pending evidence: `4` +- claim surfaces scanned: `182` +- JSON claim surfaces scanned: `89` +- metadata claim surfaces scanned: `90` +- package/runtime claim surfaces scanned: `17` +- violations: `0` +- overclaim guard active: `true` + +This guard scans public claim surfaces, machine-readable reports, and package/runtime metadata for completion language that would contradict the world-class evidence ledger. It allows evidence planning and pending-state language, but blocks completion claims until the ledger is ready. + +## Violations + +| Path | Line | Rule | Excerpt | +| --- | ---: | --- | --- | +| `none` | 0 | `none` | none | + +## Rules + +| Rule | Reason | +| --- | --- | +| `ready-to-claim-true` | ready_to_claim_world_class cannot be true while ledger evidence is pending. | +| `world-class-ready-true` | world-class readiness cannot be claimed before accepted external and human evidence exists. | +| `json-world-class-ready-true` | machine-readable claim fields must remain false until the ledger is ready. | +| `completion-phrase` | completion language is blocked until the world-class ledger is accepted. | +| `zh-completion-phrase` | 中文完成态表述必须等到 ledger ready 后才能出现。 | diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json new file mode 100644 index 00000000..c90e7d1d --- /dev/null +++ b/reports/world_class_evidence_intake.json @@ -0,0 +1,398 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "schema_present": true, + "ledger_entry_count": 4, + "template_count": 4, + "template_pass_count": 4, + "submission_count": 0, + "valid_submission_count": 0, + "invalid_submission_count": 0, + "operator_checklist_count": 4, + "operator_checklist_ready_count": 0, + "valid_packet_source_incomplete_count": 0, + "ready_for_external_collection": true, + "ready_for_ledger_review": false, + "ready_to_claim_world_class": false, + "overclaim_guard_active": true, + "decision": "awaiting-submissions" + }, + "schema": "evidence/world_class/intake.schema.json", + "templates": [ + { + "path": "evidence/world_class/templates/provider-holdout.intake.json", + "evidence_key": "provider-holdout", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 1, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + }, + { + "path": "evidence/world_class/templates/human-adjudication.intake.json", + "evidence_key": "human-adjudication", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 2, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + }, + { + "path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "evidence_key": "native-permission-enforcement", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 3, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + }, + { + "path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "evidence_key": "native-client-telemetry", + "status": "pass", + "template_only": true, + "artifact_integrity": { + "artifact_ref_count": 2, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": 0, + "required_artifact_verified_count": 0 + }, + "errors": [] + } + ], + "submissions": [], + "operator_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": true, + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": false, + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": false, + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", + "template_status": "pass", + "submission_status": "missing", + "source_accepted": false, + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_ledger": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + "pending_count": 4 + }, + "artifacts": { + "json": "reports/world_class_evidence_intake.json", + "markdown": "reports/world_class_evidence_intake.md" + } +} diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md new file mode 100644 index 00000000..50426d6d --- /dev/null +++ b/reports/world_class_evidence_intake.md @@ -0,0 +1,265 @@ +# World-Class Evidence Intake + +Generated at: `2026-06-17` + +## Summary + +- decision: `awaiting-submissions` +- schema present: `true` +- templates: `4` / `4` +- submissions: `0` valid / `0` total +- invalid submissions: `0` +- valid packet but source incomplete: `0` +- operator checklist: `0` ready / `4` total +- ready for external collection: `true` +- ready for ledger review: `false` +- ready to claim world-class: `false` +- overclaim guard active: `true` + +This report validates the intake contract for human and external evidence. A valid intake packet means the evidence is ready for ledger review; it does not by itself make a world-class claim true. + +## Templates + +| Evidence | Status | Path | Artifacts | Errors | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `pass` | `evidence/world_class/templates/provider-holdout.intake.json` | 0 existing / 0 sha256 verified / 0 required verified / 1 refs | none | +| `human-adjudication` | `pass` | `evidence/world_class/templates/human-adjudication.intake.json` | 0 existing / 0 sha256 verified / 0 required verified / 2 refs | none | +| `native-permission-enforcement` | `pass` | `evidence/world_class/templates/native-permission-enforcement.intake.json` | 0 existing / 0 sha256 verified / 0 required verified / 3 refs | none | +| `native-client-telemetry` | `pass` | `evidence/world_class/templates/native-client-telemetry.intake.json` | 0 existing / 0 sha256 verified / 0 required verified / 2 refs | none | + +## Submissions + +| Evidence | Status | Path | Artifacts | Errors | +| --- | --- | --- | --- | --- | +| `none` | `n/a` | none | none | none | + +## Operator Checklist + +| Evidence | Readiness | Submission | Next action | +| --- | --- | --- | --- | +| `provider-holdout` | `awaiting-submission` | `missing` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `human-adjudication` | `awaiting-submission` | `missing` | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | +| `native-permission-enforcement` | `awaiting-submission` | `missing` | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | +| `native-client-telemetry` | `awaiting-submission` | `missing` | Install a real client against the native host and import production metadata-only events. | + +### Provider Holdout + +- readiness: `awaiting-submission` +- blocking reason: No real evidence submission has been provided yet. +- owner: operator with provider credentials +- template: `evidence/world_class/templates/provider-holdout.intake.json` +- submission: `evidence/world_class/submissions/provider-holdout.json` + +#### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- submission_review: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +#### Must Collect + +- provenance_requirements: + - provider-backed model run + - observed timing + - observed token metadata +- success_checks: + - reports/output_execution_runs.json summary.model_executed_count > 0 + - reports/output_execution_runs.json summary.timing_observed_count > 0 + - reports/output_execution_runs.json summary.token_observed_count > 0 + - reports/skill_os2_audit.json item provider-holdout status becomes pass +- evidence_artifacts: + - reports/output_execution_runs.json + - reports/output_execution_runs.md + - reports/skill_os2_audit.json + - evidence/world_class/intake.schema.json + - evidence/world_class/templates/provider-holdout.intake.json + - reports/world_class_evidence_intake.json + - reports/world_class_evidence_intake.md +- privacy_contract: + - Do not commit provider credentials or environment dumps. + - The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +#### Source Runbook + +- Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. +- For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60 +- For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120 +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Human Adjudication + +- readiness: `awaiting-submission` +- blocking reason: No real evidence submission has been provided yet. +- owner: human reviewer +- template: `evidence/world_class/templates/human-adjudication.intake.json` +- submission: `evidence/world_class/submissions/human-adjudication.json` + +#### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- submission_review: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +#### Must Collect + +- provenance_requirements: + - real reviewer identity + - blind A/B decisions + - answer key unopened until decisions exist +- success_checks: + - reports/output_review_adjudication.json summary.pending_count == 0 + - reports/output_review_adjudication.json summary.judgment_count == summary.pair_count + - reports/output_review_adjudication.json summary.invalid_decision_count == 0 + - reports/output_review_adjudication.json summary.reviewer_metadata_present is true + - reports/output_review_adjudication.json summary.blind_review_attested is true + - reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json + - reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text + - reports/output_review_adjudication.json summary.ready_for_human_evidence is true + - reports/skill_os2_audit.json item human-adjudication status becomes pass +- evidence_artifacts: + - reports/output_blind_review_pack.md + - reports/output_review_kit.md + - reports/output_review_decisions.json + - reports/output_review_adjudication.json + - reports/output_review_adjudication.md + - scripts/import_output_review_decisions.py + - evidence/world_class/intake.schema.json + - evidence/world_class/templates/human-adjudication.intake.json + - reports/world_class_evidence_intake.json + - reports/world_class_evidence_intake.md +- privacy_contract: + - Reviewer decisions should not include raw user data or private customer detail. + - Reviewer reasons must be rubric-based and must not include raw user data or private customer detail. + - The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. + - The adjudication evidence stores prompt_sha256 instead of raw prompt text. + - The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged. + - Keep the answer key separate until after decisions are recorded. + +#### Source Runbook + +- `python3 scripts/yao.py output-review-kit --write-template` +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- `python3 scripts/adjudicate_output_review.py --write-template` +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only. +- `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- `python3 scripts/yao.py output-review` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Native Permission Enforcement + +- readiness: `awaiting-submission` +- blocking reason: No real evidence submission has been provided yet. +- owner: target client or installer integrator +- template: `evidence/world_class/templates/native-permission-enforcement.intake.json` +- submission: `evidence/world_class/submissions/native-permission-enforcement.json` + +#### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- submission_review: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +#### Must Collect + +- provenance_requirements: + - real target client or external installer runtime guard + - native enforcement flag or externally accepted guard proof + - residual risk retained for fallback targets +- success_checks: + - reports/runtime_permission_probes.json summary.native_enforcement_count > 0 + - reports/runtime_permission_probes.json summary.failure_count == 0 + - reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence + - reports/skill_os2_audit.json item native-permission-enforcement status becomes pass +- evidence_artifacts: + - dist/targets/*/adapter.json + - reports/runtime_permission_probes.json + - reports/runtime_permission_probes.md + - reports/install_simulation.json + - reports/install_simulation.md + - security/permission_policy.json + - evidence/world_class/intake.schema.json + - evidence/world_class/templates/native-permission-enforcement.intake.json + - reports/world_class_evidence_intake.json + - reports/world_class_evidence_intake.md +- privacy_contract: + - Do not mark native_enforcement true for metadata-only fallbacks. + - Keep residual risks visible for targets that still rely on operator enforcement. + +#### Source Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip` +- `python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation` +- `python3 scripts/yao.py runtime-permissions . --package-dir dist` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Native Client Telemetry + +- readiness: `awaiting-submission` +- blocking reason: No real evidence submission has been provided yet. +- owner: Browser/Chrome/IDE/provider client integrator +- template: `evidence/world_class/templates/native-client-telemetry.intake.json` +- submission: `evidence/world_class/submissions/native-client-telemetry.json` + +#### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- submission_review: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +#### Must Collect + +- provenance_requirements: + - real external client source + - metadata-only event + - local-first import path +- success_checks: + - reports/adoption_drift_report.json summary.source_types.external > 0 + - reports/adoption_drift_report.json summary.adoption_sample_count > 0 + - reports/skill_os2_audit.json item native-client-telemetry status becomes pass +- evidence_artifacts: + - reports/adoption_drift_report.json + - reports/adoption_drift_report.md + - reports/telemetry_hook_recipes.json + - scripts/telemetry_native_host.py + - evidence/world_class/intake.schema.json + - evidence/world_class/templates/native-client-telemetry.intake.json + - reports/world_class_evidence_intake.json + - reports/world_class_evidence_intake.md +- privacy_contract: + - Telemetry must remain metadata-only and local-first. + - Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. + +#### Source Runbook + +- `python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///` +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl` +- `python3 scripts/yao.py skill-atlas --workspace-root .` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +## Boundary + +- Templates and planned work do not count as accepted evidence. +- Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests. +- Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata. +- Local command-runner output does not count as provider-backed model evidence. +- Metadata fallback does not count as native permission enforcement. +- Pending reviewer work does not count as human adjudication. diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json new file mode 100644 index 00000000..89d6c6e6 --- /dev/null +++ b/reports/world_class_evidence_ledger.json @@ -0,0 +1,602 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "decision": "evidence-pending", + "ready_to_claim_world_class": false, + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "summary": { + "ledger_entry_count": 4, + "source_accepted_count": 1, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "submitted_entry_count": 0, + "reviewer_approved_submission_count": 0, + "missing_submission_count": 4, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 1, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "report_contract": { + "schema_version": "1.0", + "contract": "world-class-evidence-ledger", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "ready_to_claim_world_class", + "ledger_entry_count", + "source_accepted_count", + "accepted_count", + "pending_count", + "human_pending_count", + "external_pending_count", + "submitted_entry_count", + "reviewer_approved_submission_count", + "missing_submission_count", + "invalid_submission_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "submitted_but_pending_count", + "source_accepted_without_valid_submission_count", + "overclaim_guard_active" + ], + "source_of_truth": "summary" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "pass", + "source_accepted": true, + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "source_accepted": false, + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "source_accepted": false, + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4, + "evidence_requirement_count": 4 + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "ledger_counts_submission_as_completion": false, + "source_pass_requires_valid_submission": true + }, + "artifacts": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + "intake": "reports/world_class_evidence_intake.md" + } +} diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md new file mode 100644 index 00000000..14ee3362 --- /dev/null +++ b/reports/world_class_evidence_ledger.md @@ -0,0 +1,226 @@ +# World-Class Evidence Ledger + +Generated at: `2026-06-17` + +## Summary + +- decision: `evidence-pending` +- ready to claim world-class: `false` +- entries: `4` +- source accepted: `1` +- source checks: `11` pass / `19` total +- source blocked: `8` +- accepted: `0` +- pending: `4` +- human pending: `1` +- external pending: `3` +- submitted entries: `0` +- reviewer approved submissions: `0` +- submitted but pending: `0` +- source accepted without valid submission: `1` +- invalid submissions: `0` +- overclaim guard active: `true` + +This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence. + +## Ledger + +| Evidence | Status | Submission | Category | Current | Next action | +| --- | --- | --- | --- | --- | --- | +| `provider-holdout` | `pending` | `missing` | `external` | model-executed 10; token-observed 10 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | +| `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | +| `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | + +## Provider Holdout + +- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata. +- source status: `pass` +- observed state: `{"model_executed_count": 10, "timing_observed_count": 10, "token_observed_count": 10, "accepted": true}` +- source checks: `3` pass / `3` total +- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` + +### Provenance Requirements + +- provider-backed model run +- observed timing +- observed token metadata + +### Source Runbook + +- Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. +- For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60 +- For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120 +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Source Evidence Checks + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| Provider model run | `10` | `>0` | `pass` | +| Timing observed | `10` | `>0` | `pass` | +| Token usage observed | `10` | `>0` | `pass` | + +### Completion Assertions + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +## Human Adjudication + +- objective: Record real blind A/B reviewer decisions before claiming human output review completion. +- source status: `human_required` +- observed state: `{"pair_count": 5, "judgment_count": 0, "pending_count": 5, "invalid_decision_count": 0, "answer_revealed_count": 0, "reviewer_metadata_present": false, "reason_required": true, "blind_review_attested": false, "raw_content_excluded_attested": true, "reviewer_reason_required_attested": true, "ready_for_human_evidence": false, "raw_content_allowed": false, "raw_content_path_count": 0, "accepted": false}` +- source checks: `5` pass / `10` total +- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/human-adjudication.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` + +### Provenance Requirements + +- real reviewer identity +- blind A/B decisions +- answer key unopened until decisions exist + +### Source Runbook + +- `python3 scripts/yao.py output-review-kit --write-template` +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- `python3 scripts/adjudicate_output_review.py --write-template` +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only. +- `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- `python3 scripts/yao.py output-review` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Source Evidence Checks + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| Review pairs exist | `5` | `>0` | `pass` | +| No pending decisions | `5` | `==0` | `blocked` | +| Judgments complete | `0` | `==pair_count` | `blocked` | +| No invalid decisions | `0` | `==0` | `pass` | +| Reviewer metadata | `False` | `true` | `blocked` | +| Reason required | `True` | `true` | `pass` | +| Blind review attested | `False` | `true` | `blocked` | +| Raw content attested | `True` | `true` | `pass` | +| Raw content blocked | `False` | `false` | `pass` | +| Human evidence ready | `False` | `true` | `blocked` | + +### Completion Assertions + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/output_review_adjudication.json summary.reviewer_metadata_present is true +- reports/output_review_adjudication.json summary.blind_review_attested is true +- reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json +- reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text +- reports/output_review_adjudication.json summary.ready_for_human_evidence is true +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Reviewer reasons must be rubric-based and must not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. +- The adjudication evidence stores prompt_sha256 instead of raw prompt text. +- The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged. +- Keep the answer key separate until after decisions are recorded. + +## Native Permission Enforcement + +- objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities. +- source status: `external_required` +- observed state: `{"native_enforcement_count": 0, "metadata_fallback_count": 4, "installer_enforcement_pass_count": 4, "installer_permission_failure_count": 0, "installer_enforcement_ready": true, "residual_risk_count": 4, "failure_count": 0, "accepted": false}` +- source checks: `2` pass / `3` total +- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-permission-enforcement.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` + +### Provenance Requirements + +- real target client or external installer runtime guard +- native enforcement flag or externally accepted guard proof +- residual risk retained for fallback targets + +### Source Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip` +- `python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation` +- `python3 scripts/yao.py runtime-permissions . --package-dir dist` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Source Evidence Checks + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| Native enforcement | `0` | `>0` | `blocked` | +| Probe failures | `0` | `==0` | `pass` | +| Installer support | `True` | `true` | `pass` | + +### Completion Assertions + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +## Native Client Telemetry + +- objective: Import production metadata-only events from a real external client into the local drift loop. +- source status: `external_required` +- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` +- source checks: `1` pass / `3` total +- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` + +### Provenance Requirements + +- real external client source +- metadata-only event +- local-first import path + +### Source Runbook + +- `python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///` +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl` +- `python3 scripts/yao.py skill-atlas --workspace-root .` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Source Evidence Checks + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| External events | `0` | `>0` | `blocked` | +| Adoption sample | `0` | `>0` | `blocked` | +| Raw content blocked | `False` | `false` | `pass` | + +### Completion Assertions + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json new file mode 100644 index 00000000..2ee34bbb --- /dev/null +++ b/reports/world_class_evidence_plan.json @@ -0,0 +1,372 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "audit_decision": "continue-iteration", + "world_class_ready": false, + "audit_world_class_ready": false, + "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, + "task_count": 4, + "human_task_count": 1, + "external_task_count": 3, + "review_task_count": 0, + "decision": "collect-external-evidence" + }, + "tasks": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "evidence_requirements": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "pass", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 10; token-observed 10", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_audit": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "open_gap_count": 3 + }, + "artifacts": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "ledger": "reports/world_class_evidence_ledger.md", + "intake": "reports/world_class_evidence_intake.md" + } +} diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md new file mode 100644 index 00000000..2cf994c2 --- /dev/null +++ b/reports/world_class_evidence_plan.md @@ -0,0 +1,190 @@ +# World-Class Evidence Plan + +Generated at: `2026-06-17` + +## Summary + +- decision: `collect-external-evidence` +- audit decision: `continue-iteration` +- ready to claim world-class: `false` +- ledger completion required: `true` +- evidence requirements: `4` +- tasks: `4` +- human tasks: `1` +- external tasks: `3` + +This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions. + +## Task Table + +| Task | Status | Category | Owner | Current | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `pass` | `external` | operator with provider credentials | model-executed 10; token-observed 10 | +| `human-adjudication` | `human_required` | `human` | human reviewer | 0/5 decisions; pending 5 | +| `native-permission-enforcement` | `external_required` | `external` | target client or installer integrator | native-enforced targets 0; installer-enforced targets 4 | +| `native-client-telemetry` | `external_required` | `external` | Browser/Chrome/IDE/provider client integrator | external source events 0; adoption samples 0 | + +## Provider Holdout + +- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata. +- audit next action: Run provider-backed holdout cases with real credentials and commit only aggregate evidence. + +### Runbook + +- Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. +- For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60 +- For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120 +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Success Checks + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +### Evidence Artifacts + +- `reports/output_execution_runs.json` +- `reports/output_execution_runs.md` +- `reports/skill_os2_audit.json` +- `evidence/world_class/intake.schema.json` +- `evidence/world_class/templates/provider-holdout.intake.json` +- `reports/world_class_evidence_intake.json` +- `reports/world_class_evidence_intake.md` + +### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +## Human Adjudication + +- objective: Record real blind A/B reviewer decisions before claiming human output review completion. +- audit next action: Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. + +### Runbook + +- `python3 scripts/yao.py output-review-kit --write-template` +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- `python3 scripts/adjudicate_output_review.py --write-template` +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only. +- `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication` +- `python3 scripts/yao.py output-review` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Success Checks + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/output_review_adjudication.json summary.reviewer_metadata_present is true +- reports/output_review_adjudication.json summary.blind_review_attested is true +- reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json +- reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text +- reports/output_review_adjudication.json summary.ready_for_human_evidence is true +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +### Evidence Artifacts + +- `reports/output_blind_review_pack.md` +- `reports/output_review_kit.md` +- `reports/output_review_decisions.json` +- `reports/output_review_adjudication.json` +- `reports/output_review_adjudication.md` +- `scripts/import_output_review_decisions.py` +- `evidence/world_class/intake.schema.json` +- `evidence/world_class/templates/human-adjudication.intake.json` +- `reports/world_class_evidence_intake.json` +- `reports/world_class_evidence_intake.md` + +### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Reviewer reasons must be rubric-based and must not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. +- The adjudication evidence stores prompt_sha256 instead of raw prompt text. +- The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged. +- Keep the answer key separate until after decisions are recorded. + +## Native Permission Enforcement + +- objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities. +- audit next action: Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. + +### Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip` +- `python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation` +- `python3 scripts/yao.py runtime-permissions . --package-dir dist` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Success Checks + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +### Evidence Artifacts + +- `dist/targets/*/adapter.json` +- `reports/runtime_permission_probes.json` +- `reports/runtime_permission_probes.md` +- `reports/install_simulation.json` +- `reports/install_simulation.md` +- `security/permission_policy.json` +- `evidence/world_class/intake.schema.json` +- `evidence/world_class/templates/native-permission-enforcement.intake.json` +- `reports/world_class_evidence_intake.json` +- `reports/world_class_evidence_intake.md` + +### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +## Native Client Telemetry + +- objective: Import production metadata-only events from a real external client into the local drift loop. +- audit next action: Install a real client against the native host and import production metadata-only events. + +### Runbook + +- `python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///` +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl` +- `python3 scripts/yao.py skill-atlas --workspace-root .` +- `python3 scripts/yao.py skill-os2-audit . --generated-at ` +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` + +### Success Checks + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +### Evidence Artifacts + +- `reports/adoption_drift_report.json` +- `reports/adoption_drift_report.md` +- `reports/telemetry_hook_recipes.json` +- `scripts/telemetry_native_host.py` +- `evidence/world_class/intake.schema.json` +- `evidence/world_class/templates/native-client-telemetry.intake.json` +- `reports/world_class_evidence_intake.json` +- `reports/world_class_evidence_intake.md` + +### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html new file mode 100644 index 00000000..8c340acc --- /dev/null +++ b/reports/world_class_evidence_preflight.html @@ -0,0 +1,992 @@ + + + + + + World-Class Evidence Preflight + + + + +
+
+ Evidence Collection +

World-Class Evidence Preflight

+

This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.

+
Decisioncollection-preflight-blocked
Pending4
Ready1
Blocked3
Source11/19
Repairs12/12
Phases2/2
+
+
+
+

Submission Kit

+

Generate drafts only after real provider, human-review, native-permission, or native-client work exists. Drafts remain non-evidence until valid aggregate artifact refs and SHA-256 digests are supplied. Artifact prefill is convenience data only.

+
    +
  • submissions directory: evidence/world_class/submissions
    • +
  • drafts count as evidence: false
    • +
  • artifact prefill counts as evidence: false
    • +
  • preflight accepts evidence: false
    • +
+
+
+ Submission refs +

submission-ref

+ 7/7 ready +

Rows marked submission-ref are the aggregate paths expected in artifact_refs.

+ copy to artifact_refs: true +
+ +
+ Supporting evidence +

supporting-evidence

+ 31/31 ready +

Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.

+ copy to artifact_refs: false +
+
+

submission-ref rows are the paths expected in artifact_refs; supporting-evidence rows stay available for audit context.

+
+ +
+
+

Phase Queue

+

Repair rows are grouped into execution phases so operators can clear the first blocker before moving to later source collection. Phase queue rows remain procedural and never count as accepted evidence.

+
+
+
+ #20 · blocked +

Unblock access

+
+
+
Phase
unblock-access
+
Rows
4 / 4 blocked
+
Owners
Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator
+
Evidence
human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout
+
Next
Assign a real reviewer identity before claiming human adjudication.
+
Verify
python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Counts
does not count as completion
+
+
+ +
+
+ #40 · blocked +

Collect source

+
+
+
Phase
collect-source
+
Rows
8 / 8 blocked
+
Owners
Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator
+
Evidence
human-adjudication, native-client-telemetry, native-permission-enforcement
+
Next
Set reviewer_attestation only after choices are completed before opening the answer key.
+
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Counts
does not count as completion
+
+
+
+

Evidence Queue

+
+
+
+ external · blocked +

Provider Holdout

+
+
+
Evidence
provider-holdout
+
Ledger
pending
+
Intake
awaiting-submission
+
Review
awaiting-submission
+
Draft
evidence/world_class/submissions/provider-holdout.json
+
+
+

Next Action

+

Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.

+ python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions + python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts +
+
+

Artifact Roles

+
+
+ Submission refs +

submission-ref

+ 1/1 ready +

Rows marked submission-ref are the aggregate paths expected in artifact_refs.

+ copy to artifact_refs: true +
+ +
+ Supporting evidence +

supporting-evidence

+ 6/6 ready +

Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.

+ copy to artifact_refs: false +
+
+
+
+

Prechecks

+
+
+
+ file + Output eval cases +
+
+
Current
present
+
Status
pass
+
Action
Keep output holdout cases available before provider execution.
+
+
+ +
+
+ file + Provider runner +
+
+
Current
present
+
Status
pass
+
Action
Use the provider runner instead of the local command runner for model-backed evidence.
+
+
+ +
+
+ env_any + Provider credential +
+
+
Current
not-set
+
Status
missing
+
Action
Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
+
+
+ +
+
+ env + Provider model +
+
+
Current
not-set
+
Status
optional
+
Action
Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.
+
+
+
+
+
+

Source Checks

+
+
+
+ model_executed_count + Provider model run +
+
+
Current
10
+
Expected
>0
+
Status
pass
+
Action
Run provider-backed output-exec with real credentials.
+
+
+ +
+
+ timing_observed_count + Timing observed +
+
+
Current
10
+
Expected
>0
+
Status
pass
+
Action
Provider execution should record timing metadata.
+
+
+ +
+
+ token_observed_count + Token usage observed +
+
+
Current
10
+
Expected
>0
+
Status
pass
+
Action
Provider execution should return non-estimated token usage.
+
+
+
+
+
+

Repair Rows

+
+
+
+ precheck + provider-api-key +
+
+
Priority
20
+
Phase
unblock-access
+
Owner
operator with provider credentials
+
Status
blocked
+
Reason
Required env_any precheck is missing.
+
Action
Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
+
Verify
python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+
+
+
+

Runbook

+
  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
+
+ +
+
+ human · ready-for-human-review +

Human Adjudication

+
+
+
Evidence
human-adjudication
+
Ledger
pending
+
Intake
awaiting-submission
+
Review
awaiting-submission
+
Draft
evidence/world_class/submissions/human-adjudication.json
+
+
+

Next Action

+

Assign a real reviewer identity before claiming human adjudication.

+ python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions + python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts +
+
+

Artifact Roles

+
+
+ Submission refs +

submission-ref

+ 2/2 ready +

Rows marked submission-ref are the aggregate paths expected in artifact_refs.

+ copy to artifact_refs: true +
+ +
+ Supporting evidence +

supporting-evidence

+ 8/8 ready +

Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.

+ copy to artifact_refs: false +
+
+
+
+

Prechecks

+
+
+
+ file + Blind review kit +
+
+
Current
present
+
Status
pass
+
Action
Open the blind review kit and record real reviewer choices with required rationale.
+
+
+ +
+
+ file + Decision template +
+
+
Current
present
+
Status
pass
+
Action
Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication`.
+
+
+ +
+
+ file + Decision importer +
+
+
Current
present
+
Status
pass
+
Action
Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.
+
+
+ +
+
+ human + Human reviewer +
+
+
Current
external-human-action
+
Status
human-required
+
Action
Assign a real reviewer identity before claiming human adjudication.
+
+
+
+
+
+

Source Checks

+
+
+
+ pair_count + Review pairs exist +
+
+
Current
5
+
Expected
>0
+
Status
pass
+
Action
Generate the blind A/B review pack.
+
+
+ +
+
+ pending_count + No pending decisions +
+
+
Current
5
+
Expected
==0
+
Status
blocked
+
Action
Record a reviewer choice and reason for every pair.
+
+
+ +
+
+ judgment_count + Judgments complete +
+
+
Current
0
+
Expected
==pair_count
+
Status
blocked
+
Action
Every pair needs one valid human judgment.
+
+
+ +
+
+ invalid_decision_count + No invalid decisions +
+
+
Current
0
+
Expected
==0
+
Status
pass
+
Action
Fix malformed winner/confidence entries.
+
+
+ +
+
+ reviewer_metadata_present + Reviewer metadata +
+
+
Current
False
+
Expected
true
+
Status
blocked
+
Action
Record reviewer and reviewed_at before adjudication can count.
+
+
+ +
+
+ reason_required + Reason required +
+
+
Current
True
+
Expected
true
+
Status
pass
+
Action
Keep reason mandatory for every imported or direct reviewer decision.
+
+
+ +
+
+ blind_review_attested + Blind review attested +
+
+
Current
False
+
Expected
true
+
Status
blocked
+
Action
Set reviewer_attestation only after choices are completed before opening the answer key.
+
+
+ +
+
+ raw_content_excluded_attested + Raw content attested +
+
+
Current
True
+
Expected
true
+
Status
pass
+
Action
Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
+
+
+ +
+
+ raw_content_allowed + Raw content blocked +
+
+
Current
False
+
Expected
false
+
Status
pass
+
Action
Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
+
+
+ +
+
+ ready_for_human_evidence + Human evidence ready +
+
+
Current
False
+
Expected
true
+
Status
blocked
+
Action
Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
+
+
+
+
+
+

Repair Rows

+
+
+
+ precheck + human-reviewer +
+
+
Priority
20
+
Phase
unblock-access
+
Owner
human reviewer
+
Status
blocked
+
Reason
Required human precheck is human-required.
+
Action
Assign a real reviewer identity before claiming human adjudication.
+
Verify
python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + blind_review_attested +
+
+
Priority
40
+
Phase
collect-source
+
Owner
human reviewer
+
Status
blocked
+
Reason
Current value False does not satisfy 'true'.
+
Action
Set reviewer_attestation only after choices are completed before opening the answer key.
+
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + judgment_count +
+
+
Priority
40
+
Phase
collect-source
+
Owner
human reviewer
+
Status
blocked
+
Reason
Current value 0 does not satisfy '==pair_count'.
+
Action
Every pair needs one valid human judgment.
+
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + pending_count +
+
+
Priority
40
+
Phase
collect-source
+
Owner
human reviewer
+
Status
blocked
+
Reason
Current value 5 does not satisfy '==0'.
+
Action
Record a reviewer choice and reason for every pair.
+
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + ready_for_human_evidence +
+
+
Priority
40
+
Phase
collect-source
+
Owner
human reviewer
+
Status
blocked
+
Reason
Current value False does not satisfy 'true'.
+
Action
Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
+
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + reviewer_metadata_present +
+
+
Priority
40
+
Phase
collect-source
+
Owner
human reviewer
+
Status
blocked
+
Reason
Current value False does not satisfy 'true'.
+
Action
Record reviewer and reviewed_at before adjudication can count.
+
Verify
python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+
+
+
+

Runbook

+
  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
+
+ +
+
+ external · blocked +

Native Permission Enforcement

+
+
+
Evidence
native-permission-enforcement
+
Ledger
pending
+
Intake
awaiting-submission
+
Review
awaiting-submission
+
Draft
evidence/world_class/submissions/native-permission-enforcement.json
+
+
+

Next Action

+

Attach a real target-client or external installer runtime guard; metadata fallback is not enough.

+ python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions + python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts +
+
+

Artifact Roles

+
+
+ Submission refs +

submission-ref

+ 2/2 ready +

Rows marked submission-ref are the aggregate paths expected in artifact_refs.

+ copy to artifact_refs: true +
+ +
+ Supporting evidence +

supporting-evidence

+ 11/11 ready +

Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.

+ copy to artifact_refs: false +
+
+
+
+

Prechecks

+
+
+
+ file + Permission policy +
+
+
Current
present
+
Status
pass
+
Action
Keep approved high-permission capabilities explicit.
+
+
+ +
+
+ file + Runtime probes +
+
+
Current
present
+
Status
pass
+
Action
Refresh runtime permission probes after packaging changes.
+
+
+ +
+
+ external + Native guard +
+
+
Current
external-integration-required
+
Status
external-required
+
Action
Attach a real target-client or external installer runtime guard; metadata fallback is not enough.
+
+
+
+
+
+

Source Checks

+
+
+
+ native_enforcement_count + Native enforcement +
+
+
Current
0
+
Expected
>0
+
Status
blocked
+
Action
Collect real target-client or external runtime guard proof.
+
+
+ +
+
+ failure_count + Probe failures +
+
+
Current
0
+
Expected
==0
+
Status
pass
+
Action
Runtime permission probes must stay clean.
+
+
+ +
+
+ installer_enforcement_ready + Installer support +
+
+
Current
True
+
Expected
true
+
Status
pass
+
Action
Installer enforcement is supporting evidence, not native proof.
+
+
+
+
+
+

Repair Rows

+
+
+
+ precheck + native-guard +
+
+
Priority
20
+
Phase
unblock-access
+
Owner
target client or installer integrator
+
Status
blocked
+
Reason
Required external precheck is external-required.
+
Action
Attach a real target-client or external installer runtime guard; metadata fallback is not enough.
+
Verify
python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + native_enforcement_count +
+
+
Priority
40
+
Phase
collect-source
+
Owner
target client or installer integrator
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Collect real target-client or external runtime guard proof.
+
Verify
python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+
+
+
+

Runbook

+
  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
+
+ +
+
+ external · blocked +

Native Client Telemetry

+
+
+
Evidence
native-client-telemetry
+
Ledger
pending
+
Intake
awaiting-submission
+
Review
awaiting-submission
+
Draft
evidence/world_class/submissions/native-client-telemetry.json
+
+
+

Next Action

+

Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.

+ python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions + python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts +
+
+

Artifact Roles

+
+
+ Submission refs +

submission-ref

+ 2/2 ready +

Rows marked submission-ref are the aggregate paths expected in artifact_refs.

+ copy to artifact_refs: true +
+ +
+ Supporting evidence +

supporting-evidence

+ 6/6 ready +

Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.

+ copy to artifact_refs: false +
+
+
+
+

Prechecks

+
+
+
+ file + Native telemetry host +
+
+
Current
present
+
Status
pass
+
Action
Use the native host to receive metadata-only client events.
+
+
+ +
+
+ file + Hook recipes +
+
+
Current
present
+
Status
pass
+
Action
Refresh telemetry hook recipes before external client installation.
+
+
+ +
+
+ external + External client +
+
+
Current
external-integration-required
+
Status
external-required
+
Action
Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.
+
+
+
+
+
+

Source Checks

+
+
+
+ external_source_events + External events +
+
+
Current
0
+
Expected
>0
+
Status
blocked
+
Action
Import at least one metadata-only event from a real client.
+
+
+ +
+
+ adoption_sample_count + Adoption sample +
+
+
Current
0
+
Expected
>0
+
Status
blocked
+
Action
Telemetry must include adoption outcome evidence.
+
+
+ +
+
+ raw_content_allowed + Raw content blocked +
+
+
Current
False
+
Expected
false
+
Status
pass
+
Action
Telemetry must stay metadata-only.
+
+
+
+
+
+

Repair Rows

+
+
+
+ precheck + external-client +
+
+
Priority
20
+
Phase
unblock-access
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Status
blocked
+
Reason
Required external precheck is external-required.
+
Action
Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.
+
Verify
python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + adoption_sample_count +
+
+
Priority
40
+
Phase
collect-source
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Telemetry must include adoption outcome evidence.
+
Verify
python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+ +
+
+ source-check + external_source_events +
+
+
Priority
40
+
Phase
collect-source
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Status
blocked
+
Reason
Current value 0 does not satisfy '>0'.
+
Action
Import at least one metadata-only event from a real client.
+
Verify
python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions
+
Evidence
does not count as completion
+
+
+
+
+
+

Runbook

+
  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+
+
+
+
+
+

Safety Boundary

+
  • Environment variables are displayed only as set or not-set; secret values are never printed.
  • Human-required and external-required states are operator work, not accepted evidence.
  • The world-class ledger remains the only source of truth for ready_to_claim_world_class.
+
+
+ + diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json new file mode 100644 index 00000000..9b3b2372 --- /dev/null +++ b/reports/world_class_evidence_preflight.json @@ -0,0 +1,2038 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "precheck_count": 14, + "precheck_pass_count": 9, + "precheck_missing_count": 1, + "precheck_optional_count": 1, + "precheck_human_required_count": 1, + "precheck_external_required_count": 2, + "collection_ready_count": 1, + "collection_blocked_count": 3, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, + "repair_ready_count": 0, + "repair_phase_counts": { + "unblock-access": 4, + "collect-source": 8 + }, + "next_repair_action_id": "human-adjudication-precheck-human-reviewer", + "next_repair_phase": "unblock-access", + "next_repair_owner": "human reviewer", + "next_repair_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "repair_counts_as_completion": false, + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false, + "pending_count": 4, + "ready_to_claim_world_class": false, + "credential_value_exposed": false, + "preflight_counts_as_evidence": false, + "decision": "collection-preflight-blocked" + }, + "status_counts": { + "blocked": 3, + "ready-for-human-review": 1 + }, + "precheck_status_counts": { + "pass": 9, + "missing": 1, + "optional": 1, + "human-required": 1, + "external-required": 2 + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": true, + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-api-key", + "label": "Provider credential", + "kind": "env_any", + "required": true, + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "secret_value_redacted": true, + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], + "status": "missing", + "actual": "not-set", + "set_count": 0, + "set_envs": [] + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "provider-specific model, or pass --provider-model" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + } + ] + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "ready-for-human-review", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices with required rationale.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication`.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + } + ], + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ] + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ] + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "repair_checklist": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ] + } + ], + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-api-key", + "label": "Provider credential", + "kind": "env_any", + "required": true, + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "secret_value_redacted": true, + "env_any": [ + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY" + ], + "status": "missing", + "actual": "not-set", + "set_count": 0, + "set_envs": [] + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "provider-specific model, or pass --provider-model" + }, + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices with required rationale.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication`.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + }, + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + }, + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + }, + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "submissions": { + "directory": "evidence/world_class/submissions", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "submission_kit_prefill_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "preflight_counts_submission_as_completion": false, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 7, + "submission_ref_ready_count": 7, + "supporting_evidence_total_count": 31, + "supporting_evidence_ready_count": 31, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + } + }, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "operator_runbook": "reports/world_class_operator_runbook.json" + }, + "artifacts": { + "json": "reports/world_class_evidence_preflight.json", + "markdown": "reports/world_class_evidence_preflight.md", + "html": "reports/world_class_evidence_preflight.html" + } +} diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md new file mode 100644 index 00000000..048f9548 --- /dev/null +++ b/reports/world_class_evidence_preflight.md @@ -0,0 +1,201 @@ +# World-Class Evidence Preflight + +Generated at: `2026-06-17` + +## Summary + +- decision: `collection-preflight-blocked` +- ready to claim world-class: `false` +- preflight counts as evidence: `false` +- credential value exposed: `false` +- collection ready: `1` +- collection blocked: `3` +- source checks: `11` pass / `19` total +- repair rows: `12` blocked / `12` total +- phase queue: `2` blocked / `2` phases +- phase queue rows: `12` +- next repair action: `human-adjudication-precheck-human-reviewer` +- next repair owner: `human reviewer` +- next phase: `unblock-access` +- next phase action: `human-adjudication-precheck-human-reviewer` + +This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger. + +## Submission Kit Handoff + +- submissions directory: `evidence/world_class/submissions` +- prepare drafts: `python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions` +- prepare drafts with artifact SHA prefill: `python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts` +- validate intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard claims: `python3 scripts/yao.py world-class-claim-guard .` +- drafts count as evidence: `false` +- artifact prefill counts as evidence: `false` +- submission refs ready: `7` / `7` +- supporting evidence ready: `31` / `31` + +Generate the submission kit after the real provider, human, native-permission, or native-client work exists. The generated JSON drafts remain `template_only: true` until an operator edits them with real aggregate artifact references and matching SHA-256 digests. The prefill command only inserts local artifact SHA-256 digests; it does not make a draft count as evidence. + +| Role | Copy to artifact_refs | Ready | Meaning | +| --- | --- | --- | --- | +| `submission-ref` | `true` | `7 / 7` | Rows marked submission-ref are the aggregate paths expected in artifact_refs. | +| `supporting-evidence` | `false` | `31 / 31` | Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs. | + +`submission-ref` rows are the only checklist rows expected in `artifact_refs`; `supporting-evidence` rows stay available for audit context and reviewer traceability. + +## Phase Queue + +Phase queue rows group the same repair checklist into operator execution phases. They are procedural guidance only and do not count as completion evidence. + +| Priority | Phase | Status | Rows | Owners | Evidence | Verify | Next action | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `20` | `unblock-access` | `blocked` | 4 / 4 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Assign a real reviewer identity before claiming human adjudication. | +| `40` | `collect-source` | `blocked` | 8 / 8 blocked | Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator | human-adjudication, native-client-telemetry, native-permission-enforcement | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | + +## Evidence Items + +| Evidence | Status | Intake | Review | Next action | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `blocked` | `awaiting-submission` | `awaiting-submission` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | +| `human-adjudication` | `ready-for-human-review` | `awaiting-submission` | `awaiting-submission` | Assign a real reviewer identity before claiming human adjudication. | +| `native-permission-enforcement` | `blocked` | `awaiting-submission` | `awaiting-submission` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | +| `native-client-telemetry` | `blocked` | `awaiting-submission` | `awaiting-submission` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | + +## Repair Checklist + +Repair rows convert preflight and source blockers into a prioritized operator queue. They are guidance only and do not count as completion evidence. + +| Priority | Phase | Owner | Evidence | Type | Target | Status | Verify | Next action | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `20` | `unblock-access` | human reviewer | `human-adjudication` | `precheck` | `human-reviewer` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Assign a real reviewer identity before claiming human adjudication. | +| `20` | `unblock-access` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `precheck` | `external-client` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | +| `20` | `unblock-access` | target client or installer integrator | `native-permission-enforcement` | `precheck` | `native-guard` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | +| `20` | `unblock-access` | operator with provider credentials | `provider-holdout` | `precheck` | `provider-api-key` | `blocked` | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | +| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `blind_review_attested` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Set reviewer_attestation only after choices are completed before opening the answer key. | +| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `judgment_count` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Every pair needs one valid human judgment. | +| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `pending_count` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record a reviewer choice and reason for every pair. | +| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `ready_for_human_evidence` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. | +| `40` | `collect-source` | human reviewer | `human-adjudication` | `source-check` | `reviewer_metadata_present` | `blocked` | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Record reviewer and reviewed_at before adjudication can count. | +| `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `adoption_sample_count` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Telemetry must include adoption outcome evidence. | +| `40` | `collect-source` | Browser/Chrome/IDE/provider client integrator | `native-client-telemetry` | `source-check` | `external_source_events` | `blocked` | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Import at least one metadata-only event from a real client. | +| `40` | `collect-source` | target client or installer integrator | `native-permission-enforcement` | `source-check` | `native_enforcement_count` | `blocked` | `python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | Collect real target-client or external runtime guard proof. | + +## Provider Holdout + +- status: `blocked` +- ledger: `pending` +- submission: `evidence/world_class/submissions/provider-holdout.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions` +- prepare draft with artifact SHA prefill: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts` +- submission refs ready: `1` / `1` +- supporting evidence ready: `6` / `6` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Output eval cases | `file` | `present` | `pass` | Keep output holdout cases available before provider execution. | +| Provider runner | `file` | `present` | `pass` | Use the provider runner instead of the local command runner for model-backed evidence. | +| Provider credential | `env_any` | `not-set` | `missing` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | +| Provider model | `env` | `not-set` | `optional` | Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Provider model run | `10` | `>0` | `pass` | Run provider-backed output-exec with real credentials. | +| Timing observed | `10` | `>0` | `pass` | Provider execution should record timing metadata. | +| Token usage observed | `10` | `>0` | `pass` | Provider execution should return non-estimated token usage. | + +## Human Adjudication + +- status: `ready-for-human-review` +- ledger: `pending` +- submission: `evidence/world_class/submissions/human-adjudication.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions` +- prepare draft with artifact SHA prefill: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts` +- submission refs ready: `2` / `2` +- supporting evidence ready: `8` / `8` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Blind review kit | `file` | `present` | `pass` | Open the blind review kit and record real reviewer choices with required rationale. | +| Decision template | `file` | `present` | `pass` | Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication`. | +| Decision importer | `file` | `present` | `pass` | Use the importer to reject raw content fields and normalize reviewer decisions before adjudication. | +| Human reviewer | `human` | `external-human-action` | `human-required` | Assign a real reviewer identity before claiming human adjudication. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Review pairs exist | `5` | `>0` | `pass` | Generate the blind A/B review pack. | +| No pending decisions | `5` | `==0` | `blocked` | Record a reviewer choice and reason for every pair. | +| Judgments complete | `0` | `==pair_count` | `blocked` | Every pair needs one valid human judgment. | +| No invalid decisions | `0` | `==0` | `pass` | Fix malformed winner/confidence entries. | +| Reviewer metadata | `False` | `true` | `blocked` | Record reviewer and reviewed_at before adjudication can count. | +| Reason required | `True` | `true` | `pass` | Keep reason mandatory for every imported or direct reviewer decision. | +| Blind review attested | `False` | `true` | `blocked` | Set reviewer_attestation only after choices are completed before opening the answer key. | +| Raw content attested | `True` | `true` | `pass` | Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content. | +| Raw content blocked | `False` | `false` | `pass` | Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages. | +| Human evidence ready | `False` | `true` | `blocked` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. | + +## Native Permission Enforcement + +- status: `blocked` +- ledger: `pending` +- submission: `evidence/world_class/submissions/native-permission-enforcement.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions` +- prepare draft with artifact SHA prefill: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts` +- submission refs ready: `2` / `2` +- supporting evidence ready: `11` / `11` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Permission policy | `file` | `present` | `pass` | Keep approved high-permission capabilities explicit. | +| Runtime probes | `file` | `present` | `pass` | Refresh runtime permission probes after packaging changes. | +| Native guard | `external` | `external-integration-required` | `external-required` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Native enforcement | `0` | `>0` | `blocked` | Collect real target-client or external runtime guard proof. | +| Probe failures | `0` | `==0` | `pass` | Runtime permission probes must stay clean. | +| Installer support | `True` | `true` | `pass` | Installer enforcement is supporting evidence, not native proof. | + +## Native Client Telemetry + +- status: `blocked` +- ledger: `pending` +- submission: `evidence/world_class/submissions/native-client-telemetry.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions` +- prepare draft with artifact SHA prefill: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts` +- submission refs ready: `2` / `2` +- supporting evidence ready: `6` / `6` + +### Prechecks + +| Check | Kind | Current | Status | Next action | +| --- | --- | --- | --- | --- | +| Native telemetry host | `file` | `present` | `pass` | Use the native host to receive metadata-only client events. | +| Hook recipes | `file` | `present` | `pass` | Refresh telemetry hook recipes before external client installation. | +| External client | `external` | `external-integration-required` | `external-required` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | + +### Source Checks + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | +| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | + +## Boundary + +- Environment variables are reported only as `set` or `not-set`; values are never printed. +- Human-required and external-required states are operator actions, not accepted evidence. +- The world-class ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html new file mode 100644 index 00000000..eab34933 --- /dev/null +++ b/reports/world_class_operator_runbook.html @@ -0,0 +1,168 @@ + + + + + + World-Class Operator Runbook + + + + +
+
+ Evidence Operations +

World-Class Operator Runbook

+

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

+
Pending4
Awaiting4
Ready0
Source11/19
Queue2/2
Steps6/6
Gate4/5
Blocked8
Invalid0
+
+

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.
+

Coordination Plan

These steps tell the assistant what to run and where the user must supply real provider, reviewer, client, or telemetry input. Every row is operational guidance only and keeps counts_as_completion false.

路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
StepEvidenceOwnerUserUser actionAssistant actionCommandPass condition
prepare-evidence-sessionallassistant + usertrueConfirm provider access, reviewer availability, target client path, and telemetry client path before collection starts.Run preflight and prepare submission drafts without accepting them as evidence.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifactsPreflight lists the same pending evidence keys and no credential values are printed.
collect-provider-holdoutprovider-holdoutassistant + operator with provider credentialstrueProvide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.python3 scripts/yao.py output-exec . --provider-runner <openai|deepseek> --provider-model <model> --timeout-seconds 60reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.
collect-human-adjudicationhuman-adjudicationhuman reviewer + assistanttrueOpen the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved.Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet.python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true.
collect-native-permission-enforcementnative-permission-enforcementtarget client or installer integrator + assistanttrueSelect a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback.Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet.python3 scripts/yao.py runtime-permissions . --package-dir distreports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0.
collect-native-client-telemetrynative-client-telemetryreal client integrator + assistanttrueInstall the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event.Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source externalreports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0.
review-and-release-gateallassistant + ledger reviewertrueApprove only validated evidence packets and confirm the release wording after the claim guard passes.Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish.python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-testLedger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes.
+

Phase Queue

  • Unblock accessunblock-access: 4 / 4Assign a real reviewer identity before claiming human adjudication.
  • Collect sourcecollect-source: 8 / 8Set reviewer_attestation only after choices are completed before opening the answer key.
+

Evidence Items

+
external · awaiting-submission

Provider Holdout

+

No evidence packet has been submitted for review.

+
+
Owner
operator with provider credentials
+
Ledger
pending
+
Blocked
0
+
Queue
1
+
Submission
evidence/world_class/submissions/provider-holdout.json
+
+

Phase Queue

  • Unblock accessunblock-access: 1 / 1Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
+

Source Runbook

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • provider-backed model run
  • observed timing
  • observed token metadata
+

Success Checks

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
+

Privacy

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
+
+

Next Source Actions

  • No blocked source checks.
+

Source Evidence Snapshot

  • Provider model runmodel_executed_count: 10 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 10 / >0Provider execution should return non-estimated token usage.
+
+
human · awaiting-submission

Human Adjudication

+

No evidence packet has been submitted for review.

+
+
Owner
human reviewer
+
Ledger
pending
+
Blocked
5
+
Queue
2
+
Submission
evidence/world_class/submissions/human-adjudication.json
+
+

Phase Queue

  • Unblock accessunblock-access: 1 / 1Assign a real reviewer identity before claiming human adjudication.
  • Collect sourcecollect-source: 5 / 5Set reviewer_attestation only after choices are completed before opening the answer key.
+

Source Runbook

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
+

Success Checks

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
+

Privacy

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.
+
+

Next Source Actions

  • Record a reviewer choice and reason for every pair.
  • Every pair needs one valid human judgment.
  • Record reviewer and reviewed_at before adjudication can count.
  • Set reviewer_attestation only after choices are completed before opening the answer key.
  • Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
+

Source Evidence Snapshot

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
+
+
external · awaiting-submission

Native Permission Enforcement

+

No evidence packet has been submitted for review.

+
+
Owner
target client or installer integrator
+
Ledger
pending
+
Blocked
1
+
Queue
2
+
Submission
evidence/world_class/submissions/native-permission-enforcement.json
+
+

Phase Queue

  • Unblock accessunblock-access: 1 / 1Attach a real target-client or external installer runtime guard; metadata fallback is not enough.
  • Collect sourcecollect-source: 1 / 1Collect real target-client or external runtime guard proof.
+

Source Runbook

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
+

Success Checks

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
+

Privacy

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
+
+

Next Source Actions

  • Collect real target-client or external runtime guard proof.
+

Source Evidence Snapshot

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
+
+
external · awaiting-submission

Native Client Telemetry

+

No evidence packet has been submitted for review.

+
+
Owner
Browser/Chrome/IDE/provider client integrator
+
Ledger
pending
+
Blocked
2
+
Queue
2
+
Submission
evidence/world_class/submissions/native-client-telemetry.json
+
+

Phase Queue

  • Unblock accessunblock-access: 1 / 1Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.
  • Collect sourcecollect-source: 2 / 2Telemetry must include adoption outcome evidence.
+

Source Runbook

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
+

Commands

  • prepare submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • review queuepython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard claimpython3 scripts/yao.py world-class-claim-guard .
+
+

Must Collect

  • real external client source
  • metadata-only event
  • local-first import path
+

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
+

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
+

Next Source Actions

  • Import at least one metadata-only event from a real client.
  • Telemetry must include adoption outcome evidence.
+

Source Evidence Snapshot

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
+

Release Gate

+
Decisionblocked-until-evidence-accepted
+
Readyfalse
+
Blocked4/5
+
Countsfalse
+
+

Run make ci-test in a clean worktree and verify GitHub Actions before converting the PR out of Draft.

+
CheckCurrentExpectedStatusArtifact
World-class ledger readyevidence-pendingready_to_claim_world_class == trueblockedreports/world_class_evidence_ledger.json
Claim guard cleanviolations 0; ledger ready Falseviolation_count == 0 and ledger_ready_to_claim_world_class == trueblockedreports/world_class_claim_guard.json
Benchmark public claim readypublic_claim_ready Falsepublic_claim_ready == trueblockedreports/benchmark_reproducibility.json
Review Studio cleanblockers 0; warnings 3blocker_count == 0 and warning_count == 0blockedreports/review-studio.json
Evidence consistency cleanconsistentdecision == consistent and fail_count == 0passreports/evidence_consistency.json
+

Boundary

  • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
  • Valid intake means ready for submission review; ledger review still requires passing source evidence.
  • The world-class ledger and claim guard remain the source of truth.
+ + + diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json new file mode 100644 index 00000000..7a3f2226 --- /dev/null +++ b/reports/world_class_operator_runbook.json @@ -0,0 +1,1752 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "pending_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "ready_for_ledger_review_count": 0, + "valid_packet_source_incomplete_count": 0, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "repair_checklist_count": 12, + "repair_blocked_count": 12, + "phase_queue_count": 2, + "phase_queue_blocked_count": 2, + "phase_queue_row_count": 12, + "phase_queue_next_phase": "unblock-access", + "phase_queue_next_action_id": "human-adjudication-precheck-human-reviewer", + "phase_queue_next_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "phase_queue_counts_as_completion": false, + "coordination_step_count": 6, + "coordination_user_required_step_count": 6, + "coordination_pending_evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "coordination_counts_as_completion": false, + "release_gate_ready": false, + "release_gate_blocked_count": 4, + "release_gate_check_count": 5, + "release_gate_counts_as_completion": false, + "ready_to_claim_world_class": false, + "runbook_counts_as_completion": false, + "decision": "collect-evidence" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "runbook_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": true, + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "current": "model-executed 10; token-observed 10", + "execution_runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ] + }, + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "blocked_source_check_count": 0, + "next_source_actions": [], + "repair_checklist": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ], + "repair_blocked_count": 1, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "operator with provider credentials" + ], + "evidence_keys": [ + "provider-holdout" + ], + "next_action_id": "provider-holdout-precheck-provider-api-key", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 1, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/provider-holdout.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "current": "0/5 decisions; pending 5", + "execution_runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ] + }, + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "blocked_source_check_count": 5, + "next_source_actions": [ + "Record a reviewer choice and reason for every pair.", + "Every pair needs one valid human judgment.", + "Record reviewer and reviewed_at before adjudication can count.", + "Set reviewer_attestation only after choices are completed before opening the answer key.", + "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + ], + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "repair_blocked_count": 6, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 5, + "row_count": 5, + "owners": [ + "human reviewer" + ], + "evidence_keys": [ + "human-adjudication" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/human-adjudication.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "current": "native-enforced targets 0; installer-enforced targets 4", + "execution_runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real target client or external installer runtime guard", + "native enforcement flag or externally accepted guard proof", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ] + }, + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "blocked_source_check_count": 1, + "next_source_actions": [ + "Collect real target-client or external runtime guard proof." + ], + "repair_checklist": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "repair_blocked_count": 2, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-precheck-native-guard", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "target client or installer integrator" + ], + "evidence_keys": [ + "native-permission-enforcement" + ], + "next_action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-permission-enforcement.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "current": "external source events 0; adoption samples 0", + "execution_runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ], + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "must_collect": { + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ] + }, + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "blocked_source_check_count": 2, + "next_source_actions": [ + "Import at least one metadata-only event from a real client.", + "Telemetry must include adoption outcome evidence." + ], + "repair_checklist": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "repair_blocked_count": 3, + "repair_counts_as_completion": false, + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 1, + "row_count": 1, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-precheck-external-client", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 2, + "row_count": 2, + "owners": [ + "Browser/Chrome/IDE/provider client integrator" + ], + "evidence_keys": [ + "native-client-telemetry" + ], + "next_action_id": "native-client-telemetry-source-check-adoption_sample_count", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "phase_queue_blocked_count": 2, + "phase_queue_counts_as_completion": false, + "submission_state": { + "status": "missing", + "path": "evidence/world_class/submissions/native-client-telemetry.json", + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, + "ledger_reviewer_approved": false, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + } + } + ], + "coordination_plan": [ + { + "step_id": "prepare-evidence-session", + "phase": "prepare", + "evidence_key": "", + "owner": "assistant + user", + "requires_user_input": true, + "user_action": "Confirm provider access, reviewer availability, target client path, and telemetry client path before collection starts.", + "assistant_action": "Run preflight and prepare submission drafts without accepting them as evidence.", + "external_dependency": "User-selected provider, reviewer, and real client surfaces.", + "command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "pass_condition": "Preflight lists the same pending evidence keys and no credential values are printed.", + "artifact": "reports/world_class_evidence_preflight.json", + "privacy_boundary": "Drafts and preflight are planning artifacts only; they do not count as evidence.", + "counts_as_completion": false + }, + { + "step_id": "collect-provider-holdout", + "phase": "provider-holdout", + "evidence_key": "provider-holdout", + "owner": "assistant + operator with provider credentials", + "requires_user_input": true, + "user_action": "Provide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.", + "assistant_action": "Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.", + "external_dependency": "Valid provider credentials and a live provider endpoint.", + "command": "python3 scripts/yao.py output-exec . --provider-runner --provider-model --timeout-seconds 60", + "pass_condition": "reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.", + "artifact": "reports/output_execution_runs.json", + "privacy_boundary": "Commit aggregate metadata only; do not commit API keys, raw prompts, raw outputs, or provider request payloads.", + "counts_as_completion": false + }, + { + "step_id": "collect-human-adjudication", + "phase": "human-review", + "evidence_key": "human-adjudication", + "owner": "human reviewer + assistant", + "requires_user_input": true, + "user_action": "Open the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved.", + "assistant_action": "Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet.", + "external_dependency": "A real human reviewer who can make blind A/B judgments.", + "command": "python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .", + "pass_condition": "reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true.", + "artifact": "reports/output_review_adjudication.json", + "privacy_boundary": "Store reviewer choices, reasons, hashes, and attestations; do not store raw prompts or answer-key exposure before decisions.", + "counts_as_completion": false + }, + { + "step_id": "collect-native-permission-enforcement", + "phase": "native-permission", + "evidence_key": "native-permission-enforcement", + "owner": "target client or installer integrator + assistant", + "requires_user_input": true, + "user_action": "Select a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback.", + "assistant_action": "Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet.", + "external_dependency": "A real target runtime, extension, client, or installer guard that can block undeclared capabilities.", + "command": "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "pass_condition": "reports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0.", + "artifact": "reports/runtime_permission_probes.json", + "privacy_boundary": "Do not mark metadata fallback as native enforcement; keep residual risks visible for fallback targets.", + "counts_as_completion": false + }, + { + "step_id": "collect-native-client-telemetry", + "phase": "native-telemetry", + "evidence_key": "native-client-telemetry", + "owner": "real client integrator + assistant", + "requires_user_input": true, + "user_action": "Install the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event.", + "assistant_action": "Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet.", + "external_dependency": "A real client process that emits metadata-only telemetry through the native host.", + "command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source external", + "pass_condition": "reports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0.", + "artifact": "reports/adoption_drift_report.json", + "privacy_boundary": "Telemetry must remain metadata-only; do not store raw prompts, outputs, transcripts, notes, or messages.", + "counts_as_completion": false + }, + { + "step_id": "review-and-release-gate", + "phase": "release-gate", + "evidence_key": "", + "owner": "assistant + ledger reviewer", + "requires_user_input": true, + "user_action": "Approve only validated evidence packets and confirm the release wording after the claim guard passes.", + "assistant_action": "Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish.", + "external_dependency": "Accepted evidence packets for every pending world-class key.", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-test", + "pass_condition": "Ledger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes.", + "artifact": "reports/world_class_evidence_ledger.json", + "privacy_boundary": "Release evidence may include aggregate reports and hashes only; raw content and credentials remain out of git.", + "counts_as_completion": false + } + ], + "release_gate": { + "ready": false, + "decision": "blocked-until-evidence-accepted", + "check_count": 5, + "blocked_count": 4, + "checks": [ + { + "key": "world_class_ledger_ready", + "label": "World-class ledger ready", + "passed": false, + "current": "evidence-pending", + "expected": "ready_to_claim_world_class == true", + "artifact": "reports/world_class_evidence_ledger.json" + }, + { + "key": "claim_guard_clean", + "label": "Claim guard clean", + "passed": false, + "current": "violations 0; ledger ready False", + "expected": "violation_count == 0 and ledger_ready_to_claim_world_class == true", + "artifact": "reports/world_class_claim_guard.json" + }, + { + "key": "benchmark_public_claim_ready", + "label": "Benchmark public claim ready", + "passed": false, + "current": "public_claim_ready False", + "expected": "public_claim_ready == true", + "artifact": "reports/benchmark_reproducibility.json" + }, + { + "key": "review_studio_clean", + "label": "Review Studio clean", + "passed": false, + "current": "blockers 0; warnings 3", + "expected": "blocker_count == 0 and warning_count == 0", + "artifact": "reports/review-studio.json" + }, + { + "key": "evidence_consistency_clean", + "label": "Evidence consistency clean", + "passed": true, + "current": "consistent", + "expected": "decision == consistent and fail_count == 0", + "artifact": "reports/evidence_consistency.json" + } + ], + "blocked_checks": [ + { + "key": "world_class_ledger_ready", + "label": "World-class ledger ready", + "passed": false, + "current": "evidence-pending", + "expected": "ready_to_claim_world_class == true", + "artifact": "reports/world_class_evidence_ledger.json" + }, + { + "key": "claim_guard_clean", + "label": "Claim guard clean", + "passed": false, + "current": "violations 0; ledger ready False", + "expected": "violation_count == 0 and ledger_ready_to_claim_world_class == true", + "artifact": "reports/world_class_claim_guard.json" + }, + { + "key": "benchmark_public_claim_ready", + "label": "Benchmark public claim ready", + "passed": false, + "current": "public_claim_ready False", + "expected": "public_claim_ready == true", + "artifact": "reports/benchmark_reproducibility.json" + }, + { + "key": "review_studio_clean", + "label": "Review Studio clean", + "passed": false, + "current": "blockers 0; warnings 3", + "expected": "blocker_count == 0 and warning_count == 0", + "artifact": "reports/review-studio.json" + } + ], + "final_manual_check": "Run make ci-test in a clean worktree and verify GitHub Actions before converting the PR out of Draft.", + "counts_as_completion": false + }, + "repair_checklist": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ], + "phase_queue": [ + { + "phase": "unblock-access", + "label": "Unblock access", + "priority": 20, + "status": "blocked", + "blocked_count": 4, + "row_count": 4, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "operator with provider credentials", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout" + ], + "next_action_id": "human-adjudication-precheck-human-reviewer", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-precheck-human-reviewer", + "evidence_key": "human-adjudication", + "repair_type": "precheck", + "target": "human-reviewer", + "phase": "unblock-access", + "priority": 20, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Required human precheck is human-required.", + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-precheck-external-client", + "evidence_key": "native-client-telemetry", + "repair_type": "precheck", + "target": "external-client", + "phase": "unblock-access", + "priority": 20, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-precheck-native-guard", + "evidence_key": "native-permission-enforcement", + "repair_type": "precheck", + "target": "native-guard", + "phase": "unblock-access", + "priority": 20, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Required external precheck is external-required.", + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + }, + { + "action_id": "provider-holdout-precheck-provider-api-key", + "evidence_key": "provider-holdout", + "repair_type": "precheck", + "target": "provider-api-key", + "phase": "unblock-access", + "priority": 20, + "owner": "operator with provider credentials", + "status": "blocked", + "blocking_reason": "Required env_any precheck is missing.", + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "verification_command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "prechecks", + "counts_as_completion": false + } + ] + }, + { + "phase": "collect-source", + "label": "Collect source", + "priority": 40, + "status": "blocked", + "blocked_count": 8, + "row_count": 8, + "owners": [ + "Browser/Chrome/IDE/provider client integrator", + "human reviewer", + "target client or installer integrator" + ], + "evidence_keys": [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement" + ], + "next_action_id": "human-adjudication-source-check-blind_review_attested", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "counts_as_completion": false, + "rows": [ + { + "action_id": "human-adjudication-source-check-blind_review_attested", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "blind_review_attested", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-judgment_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "judgment_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '==pair_count'.", + "next_action": "Every pair needs one valid human judgment.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-pending_count", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "pending_count", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value 5 does not satisfy '==0'.", + "next_action": "Record a reviewer choice and reason for every pair.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-ready_for_human_evidence", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "ready_for_human_evidence", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "human-adjudication-source-check-reviewer_metadata_present", + "evidence_key": "human-adjudication", + "repair_type": "source-check", + "target": "reviewer_metadata_present", + "phase": "collect-source", + "priority": 40, + "owner": "human reviewer", + "status": "blocked", + "blocking_reason": "Current value False does not satisfy 'true'.", + "next_action": "Record reviewer and reviewed_at before adjudication can count.", + "verification_command": "python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-adoption_sample_count", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "adoption_sample_count", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Telemetry must include adoption outcome evidence.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-client-telemetry-source-check-external_source_events", + "evidence_key": "native-client-telemetry", + "repair_type": "source-check", + "target": "external_source_events", + "phase": "collect-source", + "priority": 40, + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Import at least one metadata-only event from a real client.", + "verification_command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + }, + { + "action_id": "native-permission-enforcement-source-check-native_enforcement_count", + "evidence_key": "native-permission-enforcement", + "repair_type": "source-check", + "target": "native_enforcement_count", + "phase": "collect-source", + "priority": 40, + "owner": "target client or installer integrator", + "status": "blocked", + "blocking_reason": "Current value 0 does not satisfy '>0'.", + "next_action": "Collect real target-client or external runtime guard proof.", + "verification_command": "python3 scripts/yao.py runtime-permissions . --package-dir dist && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "source": "source_checklist", + "counts_as_completion": false + } + ] + } + ], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "preflight": "reports/world_class_evidence_preflight.json", + "submission_review": "reports/world_class_submission_review.json", + "claim_guard": "reports/world_class_claim_guard.json" + }, + "artifacts": { + "json": "reports/world_class_operator_runbook.json", + "markdown": "reports/world_class_operator_runbook.md", + "html": "reports/world_class_operator_runbook.html" + } +} diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md new file mode 100644 index 00000000..a69f6e0f --- /dev/null +++ b/reports/world_class_operator_runbook.md @@ -0,0 +1,408 @@ +# World-Class Operator Runbook + +Generated at: `2026-06-17` + +## Summary + +- decision: `collect-evidence` +- ready to claim world-class: `false` +- runbook counts as completion: `false` +- evidence items: `4` +- pending: `4` +- awaiting submission: `4` +- ready for ledger review: `0` +- phase queue: `2` blocked / `2` phases +- phase queue rows: `12` +- phase queue counts as completion: `false` +- coordination steps: `6` user-required / `6` total +- coordination pending keys: `human-adjudication, native-client-telemetry, native-permission-enforcement, provider-holdout` +- coordination counts as completion: `false` +- release gate ready: `false` +- release gate blocked checks: `4` / `5` +- release gate counts as completion: `false` + +This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true. + +## Fast Path + +1. Run the real external or human work for one evidence item. +2. Generate the matching submission draft. +3. Replace template-only fields with aggregate evidence and provenance. +4. Validate intake, review the queue, refresh the ledger, then run the claim guard. + +## Coordination Plan + +| Step | Evidence | Owner | Needs user | User action | Assistant action | Command | Pass condition | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `prepare-evidence-session` | `all` | assistant + user | `true` | Confirm provider access, reviewer availability, target client path, and telemetry client path before collection starts. | Run preflight and prepare submission drafts without accepting them as evidence. | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts` | Preflight lists the same pending evidence keys and no credential values are printed. | +| `collect-provider-holdout` | `provider-holdout` | assistant + operator with provider credentials | `true` | Provide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use. | Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet. | `python3 scripts/yao.py output-exec . --provider-runner --provider-model --timeout-seconds 60` | reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0. | +| `collect-human-adjudication` | `human-adjudication` | human reviewer + assistant | `true` | Open the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved. | Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet. | `python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .` | reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true. | +| `collect-native-permission-enforcement` | `native-permission-enforcement` | target client or installer integrator + assistant | `true` | Select a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback. | Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet. | `python3 scripts/yao.py runtime-permissions . --package-dir dist` | reports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0. | +| `collect-native-client-telemetry` | `native-client-telemetry` | real client integrator + assistant | `true` | Install the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event. | Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet. | `python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source external` | reports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0. | +| `review-and-release-gate` | `all` | assistant + ledger reviewer | `true` | Approve only validated evidence packets and confirm the release wording after the claim guard passes. | Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish. | `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-test` | Ledger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes. | + +## Phase Queue + +| Phase | Status | Rows | Blocked | Owners | Next action | Verify | +| --- | --- | ---: | ---: | --- | --- | --- | +| `unblock-access` | `blocked` | `4` | `4` | Browser/Chrome/IDE/provider client integrator, human reviewer, operator with provider credentials, target client or installer integrator | Assign a real reviewer identity before claiming human adjudication. | `python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | +| `collect-source` | `blocked` | `8` | `8` | Browser/Chrome/IDE/provider client integrator, human reviewer, target client or installer integrator | Set reviewer_attestation only after choices are completed before opening the answer key. | `python3 scripts/yao.py output-review && python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions` | + +## Evidence Items + +| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner | +| --- | --- | --- | --- | ---: | --- | --- | +| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `0` | none | operator with provider credentials | +| `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `5` | Record a reviewer choice and reason for every pair. | human reviewer | +| `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator | +| `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | + +## Provider Holdout + +- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata. +- blocking reason: No evidence packet has been submitted for review. +- blocked source checks: `0` +- repair rows: `1` blocked +- phase queue: `1` blocked phases +- submission: `evidence/world_class/submissions/provider-holdout.json` +- template: `evidence/world_class/templates/provider-holdout.intake.json` + +### Phase Queue + +| Phase | Status | Rows | Blocked | Next action | +| --- | --- | ---: | ---: | --- | +| `unblock-access` | `blocked` | `1` | `1` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | + +### Source Runbook + +- Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. +- For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60 +- For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120 +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- provider-backed model run +- observed timing +- observed token metadata + +### Success Checks + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +### Evidence Artifacts + +- reports/output_execution_runs.json +- reports/output_execution_runs.md +- reports/skill_os2_audit.json +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/provider-holdout.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +### Next Source Actions + +- No blocked source checks. + +### Source Evidence Snapshot + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Provider model run | `10` | `>0` | `pass` | Run provider-backed output-exec with real credentials. | +| Timing observed | `10` | `>0` | `pass` | Provider execution should record timing metadata. | +| Token usage observed | `10` | `>0` | `pass` | Provider execution should return non-estimated token usage. | + +## Human Adjudication + +- objective: Record real blind A/B reviewer decisions before claiming human output review completion. +- blocking reason: No evidence packet has been submitted for review. +- blocked source checks: `5` +- repair rows: `6` blocked +- phase queue: `2` blocked phases +- submission: `evidence/world_class/submissions/human-adjudication.json` +- template: `evidence/world_class/templates/human-adjudication.intake.json` + +### Phase Queue + +| Phase | Status | Rows | Blocked | Next action | +| --- | --- | ---: | ---: | --- | +| `unblock-access` | `blocked` | `1` | `1` | Assign a real reviewer identity before claiming human adjudication. | +| `collect-source` | `blocked` | `5` | `5` | Set reviewer_attestation only after choices are completed before opening the answer key. | + +### Source Runbook + +- python3 scripts/yao.py output-review-kit --write-template +- Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key. +- python3 scripts/adjudicate_output_review.py --write-template +- Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only. +- python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication +- python3 scripts/yao.py output-review +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- real reviewer identity +- blind A/B decisions +- answer key unopened until decisions exist + +### Success Checks + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/output_review_adjudication.json summary.reviewer_metadata_present is true +- reports/output_review_adjudication.json summary.blind_review_attested is true +- reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json +- reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text +- reports/output_review_adjudication.json summary.ready_for_human_evidence is true +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Reviewer reasons must be rubric-based and must not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. +- The adjudication evidence stores prompt_sha256 instead of raw prompt text. +- The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged. +- Keep the answer key separate until after decisions are recorded. + +### Evidence Artifacts + +- reports/output_blind_review_pack.md +- reports/output_review_kit.md +- reports/output_review_decisions.json +- reports/output_review_adjudication.json +- reports/output_review_adjudication.md +- scripts/import_output_review_decisions.py +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/human-adjudication.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +### Next Source Actions + +- Record a reviewer choice and reason for every pair. +- Every pair needs one valid human judgment. +- Record reviewer and reviewed_at before adjudication can count. +- Set reviewer_attestation only after choices are completed before opening the answer key. +- Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. + +### Source Evidence Snapshot + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Review pairs exist | `5` | `>0` | `pass` | Generate the blind A/B review pack. | +| No pending decisions | `5` | `==0` | `blocked` | Record a reviewer choice and reason for every pair. | +| Judgments complete | `0` | `==pair_count` | `blocked` | Every pair needs one valid human judgment. | +| No invalid decisions | `0` | `==0` | `pass` | Fix malformed winner/confidence entries. | +| Reviewer metadata | `False` | `true` | `blocked` | Record reviewer and reviewed_at before adjudication can count. | +| Reason required | `True` | `true` | `pass` | Keep reason mandatory for every imported or direct reviewer decision. | +| Blind review attested | `False` | `true` | `blocked` | Set reviewer_attestation only after choices are completed before opening the answer key. | +| Raw content attested | `True` | `true` | `pass` | Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content. | +| Raw content blocked | `False` | `false` | `pass` | Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages. | +| Human evidence ready | `False` | `true` | `blocked` | Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints. | + +## Native Permission Enforcement + +- objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities. +- blocking reason: No evidence packet has been submitted for review. +- blocked source checks: `1` +- repair rows: `2` blocked +- phase queue: `2` blocked phases +- submission: `evidence/world_class/submissions/native-permission-enforcement.json` +- template: `evidence/world_class/templates/native-permission-enforcement.intake.json` + +### Phase Queue + +| Phase | Status | Rows | Blocked | Next action | +| --- | --- | ---: | ---: | --- | +| `unblock-access` | `blocked` | `1` | `1` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | +| `collect-source` | `blocked` | `1` | `1` | Collect real target-client or external runtime guard proof. | + +### Source Runbook + +- Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities. +- Update the generated target adapter only when the guard is actually enforced by that target. +- python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip +- python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation +- python3 scripts/yao.py runtime-permissions . --package-dir dist +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- real target client or external installer runtime guard +- native enforcement flag or externally accepted guard proof +- residual risk retained for fallback targets + +### Success Checks + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +### Evidence Artifacts + +- dist/targets/*/adapter.json +- reports/runtime_permission_probes.json +- reports/runtime_permission_probes.md +- reports/install_simulation.json +- reports/install_simulation.md +- security/permission_policy.json +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/native-permission-enforcement.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +### Next Source Actions + +- Collect real target-client or external runtime guard proof. + +### Source Evidence Snapshot + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| Native enforcement | `0` | `>0` | `blocked` | Collect real target-client or external runtime guard proof. | +| Probe failures | `0` | `==0` | `pass` | Runtime permission probes must stay clean. | +| Installer support | `True` | `true` | `pass` | Installer enforcement is supporting evidence, not native proof. | + +## Native Client Telemetry + +- objective: Import production metadata-only events from a real external client into the local drift loop. +- blocking reason: No evidence packet has been submitted for review. +- blocked source checks: `2` +- repair rows: `3` blocked +- phase queue: `2` blocked phases +- submission: `evidence/world_class/submissions/native-client-telemetry.json` +- template: `evidence/world_class/templates/native-client-telemetry.intake.json` + +### Phase Queue + +| Phase | Status | Rows | Blocked | Next action | +| --- | --- | ---: | ---: | --- | +| `unblock-access` | `blocked` | `1` | `1` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | +| `collect-source` | `blocked` | `2` | `2` | Telemetry must include adoption outcome evidence. | + +### Source Runbook + +- python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:/// +- Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event. +- python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl +- python3 scripts/yao.py skill-atlas --workspace-root . +- python3 scripts/yao.py skill-os2-audit . --generated-at +- Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields. +- python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions + +### Commands + +- prepare_submission: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions` +- validate_intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review_queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh_ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard_claim: `python3 scripts/yao.py world-class-claim-guard .` + +### Must Collect + +- real external client source +- metadata-only event +- local-first import path + +### Success Checks + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. + +### Evidence Artifacts + +- reports/adoption_drift_report.json +- reports/adoption_drift_report.md +- reports/telemetry_hook_recipes.json +- scripts/telemetry_native_host.py +- evidence/world_class/intake.schema.json +- evidence/world_class/templates/native-client-telemetry.intake.json +- reports/world_class_evidence_intake.json +- reports/world_class_evidence_intake.md + +### Next Source Actions + +- Import at least one metadata-only event from a real client. +- Telemetry must include adoption outcome evidence. + +### Source Evidence Snapshot + +| Check | Current | Expected | Status | Next action | +| --- | --- | --- | --- | --- | +| External events | `0` | `>0` | `blocked` | Import at least one metadata-only event from a real client. | +| Adoption sample | `0` | `>0` | `blocked` | Telemetry must include adoption outcome evidence. | +| Raw content blocked | `False` | `false` | `pass` | Telemetry must stay metadata-only. | + +## Release Gate + +- decision: `blocked-until-evidence-accepted` +- ready: `false` +- blocked checks: `4` / `5` +- counts as completion: `false` +- final manual check: Run make ci-test in a clean worktree and verify GitHub Actions before converting the PR out of Draft. + +| Check | Current | Expected | Status | Artifact | +| --- | --- | --- | --- | --- | +| World-class ledger ready | `evidence-pending` | `ready_to_claim_world_class == true` | `blocked` | `reports/world_class_evidence_ledger.json` | +| Claim guard clean | `violations 0; ledger ready False` | `violation_count == 0 and ledger_ready_to_claim_world_class == true` | `blocked` | `reports/world_class_claim_guard.json` | +| Benchmark public claim ready | `public_claim_ready False` | `public_claim_ready == true` | `blocked` | `reports/benchmark_reproducibility.json` | +| Review Studio clean | `blockers 0; warnings 3` | `blocker_count == 0 and warning_count == 0` | `blocked` | `reports/review-studio.json` | +| Evidence consistency clean | `consistent` | `decision == consistent and fail_count == 0` | `pass` | `reports/evidence_consistency.json` | + +## Boundary + +- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion. +- Valid intake means ready for submission review; ledger review still requires passing source evidence. +- The world-class ledger and claim guard remain the source of truth. diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json new file mode 100644 index 00000000..50689d71 --- /dev/null +++ b/reports/world_class_submission_review.json @@ -0,0 +1,442 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-17", + "skill_dir": ".", + "decision": "awaiting-submissions", + "ready_to_claim_world_class": false, + "review_item_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "valid_packet_source_incomplete_count": 0, + "ready_for_ledger_review_count": 0, + "fix_submission_count": 0, + "unmatched_submission_count": 0, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "review_counts_submission_as_completion": false, + "summary": { + "review_item_count": 4, + "accepted_count": 0, + "awaiting_submission_count": 4, + "valid_packet_source_incomplete_count": 0, + "ready_for_ledger_review_count": 0, + "fix_submission_count": 0, + "unmatched_submission_count": 0, + "invalid_submission_count": 0, + "source_check_count": 19, + "source_pass_count": 11, + "source_blocked_count": 8, + "ready_to_claim_world_class": false, + "review_counts_submission_as_completion": false, + "decision": "awaiting-submissions" + }, + "report_contract": { + "schema_version": "1.0", + "contract": "world-class-submission-review", + "top_level_mirrors_summary": true, + "summary_fields": [ + "decision", + "ready_to_claim_world_class", + "review_item_count", + "accepted_count", + "awaiting_submission_count", + "valid_packet_source_incomplete_count", + "ready_for_ledger_review_count", + "fix_submission_count", + "unmatched_submission_count", + "invalid_submission_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "review_counts_submission_as_completion" + ], + "source_of_truth": "summary" + }, + "submissions": { + "directory": "evidence/world_class/submissions", + "review_counts_submission_as_completion": false + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": true, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "model_executed_count": 10, + "timing_observed_count": 10, + "token_observed_count": 10, + "accepted": true + }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": true, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 3, + "source_blocked_count": 0, + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "reviewer_metadata_present": false, + "reason_required": true, + "blind_review_attested": false, + "raw_content_excluded_attested": true, + "reviewer_reason_required_attested": true, + "ready_for_human_evidence": false, + "raw_content_allowed": false, + "raw_content_path_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice and reason for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "human-adjudication", + "label": "Reviewer metadata", + "field": "reviewer_metadata_present", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Record reviewer and reviewed_at before adjudication can count." + }, + { + "evidence_key": "human-adjudication", + "label": "Reason required", + "field": "reason_required", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Keep reason mandatory for every imported or direct reviewer decision." + }, + { + "evidence_key": "human-adjudication", + "label": "Blind review attested", + "field": "blind_review_attested", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Set reviewer_attestation only after choices are completed before opening the answer key." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content attested", + "field": "raw_content_excluded_attested", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content." + }, + { + "evidence_key": "human-adjudication", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages." + }, + { + "evidence_key": "human-adjudication", + "label": "Human evidence ready", + "field": "ready_for_human_evidence", + "expected": "true", + "actual": false, + "status": "blocked", + "source_accepted": false, + "next_action": "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints." + } + ], + "source_check_count": 10, + "source_pass_count": 5, + "source_blocked_count": 5, + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded." + ], + "next_action": "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "installer_enforcement_pass_count": 4, + "installer_permission_failure_count": 0, + "installer_enforcement_ready": true, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "ledger_status": "pending", + "submission_status": "missing", + "intake_status": "missing", + "source_accepted": false, + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "unmatched_submissions": [], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json" + }, + "artifacts": { + "json": "reports/world_class_submission_review.json", + "markdown": "reports/world_class_submission_review.md" + } +} diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md new file mode 100644 index 00000000..5734e5db --- /dev/null +++ b/reports/world_class_submission_review.md @@ -0,0 +1,178 @@ +# World-Class Submission Review + +Generated at: `2026-06-17` + +## Summary + +- decision: `awaiting-submissions` +- review items: `4` +- accepted: `0` +- awaiting submission: `4` +- valid packet but source incomplete: `0` +- ready for ledger review: `0` +- fix submission: `0` +- unmatched submissions: `0` +- ready to claim world-class: `false` +- review counts submission as completion: `false` + +This report is a read-only reviewer queue. It does not accept evidence or make world-class completion true. + +## Queue + +| Evidence | Review state | Intake | Source accepted | Submission | Next action | +| --- | --- | --- | --- | --- | --- | +| `provider-holdout` | `awaiting-submission` | `missing` | `true` | `missing` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `human-adjudication` | `awaiting-submission` | `missing` | `false` | `missing` | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | +| `native-permission-enforcement` | `awaiting-submission` | `missing` | `false` | `missing` | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | +| `native-client-telemetry` | `awaiting-submission` | `missing` | `false` | `missing` | Install a real client against the native host and import production metadata-only events. | + +## Details + +### Provider Holdout + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `true` +- submission path: `evidence/world_class/submissions/provider-holdout.json` + +#### Source Checks + +- Provider model run: 10 / >0 => pass +- Timing observed: 10 / >0 => pass +- Token usage observed: 10 / >0 => pass + +#### Completion Assertions + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +### Human Adjudication + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/human-adjudication.json` + +#### Source Checks + +- Review pairs exist: 5 / >0 => pass +- No pending decisions: 5 / ==0 => blocked +- Judgments complete: 0 / ==pair_count => blocked +- No invalid decisions: 0 / ==0 => pass +- Reviewer metadata: False / true => blocked +- Reason required: True / true => pass +- Blind review attested: False / true => blocked +- Raw content attested: True / true => pass +- Raw content blocked: False / false => pass +- Human evidence ready: False / true => blocked + +#### Completion Assertions + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/output_review_adjudication.json summary.reviewer_metadata_present is true +- reports/output_review_adjudication.json summary.blind_review_attested is true +- reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json +- reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text +- reports/output_review_adjudication.json summary.ready_for_human_evidence is true +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Reviewer reasons must be rubric-based and must not include raw user data or private customer detail. +- The decision importer rejects raw prompt, output, transcript, message, and answer-key fields. +- The adjudication evidence stores prompt_sha256 instead of raw prompt text. +- The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged. +- Keep the answer key separate until after decisions are recorded. + +### Native Permission Enforcement + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/native-permission-enforcement.json` + +#### Source Checks + +- Native enforcement: 0 / >0 => blocked +- Probe failures: 0 / ==0 => pass +- Installer support: True / true => pass + +#### Completion Assertions + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +### Native Client Telemetry + +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. +- ledger status: `pending` +- submission status: `missing` +- intake status: `missing` +- source accepted: `false` +- submission path: `evidence/world_class/submissions/native-client-telemetry.json` + +#### Source Checks + +- External events: 0 / >0 => blocked +- Adoption sample: 0 / >0 => blocked +- Raw content blocked: False / false => pass + +#### Completion Assertions + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +#### Intake Errors + +- No intake errors. + +#### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. + +## Boundary + +- A valid submission packet is not accepted evidence by itself. +- Planned work, metadata fallback, pending human review, and local command-runner output still do not count. +- The world-class ledger remains the source of truth for `ready_to_claim_world_class`. diff --git a/schemas/adaptation-proposal.schema.json b/schemas/adaptation-proposal.schema.json new file mode 100644 index 00000000..2fbc999c --- /dev/null +++ b/schemas/adaptation-proposal.schema.json @@ -0,0 +1,226 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.local/schemas/adaptation-proposal.schema.json", + "title": "Yao Adaptation Proposal Report", + "type": "object", + "required": [ + "schema_version", + "ok", + "generated_at", + "skill_dir", + "pattern_count", + "proposal_count", + "apply_supported", + "failure_count", + "proposal_only", + "approval_required", + "writes_repository_files", + "approval_draft_supported", + "summary", + "proposal_contract", + "report_contract", + "proposals", + "artifacts" + ], + "properties": { + "schema_version": { + "const": "1.0" + }, + "ok": { + "type": "boolean" + }, + "generated_at": { + "type": "string" + }, + "skill_dir": { + "type": "string" + }, + "pattern_count": { + "type": "integer", + "minimum": 0 + }, + "proposal_count": { + "type": "integer", + "minimum": 0 + }, + "apply_supported": { + "type": "boolean" + }, + "failure_count": { + "type": "integer", + "minimum": 0 + }, + "proposal_only": { + "const": true + }, + "approval_required": { + "const": true + }, + "writes_repository_files": { + "const": false + }, + "approval_draft_supported": { + "const": true + }, + "summary": { + "type": "object", + "required": ["pattern_count", "proposal_count", "apply_supported"], + "properties": { + "pattern_count": { + "type": "integer", + "minimum": 0 + }, + "proposal_count": { + "type": "integer", + "minimum": 0 + }, + "apply_supported": { + "type": "boolean" + } + }, + "additionalProperties": true + }, + "proposal_contract": { + "type": "object", + "required": [ + "proposal_only", + "approval_required", + "writes_repository_files", + "allowlisted_targets_required", + "target_file_sha256_required_for_apply", + "approval_draft_supported", + "rollback_required_for_apply", + "apply_command_available" + ], + "properties": { + "proposal_only": { + "const": true + }, + "approval_required": { + "const": true + }, + "writes_repository_files": { + "const": false + }, + "allowlisted_targets_required": { + "const": true + }, + "target_file_sha256_required_for_apply": { + "const": true + }, + "approval_draft_supported": { + "const": true + }, + "rollback_required_for_apply": { + "const": true + }, + "apply_command_available": { + "type": "boolean" + } + }, + "additionalProperties": true + }, + "report_contract": { + "type": "object", + "required": [ + "schema_version", + "contract", + "top_level_mirrors_summary", + "top_level_mirrors_proposal_contract", + "summary_fields", + "proposal_contract_fields", + "source_of_truth" + ], + "properties": { + "schema_version": { + "const": "1.0" + }, + "contract": { + "const": "adaptation-proposals" + }, + "top_level_mirrors_summary": { + "const": true + }, + "top_level_mirrors_proposal_contract": { + "const": true + }, + "summary_fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "proposal_contract_fields": { + "type": "array", + "items": { + "type": "string" + } + }, + "source_of_truth": { + "type": "array", + "items": { + "enum": [ + "summary", + "proposal_contract" + ] + } + } + }, + "additionalProperties": true + }, + "proposals": { + "type": "array", + "items": { + "type": "object", + "required": [ + "proposal_id", + "status", + "requires_approval", + "write_allowed_without_approval", + "target_files", + "verification_commands", + "rollback_plan" + ], + "properties": { + "proposal_id": { + "type": "string", + "minLength": 1 + }, + "status": { + "const": "proposal-only" + }, + "requires_approval": { + "const": true + }, + "write_allowed_without_approval": { + "const": false + }, + "risk_level": { + "enum": ["low", "medium", "high"] + }, + "target_files": { + "type": "array", + "items": { + "type": "string" + } + }, + "verification_commands": { + "type": "array", + "items": { + "type": "string" + } + }, + "rollback_plan": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": true + } + }, + "artifacts": { + "type": "object" + } + }, + "additionalProperties": true +} diff --git a/schemas/skill-interpretation.schema.json b/schemas/skill-interpretation.schema.json new file mode 100644 index 00000000..7a295174 --- /dev/null +++ b/schemas/skill-interpretation.schema.json @@ -0,0 +1,115 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.local/schemas/skill-interpretation.schema.json", + "title": "Yao Skill Interpretation Report", + "type": "object", + "required": [ + "skill_summary", + "scorecard", + "capability_profile", + "principle_model", + "contract_boundary", + "quality_review", + "risk_governance", + "package_assets", + "iteration_roadmap", + "report_contract", + "interpretation_contract" + ], + "properties": { + "report_contract": { + "type": "object", + "required": [ + "schema_version", + "report_kind", + "html_report", + "json_report", + "default_language", + "languages", + "layout", + "nav_labels" + ], + "properties": { + "schema_version": { + "const": "2.0" + }, + "report_kind": { + "const": "skill-interpretation" + }, + "html_report": { + "const": "reports/skill-interpretation.html" + }, + "json_report": { + "const": "reports/skill-interpretation.json" + }, + "default_language": { + "const": "zh-CN" + }, + "languages": { + "type": "array", + "prefixItems": [ + { + "const": "zh-CN" + }, + { + "const": "en" + } + ], + "minItems": 2 + }, + "layout": { + "const": "kami-white-audit-v2" + }, + "nav_labels": { + "type": "array", + "minItems": 9, + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + }, + "interpretation_contract": { + "type": "object", + "required": [ + "schema_version", + "source_model", + "source_model_reused", + "overview_report", + "html_report", + "json_report", + "includes" + ], + "properties": { + "schema_version": { + "const": "2.0" + }, + "source_model": { + "const": "skill-overview-v2" + }, + "source_model_reused": { + "const": true + }, + "overview_report": { + "const": "reports/skill-overview.html" + }, + "html_report": { + "const": "reports/skill-interpretation.html" + }, + "json_report": { + "const": "reports/skill-interpretation.json" + }, + "includes": { + "type": "array", + "minItems": 10, + "items": { + "type": "string" + } + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true +} diff --git a/schemas/skillops-opportunity.schema.json b/schemas/skillops-opportunity.schema.json new file mode 100644 index 00000000..ed82453c --- /dev/null +++ b/schemas/skillops-opportunity.schema.json @@ -0,0 +1,112 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.local/schemas/skillops-opportunity.schema.json", + "title": "SkillOps Opportunity", + "type": "object", + "required": [ + "opportunity_id", + "pattern_id", + "title", + "opportunity_type", + "action_type", + "decision", + "priority", + "score", + "risk_level", + "requires_approval", + "write_allowed_without_approval", + "evidence_count", + "target_files", + "verification_commands" + ], + "properties": { + "opportunity_id": { + "type": "string", + "pattern": "^skillops-[a-f0-9]{10}$" + }, + "proposal_id": { + "type": "string" + }, + "pattern_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "opportunity_type": { + "type": "string" + }, + "action_type": { + "type": "string", + "enum": [ + "add_eval", + "agents_update", + "patch_existing_skill", + "report_only" + ] + }, + "decision": { + "type": "string", + "enum": [ + "ready_for_approval_review", + "proposal_review", + "observe_more_evidence", + "report_only", + "no_action" + ] + }, + "priority": { + "type": "string", + "enum": [ + "high", + "medium", + "low" + ] + }, + "score": { + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + "score_reasons": { + "type": "array", + "items": { + "type": "string" + } + }, + "policy_reason": { + "type": "string" + }, + "risk_level": { + "type": "string", + "enum": [ + "low", + "medium", + "high" + ] + }, + "requires_approval": { + "type": "boolean" + }, + "write_allowed_without_approval": { + "const": false + }, + "evidence_count": { + "type": "integer", + "minimum": 0 + }, + "target_files": { + "type": "array", + "items": { + "type": "string" + } + }, + "verification_commands": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "additionalProperties": true +} diff --git a/scripts/adaptation_patch_safety.py b/scripts/adaptation_patch_safety.py new file mode 100644 index 00000000..83a8ecd3 --- /dev/null +++ b/scripts/adaptation_patch_safety.py @@ -0,0 +1,76 @@ +"""Patch target and baseline hash checks for approval-gated adaptation.""" + +import hashlib +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Shared adaptation patch target parsing and target-file baseline verification helpers." + +BLOCKED_PATH_PARTS = {".git", "__pycache__", ".pytest_cache", "dist"} +ABSENT_FILE_SHA256 = "__absent__" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def target_file_sha256(skill_dir: Path, target_files: list[str]) -> dict[str, str]: + observed: dict[str, str] = {} + for target in target_files: + path = skill_dir / target + observed[target] = sha256_file(path) if path.exists() else ABSENT_FILE_SHA256 + return observed + + +def normalize_patch_path(raw: str) -> str | None: + token = raw.strip().split("\t", 1)[0].split(" ", 1)[0] + if token == "/dev/null": + return None + if token.startswith("a/") or token.startswith("b/"): + token = token[2:] + path = Path(token) + if path.is_absolute() or ".." in path.parts or any(part in BLOCKED_PATH_PARTS for part in path.parts): + raise ValueError(f"Unsafe patch path: {raw}") + if not token or token == ".": + raise ValueError(f"Empty patch path: {raw}") + return token + + +def patch_target_files(patch_text: str) -> list[str]: + targets: set[str] = set() + for line in patch_text.splitlines(): + if line.startswith("--- ") or line.startswith("+++ "): + raw = line[4:].strip() + path = normalize_patch_path(raw) + if path: + targets.add(path) + return sorted(targets) + + +def validate_target_file_sha256( + skill_dir: Path, + target_files: list[str], + expected_sha256: dict[str, Any], +) -> tuple[list[str], dict[str, str]]: + failures: list[str] = [] + observed: dict[str, str] = {} + for target in target_files: + path = skill_dir / target + expected = str(expected_sha256.get(target, "")) + if not expected: + failures.append(f"Approval target_file_sha256 missing target: {target}") + continue + if path.exists() and not path.is_file(): + failures.append(f"Patch target is not a file: {target}") + continue + current = sha256_file(path) if path.exists() else ABSENT_FILE_SHA256 + observed[target] = current + if current != expected: + failures.append(f"Target file baseline sha256 does not match approval ledger: {target}") + return failures, observed diff --git a/scripts/adaptation_report_contracts.py b/scripts/adaptation_report_contracts.py new file mode 100644 index 00000000..9114bd80 --- /dev/null +++ b/scripts/adaptation_report_contracts.py @@ -0,0 +1,111 @@ +"""Shared report contracts for approval-gated skill adaptation.""" + +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Shared adaptation approval and regression report contract decoration helpers." + +APPROVAL_SUMMARY_FIELDS = [ + "approval_count", + "active_approval_count", + "pending_review_count", + "applied_count", + "rollback_count", +] +APPROVAL_CONTRACT_FIELDS = [ + "approval_required", + "patch_sha256_required", + "allowlisted_targets_required", + "target_file_sha256_required", + "approval_draft_supported", + "dry_run_default", + "writes_repository_files_only_with_apply", + "rollback_required", +] +REGRESSION_SUMMARY_FIELDS = [ + "apply_supported", + "attempt_count", + "approval_draft_count", + "applied_count", + "dry_run_count", + "rollback_count", + "regression_run_count", + "regression_pass_count", + "failure_count", +] +APPLY_CONTRACT_FIELDS = [ + *APPROVAL_CONTRACT_FIELDS, + "safe_regression_commands_only", + "rollback_on_failure_default", +] +APPROVAL_CONTRACT = { + "approval_required": True, + "patch_sha256_required": True, + "allowlisted_targets_required": True, + "target_file_sha256_required": True, + "approval_draft_supported": True, + "dry_run_default": True, + "writes_repository_files_only_with_apply": True, + "rollback_required": True, +} +APPLY_CONTRACT = { + **APPROVAL_CONTRACT, + "safe_regression_commands_only": True, + "rollback_on_failure_default": True, +} + + +def top_level_mirrors( + summary: dict[str, Any], + contract: dict[str, Any], + summary_fields: list[str], + contract_fields: list[str], +) -> dict[str, Any]: + mirrored = {key: summary[key] for key in summary_fields if key in summary} + mirrored.update({key: contract[key] for key in contract_fields if key in contract}) + return mirrored + + +def report_contract(name: str, contract_key: str, summary_fields: list[str], contract_fields: list[str]) -> dict[str, Any]: + return { + "schema_version": "1.0", + "contract": name, + "top_level_mirrors_summary": True, + f"top_level_mirrors_{contract_key}": True, + "summary_fields": summary_fields, + f"{contract_key}_fields": contract_fields, + "source_of_truth": ["summary", contract_key], + } + + +def decorate_approval_ledger(ledger: dict[str, Any]) -> dict[str, Any]: + summary = ledger.get("summary", {}) if isinstance(ledger.get("summary"), dict) else {} + approval_contract = ( + ledger.get("approval_contract", {}) if isinstance(ledger.get("approval_contract"), dict) else {} + ) or dict(APPROVAL_CONTRACT) + ledger.update(top_level_mirrors(summary, approval_contract, APPROVAL_SUMMARY_FIELDS, APPROVAL_CONTRACT_FIELDS)) + ledger["approval_contract"] = approval_contract + ledger["report_contract"] = report_contract( + "adaptation-approval-ledger", + "approval_contract", + APPROVAL_SUMMARY_FIELDS, + APPROVAL_CONTRACT_FIELDS, + ) + return ledger + + +def decorate_regression_report(report: dict[str, Any]) -> dict[str, Any]: + summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {} + apply_contract = ( + report.get("apply_contract", {}) if isinstance(report.get("apply_contract"), dict) else {} + ) or dict(APPLY_CONTRACT) + report.update(top_level_mirrors(summary, apply_contract, REGRESSION_SUMMARY_FIELDS, APPLY_CONTRACT_FIELDS)) + report["apply_contract"] = apply_contract + report["report_contract"] = report_contract( + "adaptation-regression-report", + "apply_contract", + REGRESSION_SUMMARY_FIELDS, + APPLY_CONTRACT_FIELDS, + ) + return report diff --git a/scripts/adjudicate_output_review.py b/scripts/adjudicate_output_review.py index 759bc530..69ac465e 100644 --- a/scripts/adjudicate_output_review.py +++ b/scripts/adjudicate_output_review.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import hashlib import json from pathlib import Path from typing import Any @@ -59,6 +60,45 @@ def confidence_value(value: Any) -> tuple[float | None, str | None]: return round(parsed, 3), None +def prompt_sha256(pair: dict[str, Any]) -> str: + return hashlib.sha256(str(pair.get("prompt", "")).encode("utf-8")).hexdigest() + + +def canonical_sha256(value: Any) -> str: + encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def review_integrity(blind_pack: dict[str, Any]) -> dict[str, Any]: + pairs = blind_pack.get("pairs", []) + case_ids: list[str] = [] + prompt_hashes: dict[str, str] = {} + if isinstance(pairs, list): + for pair in pairs: + if not isinstance(pair, dict): + continue + case_id = str(pair.get("case_id", "")).strip() + if not case_id: + continue + case_ids.append(case_id) + prompt_hashes[case_id] = prompt_sha256(pair) + return { + "blind_pack_sha256": canonical_sha256(blind_pack), + "case_count": len(case_ids), + "case_ids": case_ids, + "prompt_sha256_by_case": prompt_hashes, + } + + +def default_reviewer_attestation() -> dict[str, Any]: + return { + "blind_review_completed_before_answer_key": False, + "answer_key_not_opened_before_decisions": False, + "raw_content_excluded": True, + "reviewer_reason_required": True, + } + + def answer_index(answer_key: dict[str, Any]) -> dict[str, dict[str, Any]]: answers = answer_key.get("answers", []) if not isinstance(answers, list): @@ -113,10 +153,13 @@ def build_decision_template(blind_pack: dict[str, Any]) -> dict[str, Any]: "schema_version": "1.0", "reviewer": "", "reviewed_at": "", + "review_integrity": review_integrity(blind_pack), + "reviewer_attestation": default_reviewer_attestation(), "decision_contract": { "winner_variant": "Use A or B after reading the blind review pack. Leave blank when pending.", "confidence": "Optional number from 0 to 1.", - "reason": "Short reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.", + "reason": "Required reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.", + "reviewer_attestation": "Set blind_review_completed_before_answer_key and answer_key_not_opened_before_decisions to true only after a real blind review.", }, "decisions": template_decisions, } @@ -127,6 +170,7 @@ def adjudicate_pair( pair: dict[str, Any], answer: dict[str, Any] | None, decision: dict[str, Any] | None, + reviewer_metadata_present: bool = True, ) -> tuple[dict[str, Any], list[str]]: failures: list[str] = [] expected = normalize_variant(answer.get("expected_winner_variant", "") if answer else "") @@ -137,11 +181,12 @@ def adjudicate_pair( { "case_id": case_id, "status": "pending", - "expected_winner_variant": expected, + "expected_winner_variant": "", + "expected_revealed": False, "reviewer_winner_variant": "", "confidence": None, "reason": "", - "prompt": str(pair.get("prompt", "")), + "prompt_sha256": prompt_sha256(pair), }, failures, ) @@ -156,43 +201,77 @@ def adjudicate_pair( { "case_id": case_id, "status": "pending", - "expected_winner_variant": expected, + "expected_winner_variant": "", + "expected_revealed": False, "reviewer_winner_variant": "", "confidence": confidence, "reason": reason, - "prompt": str(pair.get("prompt", "")), + "prompt_sha256": prompt_sha256(pair), }, failures, ) - if reviewer not in {"A", "B"}: + if expected not in {"A", "B"}: + status = "invalid" + elif reviewer not in {"A", "B"}: failures.append(f"{case_id}: winner_variant must be A or B") status = "invalid" elif confidence_failure: failures.append(f"{case_id}: {confidence_failure}") status = "invalid" + elif not reason: + failures.append(f"{case_id}: reason is required before answer key can be revealed") + status = "invalid" + elif not reviewer_metadata_present: + failures.append(f"{case_id}: reviewer and reviewed_at are required before answer key can be revealed") + status = "invalid" else: status = "match" if reviewer == expected else "disagree" + expected_revealed = status in {"match", "disagree"} return ( { "case_id": case_id, "status": status, - "expected_winner_variant": expected, + "expected_winner_variant": expected if expected_revealed else "", + "expected_revealed": expected_revealed, "reviewer_winner_variant": reviewer, "confidence": confidence, "reason": reason, - "prompt": str(pair.get("prompt", "")), + "prompt_sha256": prompt_sha256(pair), }, failures, ) -def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]: +def reviewer_attestation_state(decisions_payload: dict[str, Any]) -> dict[str, bool]: + attestation = decisions_payload.get("reviewer_attestation", {}) + if not isinstance(attestation, dict): + attestation = {} + blind_review_completed = attestation.get("blind_review_completed_before_answer_key") is True + answer_key_not_opened = attestation.get("answer_key_not_opened_before_decisions") is True + raw_content_excluded = attestation.get("raw_content_excluded") is True + reviewer_reason_required = attestation.get("reviewer_reason_required") is True + return { + "blind_review_attested": blind_review_completed and answer_key_not_opened, + "blind_review_completed_before_answer_key": blind_review_completed, + "answer_key_not_opened_before_decisions": answer_key_not_opened, + "raw_content_excluded_attested": raw_content_excluded, + "reviewer_reason_required_attested": reviewer_reason_required, + } + + +def build_summary( + pairs: list[dict[str, Any]], + failures: list[str], + reviewer_metadata_present: bool, + attestation_state: dict[str, bool], +) -> dict[str, Any]: pair_count = len(pairs) judgment_count = sum(1 for item in pairs if item["status"] in {"match", "disagree"}) pending_count = sum(1 for item in pairs if item["status"] == "pending") agreement_count = sum(1 for item in pairs if item["status"] == "match") disagreement_count = sum(1 for item in pairs if item["status"] == "disagree") invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid") + answer_revealed_count = sum(1 for item in pairs if item.get("expected_revealed")) agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None return { "pair_count": pair_count, @@ -201,12 +280,122 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, "agreement_count": agreement_count, "disagreement_count": disagreement_count, "invalid_decision_count": invalid_decision_count, + "answer_revealed_count": answer_revealed_count, + "pending_answer_hidden_count": sum(1 for item in pairs if item["status"] in {"pending", "invalid"} and not item.get("expected_revealed")), "agreement_rate": agreement_rate, "needs_review": pending_count > 0, + "reviewer_metadata_present": reviewer_metadata_present, + "reason_required": True, + **attestation_state, + "ready_for_human_evidence": bool(pair_count) + and judgment_count == pair_count + and pending_count == 0 + and invalid_decision_count == 0 + and reviewer_metadata_present + and attestation_state["blind_review_attested"] + and attestation_state["raw_content_excluded_attested"] + and attestation_state["reviewer_reason_required_attested"] + and len(failures) == 0, "failure_count": len(failures), } +def checklist_readiness(pair: dict[str, Any]) -> tuple[str, str]: + status = pair.get("status") + if status in {"match", "disagree"}: + return "adjudicated", "Reviewer decision is valid; answer key is revealed for this case." + if status == "invalid": + return "fix-decision", "Reviewer decision exists but failed validation; answer key remains hidden." + return "awaiting-decision", "Reviewer has not selected A or B yet; answer key remains hidden." + + +def build_reviewer_checklist( + pairs: list[dict[str, Any]], + blind_pack_path: Path, + decisions_path: Path, +) -> list[dict[str, Any]]: + checklist = [] + for pair in pairs: + readiness, blocking_reason = checklist_readiness(pair) + checklist.append( + { + "case_id": pair.get("case_id", ""), + "readiness": readiness, + "blocking_reason": blocking_reason, + "status": pair.get("status", "pending"), + "reviewer_winner_variant": pair.get("reviewer_winner_variant", ""), + "answer_key_visible": bool(pair.get("expected_revealed")), + "prompt_sha256": pair.get("prompt_sha256", ""), + "blind_pack_path": display_path(blind_pack_path), + "decisions_path": display_path(decisions_path), + "commands": { + "prepare_review_kit": "python3 scripts/yao.py output-review-kit", + "write_template": "python3 scripts/adjudicate_output_review.py --write-template", + "import_decisions": "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "adjudicate": "python3 scripts/yao.py output-review", + "refresh_review_studio": "python3 scripts/yao.py review-studio .", + }, + "required_fields": { + "winner_variant": "A or B after reading only the blind review pack.", + "confidence": "Optional number from 0 to 1.", + "reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.", + "reviewer": "Human reviewer name or review group at the decision-file top level.", + "reviewed_at": "Review date or timestamp at the decision-file top level.", + "reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.", + "reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded.", + }, + "privacy_contract": [ + "Do not paste raw private user data into the decision reason.", + "Do not open the answer key before reviewer choices are recorded.", + "Leave winner_variant blank when the reviewer is not ready to decide.", + ], + } + ) + return checklist + + +def add_checklist_summary(summary: dict[str, Any], checklist: list[dict[str, Any]]) -> dict[str, Any]: + enriched = dict(summary) + enriched["reviewer_checklist_count"] = len(checklist) + enriched["reviewer_checklist_pending_count"] = sum(1 for item in checklist if item["readiness"] == "awaiting-decision") + enriched["reviewer_checklist_invalid_count"] = sum(1 for item in checklist if item["readiness"] == "fix-decision") + enriched["reviewer_checklist_ready_count"] = sum(1 for item in checklist if item["readiness"] == "adjudicated") + return enriched + + +def render_reviewer_checklist(checklist: list[dict[str, Any]]) -> list[str]: + lines = [ + "## Reviewer Checklist", + "", + "| Case | Readiness | Answer key | Decision file |", + "| --- | --- | --- | --- |", + ] + if not checklist: + lines.append("| `none` | `adjudicated` | n/a | none |") + return lines + for item in checklist: + answer_key = "visible" if item.get("answer_key_visible") else "hidden" + lines.append( + f"| `{item['case_id']}` | `{item['readiness']}` | `{answer_key}` | `{item['decisions_path']}` |" + ) + for item in checklist: + lines.extend(["", f"### {item['case_id']}", ""]) + lines.append(f"- readiness: `{item['readiness']}`") + lines.append(f"- blocking reason: {item['blocking_reason']}") + lines.append(f"- answer key visible: `{str(item['answer_key_visible']).lower()}`") + lines.append(f"- blind pack: `{item['blind_pack_path']}`") + lines.append(f"- decisions: `{item['decisions_path']}`") + lines.extend(["", "#### Commands", ""]) + for label, command in item.get("commands", {}).items(): + lines.append(f"- {label}: `{command}`") + lines.extend(["", "#### Required Fields", ""]) + for label, description in item.get("required_fields", {}).items(): + lines.append(f"- {label}: {description}") + lines.extend(["", "#### Privacy Contract", ""]) + lines.extend(f"- {contract}" for contract in item.get("privacy_contract", [])) + return lines + + def render_markdown(payload: dict[str, Any]) -> str: summary = payload["summary"] lines = [ @@ -219,6 +408,13 @@ def render_markdown(payload: dict[str, Any]) -> str: f"- Pending: `{summary['pending_count']}`", f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`", f"- Invalid decisions: `{summary['invalid_decision_count']}`", + f"- Answer keys revealed: `{summary['answer_revealed_count']}`", + f"- Pending/invalid answers hidden: `{summary['pending_answer_hidden_count']}`", + f"- Reviewer checklist: `{summary['reviewer_checklist_ready_count']}` ready / `{summary['reviewer_checklist_count']}` total", + f"- Reviewer metadata present: `{str(summary['reviewer_metadata_present']).lower()}`", + f"- Blind review attested: `{str(summary['blind_review_attested']).lower()}`", + f"- Raw content excluded: `{str(summary['raw_content_excluded_attested']).lower()}`", + f"- Ready for human evidence: `{str(summary['ready_for_human_evidence']).lower()}`", "", ] if summary["judgment_count"] == 0: @@ -227,6 +423,7 @@ def render_markdown(payload: dict[str, Any]) -> str: "No reviewer decisions recorded yet.", "", "Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.", + "Expected winners stay hidden until a valid reviewer decision is recorded.", "", ] ) @@ -241,14 +438,16 @@ def render_markdown(payload: dict[str, Any]) -> str: for item in payload["pairs"]: confidence = "" if item.get("confidence") is None else str(item["confidence"]) reason = str(item.get("reason", "")).replace("|", "\\|") or "" + expected = item.get("expected_winner_variant", "") if item.get("expected_revealed") else "hidden" lines.append( f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | " - f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |" + f"{expected} | {item['status']} | {confidence} | {reason} |" ) if payload.get("failures"): lines.extend(["", "## Failures", ""]) for failure in payload["failures"]: lines.append(f"- {failure}") + lines.extend(["", *render_reviewer_checklist(payload.get("reviewer_checklist", []))]) lines.extend( [ "", @@ -289,6 +488,11 @@ def adjudicate_output_review( answers_by_id = answer_index(answer_key) decisions_by_id, index_failures = decision_index(decisions_payload) failures.extend(index_failures) + reviewer_metadata_present = bool( + str(decisions_payload.get("reviewer", "")).strip() + and str(decisions_payload.get("reviewed_at", "")).strip() + ) + attestation_state = reviewer_attestation_state(decisions_payload) for case_id in decisions_by_id: if case_id not in pairs_by_id: @@ -296,17 +500,27 @@ def adjudicate_output_review( adjudicated_pairs: list[dict[str, Any]] = [] for case_id, pair in pairs_by_id.items(): - adjudicated, pair_failures = adjudicate_pair(case_id, pair, answers_by_id.get(case_id), decisions_by_id.get(case_id)) + adjudicated, pair_failures = adjudicate_pair( + case_id, + pair, + answers_by_id.get(case_id), + decisions_by_id.get(case_id), + reviewer_metadata_present=reviewer_metadata_present, + ) adjudicated_pairs.append(adjudicated) failures.extend(pair_failures) - summary = build_summary(adjudicated_pairs, failures) + summary = build_summary(adjudicated_pairs, failures, reviewer_metadata_present, attestation_state) + reviewer_checklist = build_reviewer_checklist(adjudicated_pairs, blind_pack_path, decisions_path) + summary = add_checklist_summary(summary, reviewer_checklist) payload = { "schema_version": "1.0", "ok": not failures, "summary": summary, "reviewer": decisions_payload.get("reviewer", ""), "reviewed_at": decisions_payload.get("reviewed_at", ""), + "review_integrity": review_integrity(blind_pack), + "reviewer_attestation": decisions_payload.get("reviewer_attestation", {}), "artifacts": { "blind_pack": display_path(blind_pack_path), "answer_key": display_path(answer_key_path), @@ -316,6 +530,7 @@ def adjudicate_output_review( }, "template_written": template_written, "pairs": adjudicated_pairs, + "reviewer_checklist": reviewer_checklist, "failures": failures, } output_json.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/apply_adaptation.py b/scripts/apply_adaptation.py new file mode 100644 index 00000000..52ecdc16 --- /dev/null +++ b/scripts/apply_adaptation.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Apply an approved adaptation patch with allowlisted targets and regression evidence.""" + +import argparse +import json +import os +import shlex +import subprocess +import sys +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any + +from adaptation_patch_safety import patch_target_files, sha256_file, target_file_sha256, validate_target_file_sha256 +from adaptation_report_contracts import APPLY_CONTRACT, APPROVAL_CONTRACT, decorate_approval_ledger, decorate_regression_report + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Approval-gated adaptive patch application with dry-run, allowlist, regression, and rollback evidence." + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def display_path(path: Path, skill_dir: Path) -> str: + try: + return str(path.resolve().relative_to(skill_dir.resolve())) + except ValueError: + return str(path) + + +def resolve_path(skill_dir: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else skill_dir / path + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def approved_entries(ledger: dict[str, Any]) -> list[dict[str, Any]]: + entries = ledger.get("entries") + return [item for item in entries if isinstance(item, dict)] if isinstance(entries, list) else [] + + +def find_proposal(proposals: dict[str, Any], proposal_id: str) -> dict[str, Any]: + for item in proposals.get("proposals", []): + if isinstance(item, dict) and item.get("proposal_id") == proposal_id: + return item + return {} + + +def find_approval(ledger: dict[str, Any], proposal_id: str) -> dict[str, Any]: + for item in approved_entries(ledger): + if item.get("proposal_id") == proposal_id and item.get("decision") == "approved": + return item + return {} + + +def refresh_ledger_summary(ledger: dict[str, Any]) -> None: + entries = approved_entries(ledger) + approved = [item for item in entries if item.get("decision") == "approved"] + pending = [item for item in entries if item.get("decision") == "pending-review"] + ledger["summary"] = { + "approval_count": len(approved), + "active_approval_count": len(approved), + "pending_review_count": len(pending), + "applied_count": 0, + "rollback_count": 0, + } + decorate_approval_ledger(ledger) + + +def upsert_ledger_entry(ledger: dict[str, Any], entry: dict[str, Any]) -> None: + entries = approved_entries(ledger) + proposal_id = entry.get("proposal_id") + kept = [item for item in entries if item.get("proposal_id") != proposal_id] + ledger["entries"] = [*kept, entry] + refresh_ledger_summary(ledger) + + +def parse_date(value: str) -> date | None: + if not value: + return None + try: + return date.fromisoformat(value[:10]) + except ValueError: + return None + + +def validate_approval(approval: dict[str, Any], today: date) -> list[str]: + failures = [] + required = [ + "reviewer", + "reason", + "approved_at", + "patch_sha256", + "target_files", + "target_file_sha256", + "verification_commands", + "rollback_plan", + ] + for key in required: + if not approval.get(key): + failures.append(f"Approval entry missing required field: {key}") + expires_at = parse_date(str(approval.get("expires_at", ""))) + if expires_at and expires_at < today: + failures.append(f"Approval entry is expired: {approval.get('expires_at')}") + if not isinstance(approval.get("target_files"), list): + failures.append("Approval target_files must be a list.") + if not isinstance(approval.get("target_file_sha256"), dict): + failures.append("Approval target_file_sha256 must be an object.") + if not isinstance(approval.get("verification_commands"), list): + failures.append("Approval verification_commands must be a list.") + return failures + + +def safe_command(command: str) -> tuple[bool, list[str], str]: + try: + parts = shlex.split(command) + except ValueError as exc: + return False, [], f"cannot parse command: {exc}" + if not parts: + return False, [], "empty command" + if parts[0] == "make" and len(parts) == 2 and all(ch.isalnum() or ch in {"-", "_"} for ch in parts[1]): + return True, parts, "make target" + if parts[0] in {"python3", sys.executable} and len(parts) >= 2: + script = Path(parts[1]) + if not script.is_absolute() and script.parts and script.parts[0] in {"tests", "scripts"} and script.suffix == ".py": + return True, parts, "local python verifier" + return False, parts, "command is not in the safe regression allowlist" + + +def run_command(command: str, skill_dir: Path) -> dict[str, Any]: + allowed, parts, reason = safe_command(command) + if not allowed: + return {"command": command, "ok": False, "returncode": None, "stdout": "", "stderr": reason} + proc = subprocess.run(parts, cwd=skill_dir, capture_output=True, text=True) + return { + "command": command, + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "stdout": proc.stdout[-4000:], + "stderr": proc.stderr[-4000:], + } + + +def git_apply_check(skill_dir: Path, patch_file: Path) -> dict[str, Any]: + env = dict(os.environ) + env["GIT_CEILING_DIRECTORIES"] = str(skill_dir.parent) + proc = subprocess.run(["git", "apply", "--check", str(patch_file)], cwd=skill_dir, capture_output=True, text=True, env=env) + return {"ok": proc.returncode == 0, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + + +def git_apply(skill_dir: Path, patch_file: Path) -> dict[str, Any]: + env = dict(os.environ) + env["GIT_CEILING_DIRECTORIES"] = str(skill_dir.parent) + proc = subprocess.run(["git", "apply", str(patch_file)], cwd=skill_dir, capture_output=True, text=True, env=env) + return {"ok": proc.returncode == 0, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + + +def git_apply_reverse(skill_dir: Path, patch_file: Path) -> dict[str, Any]: + env = dict(os.environ) + env["GIT_CEILING_DIRECTORIES"] = str(skill_dir.parent) + proc = subprocess.run(["git", "apply", "-R", str(patch_file)], cwd=skill_dir, capture_output=True, text=True, env=env) + return {"ok": proc.returncode == 0, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr} + + +def empty_approval_ledger(generated_at: str) -> dict[str, Any]: + return decorate_approval_ledger({ + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "summary": { + "approval_count": 0, + "active_approval_count": 0, + "pending_review_count": 0, + "applied_count": 0, + "rollback_count": 0, + }, + "approval_contract": dict(APPROVAL_CONTRACT), + "entries": [], + }) + + +def empty_regression_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: + return decorate_regression_report({ + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + "summary": { + "apply_supported": True, + "attempt_count": 0, + "approval_draft_count": 0, + "applied_count": 0, + "dry_run_count": 0, + "rollback_count": 0, + "regression_run_count": 0, + "regression_pass_count": 0, + "failure_count": 0, + }, + "apply_contract": dict(APPLY_CONTRACT), + "attempts": [], + "failures": [], + "artifacts": { + "json": "reports/adaptation_regression_report.json", + "approval_ledger": "reports/adaptation_approval_ledger.json", + }, + }) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Adaptation Regression Report", + "", + f"- generated_at: `{report['generated_at']}`", + f"- apply_supported: `{str(summary['apply_supported']).lower()}`", + f"- attempts: `{summary['attempt_count']}`", + f"- applied: `{summary['applied_count']}`", + f"- dry runs: `{summary['dry_run_count']}`", + f"- rollbacks: `{summary.get('rollback_count', 0)}`", + f"- failures: `{summary['failure_count']}`", + "", + "This report proves the adaptive apply harness behavior. It does not count proposals as applied changes.", + "", + ] + for attempt in report.get("attempts", []): + lines.extend( + [ + f"## {attempt['proposal_id']}", + "", + f"- mode: `{attempt['mode']}`", + f"- status: `{attempt['status']}`", + f"- patch: `{attempt['patch']}`", + f"- patch sha256: `{attempt['patch_sha256']}`", + f"- targets: `{', '.join(attempt['target_files'])}`", + f"- rollback: {attempt['rollback']['plan']}", + ] + ) + if attempt.get("regression_runs"): + lines.append("- regression:") + lines.extend(f" - `{item['command']}`: `{str(item['ok']).lower()}`" for item in attempt["regression_runs"]) + lines.append("") + if report.get("failures"): + lines.extend(["## Failures", ""]) + lines.extend(f"- {failure}" for failure in report["failures"]) + draft = report.get("approval_draft") + if isinstance(draft, dict): + lines.extend( + [ + "", + "## Approval Draft", + "", + f"- proposal: `{draft.get('proposal_id', '')}`", + f"- decision: `{draft.get('decision', '')}`", + f"- patch sha256: `{draft.get('patch_sha256', '')}`", + f"- targets: `{', '.join(draft.get('target_files', []))}`", + "", + "A human reviewer must set `decision` to `approved` and fill reviewer, reason, and approval date before apply.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def build_report(args: argparse.Namespace) -> dict[str, Any]: + skill_dir = Path(args.skill_dir).resolve() + generated_at = args.generated_at or utc_now() + ledger_path = resolve_path(skill_dir, args.approval_ledger) + output_json = resolve_path(skill_dir, args.output_json) + output_md = resolve_path(skill_dir, args.output_md) + if args.write_template: + ledger = empty_approval_ledger(generated_at) + report = empty_regression_report(skill_dir, generated_at) + write_json(ledger_path, ledger) + write_json(output_json, report) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + failures: list[str] = [] + proposals_path = resolve_path(skill_dir, args.proposals_json) + patch_path = resolve_path(skill_dir, args.patch_file) if args.patch_file else Path("") + proposals = load_json(proposals_path) + ledger = load_json(ledger_path) + if not proposals: + failures.append(f"Proposal report missing or invalid: {display_path(proposals_path, skill_dir)}") + if args.prepare_approval and not ledger and not ledger_path.exists(): + ledger = empty_approval_ledger(generated_at) + if not args.prepare_approval and not ledger: + failures.append(f"Approval ledger missing or invalid: {display_path(ledger_path, skill_dir)}") + if args.prepare_approval and ledger_path.exists() and not ledger: + failures.append(f"Approval ledger exists but is invalid: {display_path(ledger_path, skill_dir)}") + if not args.proposal_id: + failures.append("--proposal-id is required unless --write-template is used.") + if not args.patch_file or not patch_path.exists(): + failures.append("--patch-file must point to an existing unified diff.") + + today = parse_date(str(args.today or date.today().isoformat())) + if today is None: + failures.append("--today must be an ISO date when provided.") + + proposal = find_proposal(proposals, args.proposal_id) if not failures else {} + if not failures and not proposal: + failures.append(f"Proposal id is not present in proposal report: {args.proposal_id}") + + patch_sha = sha256_file(patch_path) if patch_path.exists() else "" + target_files: list[str] = [] + observed_target_file_sha256: dict[str, str] = {} + if not failures: + try: + target_files = patch_target_files(patch_path.read_text(encoding="utf-8", errors="replace")) + except ValueError as exc: + failures.append(str(exc)) + + if args.prepare_approval: + if not failures: + proposal_targets = set(str(item) for item in proposal.get("target_files", [])) + patch_targets = set(target_files) + if not patch_targets: + failures.append("Patch does not declare any target files.") + if not patch_targets <= proposal_targets: + failures.append("Patch touches files outside proposal target_files.") + check = git_apply_check(skill_dir, patch_path) if not failures else {"ok": False, "returncode": None, "stdout": "", "stderr": ""} + if not failures and not check["ok"]: + failures.append(f"git apply --check failed: {check['stderr'].strip()}") + draft: dict[str, Any] = {} + if not failures: + observed_target_file_sha256 = target_file_sha256(skill_dir, target_files) + draft = { + "proposal_id": args.proposal_id, + "decision": "pending-review", + "reviewer": "", + "reason": "", + "approved_at": "", + "expires_at": "", + "created_at": generated_at, + "patch": display_path(patch_path, skill_dir), + "patch_sha256": patch_sha, + "target_files": target_files, + "target_file_sha256": observed_target_file_sha256, + "verification_commands": [ + str(command) for command in proposal.get("verification_commands", []) if command + ], + "rollback_plan": proposal.get("rollback_plan", f"git apply -R {display_path(patch_path, skill_dir)}"), + } + upsert_ledger_entry(ledger, draft) + write_json(ledger_path, ledger) + report = decorate_regression_report({ + "schema_version": "1.0", + "ok": not failures, + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + "summary": { + "apply_supported": True, + "attempt_count": 0, + "approval_draft_count": 1 if draft else 0, + "applied_count": 0, + "dry_run_count": 0, + "rollback_count": 0, + "regression_run_count": 0, + "regression_pass_count": 0, + "failure_count": len(failures), + }, + "apply_contract": dict(APPLY_CONTRACT), + "approval_draft": draft, + "attempts": [], + "failures": failures, + "artifacts": { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + "approval_ledger": display_path(ledger_path, skill_dir), + }, + }) + write_json(output_json, report) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + approval = find_approval(ledger, args.proposal_id) if not failures else {} + if not failures and not approval: + failures.append(f"Proposal id is not approved in the ledger: {args.proposal_id}") + if approval: + refresh_ledger_summary(ledger) + write_json(ledger_path, ledger) + failures.extend(validate_approval(approval, today or date.today())) + + if not failures: + approved_targets = set(str(item) for item in approval.get("target_files", [])) + proposal_targets = set(str(item) for item in proposal.get("target_files", [])) + patch_targets = set(target_files) + if patch_sha != approval.get("patch_sha256"): + failures.append("Patch sha256 does not match approval ledger.") + if not patch_targets: + failures.append("Patch does not declare any target files.") + if not patch_targets <= approved_targets: + failures.append("Patch touches files outside approval target_files.") + if not patch_targets <= proposal_targets: + failures.append("Patch touches files outside proposal target_files.") + if not failures: + baseline_failures, observed_target_file_sha256 = validate_target_file_sha256( + skill_dir, + target_files, + approval.get("target_file_sha256", {}), + ) + failures.extend(baseline_failures) + + check = git_apply_check(skill_dir, patch_path) if not failures else {"ok": False, "returncode": None, "stdout": "", "stderr": ""} + if not failures and not check["ok"]: + failures.append(f"git apply --check failed: {check['stderr'].strip()}") + + mode = "apply" if args.apply else "dry-run" + applied = False + apply_result = {"ok": False, "returncode": None, "stdout": "", "stderr": "not requested"} + if not failures and args.apply: + apply_result = git_apply(skill_dir, patch_path) + applied = bool(apply_result["ok"]) + if not applied: + failures.append(f"git apply failed: {apply_result['stderr'].strip()}") + + regression_runs: list[dict[str, Any]] = [] + if not failures and args.run_verification: + for command in approval.get("verification_commands", []): + regression_runs.append(run_command(str(command), skill_dir)) + if any(not item["ok"] for item in regression_runs): + failures.append("One or more regression commands failed or were not allowed.") + + rollback_result = {"ok": None, "returncode": None, "stdout": "", "stderr": "not needed"} + rolled_back = False + if applied and failures and args.rollback_on_failure: + rollback_result = git_apply_reverse(skill_dir, patch_path) + rolled_back = bool(rollback_result["ok"]) + if not rolled_back: + failures.append(f"Automatic rollback failed: {rollback_result['stderr'].strip()}") + + attempt = { + "proposal_id": args.proposal_id or "", + "mode": mode, + "status": "failed-rolled-back" if rolled_back else ("failed" if failures else ("applied" if applied else "dry-run-pass")), + "patch": display_path(patch_path, skill_dir) if args.patch_file else "", + "patch_sha256": patch_sha, + "target_files": target_files, + "approval": { + "reviewer": approval.get("reviewer", ""), + "approved_at": approval.get("approved_at", ""), + "expires_at": approval.get("expires_at", ""), + }, + "target_file_sha256": { + "expected": { + target: str(approval.get("target_file_sha256", {}).get(target, "")) + for target in target_files + }, + "observed": observed_target_file_sha256, + }, + "git_apply_check": check, + "git_apply": apply_result, + "regression_runs": regression_runs, + "rollback_result": rollback_result, + "rollback": { + "plan": approval.get("rollback_plan", ""), + "command": f"git apply -R {display_path(patch_path, skill_dir)}" if args.patch_file else "", + }, + } + report = decorate_regression_report({ + "schema_version": "1.0", + "ok": not failures, + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + "summary": { + "apply_supported": True, + "attempt_count": 1, + "approval_draft_count": 0, + "applied_count": 1 if applied and not rolled_back else 0, + "dry_run_count": 0 if args.apply else 1, + "rollback_count": 1 if rolled_back else 0, + "regression_run_count": len(regression_runs), + "regression_pass_count": sum(1 for item in regression_runs if item["ok"]), + "failure_count": len(failures), + }, + "apply_contract": dict(APPLY_CONTRACT), + "attempts": [attempt], + "failures": failures, + "artifacts": { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + "approval_ledger": display_path(ledger_path, skill_dir), + }, + }) + write_json(output_json, report) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Apply an approved adaptation patch with review and regression evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--proposal-id") + parser.add_argument("--patch-file") + parser.add_argument("--proposals-json", default="reports/adaptation_proposals.json") + parser.add_argument("--approval-ledger", default="reports/adaptation_approval_ledger.json") + parser.add_argument("--output-json", default="reports/adaptation_regression_report.json") + parser.add_argument("--output-md", default="reports/adaptation_regression_report.md") + parser.add_argument("--generated-at") + parser.add_argument("--today") + parser.add_argument("--write-template", action="store_true") + parser.add_argument( + "--prepare-approval", + action="store_true", + help="Create or update a pending approval ledger entry with patch and target baseline hashes.", + ) + parser.add_argument("--apply", action="store_true", help="Write the patch after every approval and allowlist check passes.") + parser.add_argument("--run-verification", action="store_true") + parser.add_argument( + "--no-rollback-on-failure", + dest="rollback_on_failure", + action="store_false", + help="Leave an applied patch in place if verification fails. Default is to reverse the patch.", + ) + parser.set_defaults(rollback_on_failure=True) + args = parser.parse_args() + + report = build_report(args) + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark_release_lock.py b/scripts/benchmark_release_lock.py new file mode 100644 index 00000000..cd8157e8 --- /dev/null +++ b/scripts/benchmark_release_lock.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Git source/generated dirty classification for benchmark release locks.""" + +import subprocess +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_benchmark_reproducibility.py to keep release-lock git status classification out of the report renderer." + + +GENERATED_DIRTY_PREFIXES = ( + "dist/", + "registry/index.json", + "registry/packages/", + "reports/", + "skill_atlas/", + "skill-ir/examples/", +) + + +def status_path(line: str) -> str: + path = line[3:].strip() if len(line) > 3 else line.strip() + if " -> " in path: + path = path.rsplit(" -> ", 1)[-1].strip() + return path + + +def is_generated_dirty_path(path: str) -> bool: + return any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in GENERATED_DIRTY_PREFIXES) + + +def git_status(skill_dir: Path) -> dict[str, Any]: + try: + proc = subprocess.run( + ["git", "status", "--short", "--untracked-files=all"], + cwd=skill_dir, + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return { + "available": False, + "dirty": None, + "changed_file_count": None, + "source_dirty": None, + "source_changed_file_count": None, + "generated_dirty": None, + "generated_changed_file_count": None, + } + lines = [line for line in proc.stdout.splitlines() if line.strip()] + paths = [status_path(line) for line in lines] + generated_lines = [line for line, path in zip(lines, paths) if is_generated_dirty_path(path)] + source_lines = [line for line, path in zip(lines, paths) if not is_generated_dirty_path(path)] + return { + "available": True, + "dirty": bool(lines), + "changed_file_count": len(lines), + "generated_dirty": bool(generated_lines), + "generated_changed_file_count": len(generated_lines), + "source_dirty": bool(source_lines), + "source_changed_file_count": len(source_lines), + "sample": lines[:12], + "source_sample": source_lines[:12], + "generated_sample": generated_lines[:12], + "generated_dirty_prefixes": list(GENERATED_DIRTY_PREFIXES), + "scope": "generation-time status before this report is written", + } + + +def release_lock_status(status: dict[str, Any], commit: str) -> dict[str, Any]: + available = status.get("available") is True + source_clean = status.get("source_dirty") is False + known_commit = bool(commit and commit != "unknown") + ready = available and source_clean and known_commit + reasons = [] + if not available: + reasons.append("git status unavailable") + if not known_commit: + reasons.append("git commit unavailable") + if status.get("source_dirty") is True: + reasons.append("source files were dirty at generation time") + if status.get("source_dirty") is None: + reasons.append("working tree cleanliness unknown") + if status.get("generated_dirty") is True and status.get("source_dirty") is False: + reasons.append("only generated evidence artifacts were dirty at generation time") + if not reasons: + reasons.append("clean source tree at generation-time HEAD") + return { + "ready": ready, + "commit": commit, + "status_scope": status.get("scope", "generation-time status"), + "source_changed_file_count": status.get("source_changed_file_count"), + "generated_changed_file_count": status.get("generated_changed_file_count"), + "reason": "; ".join(reasons), + } diff --git a/scripts/build_skill_atlas.py b/scripts/build_skill_atlas.py index 4e1b3f49..049d0c01 100644 --- a/scripts/build_skill_atlas.py +++ b/scripts/build_skill_atlas.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import argparse import csv -import html import json import re from collections import Counter, defaultdict @@ -10,6 +9,9 @@ from io import StringIO from pathlib import Path from typing import Any +from build_skill_atlas_opportunities import no_route_opportunities +from build_skill_atlas_layout import render_html + try: import yaml except ImportError: # pragma: no cover @@ -57,6 +59,7 @@ DEFAULT_SCOPE = { "actionable": True, "scope_reason": "default release-actionable skill", } +TELEMETRY_REQUIRED_MATURITIES = {"production", "library", "governed"} def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]: @@ -217,6 +220,105 @@ def collect_skill(workspace_root: Path, skill_dir: Path, policy: dict[str, Any]) } +def load_telemetry_profile(workspace_root: Path, skill_dir: Path, skill: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + report_path = skill_dir / "reports" / "adoption_drift_report.json" + rel_report = safe_rel(workspace_root, report_path) + maturity = str(skill.get("maturity", "")).casefold() + report_present = report_path.exists() + telemetry = { + "report_present": report_present, + "report": rel_report, + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0, + } + signals: list[dict[str, Any]] = [] + if not report_present: + if skill.get("actionable") and maturity in TELEMETRY_REQUIRED_MATURITIES: + signals.append( + { + "name": skill["name"], + "path": skill["path"], + "source": rel_report, + "risk_band": "no-data", + "signal_types": ["no telemetry"], + "recommendation": "Render adoption drift evidence or record a small metadata-only sample before release review.", + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), + "summary": {"event_count": 0, "adoption_sample_count": 0}, + } + ) + return telemetry, signals + + payload = load_json(report_path) + summary = payload.get("summary", {}) if isinstance(payload.get("summary"), dict) else {} + candidates = payload.get("next_iteration_candidates", []) + candidates = candidates if isinstance(candidates, list) else [] + risk_band = str(summary.get("risk_band") or "unknown") + telemetry.update( + { + "risk_band": risk_band, + "event_count": int(summary.get("event_count") or 0), + "adoption_sample_count": int(summary.get("adoption_sample_count") or 0), + "adoption_rate": summary.get("adoption_rate", 0), + "candidate_count": len(candidates), + } + ) + + signal_types: list[str] = [] + if telemetry["event_count"] == 0 and maturity in TELEMETRY_REQUIRED_MATURITIES: + signal_types.append("no telemetry") + if int(summary.get("missed_trigger_count") or 0): + signal_types.append("missed trigger") + if int(summary.get("wrong_trigger_count") or 0): + signal_types.append("wrong trigger") + if int(summary.get("bad_output_count") or 0): + signal_types.append("bad output") + if int(summary.get("missing_resource_count") or 0): + signal_types.append("missing resource") + if int(summary.get("script_error_count") or 0): + signal_types.append("script error") + if int(summary.get("review_overdue_count") or 0): + signal_types.append("review overdue") + if risk_band in {"medium", "high"} and not signal_types: + signal_types.append("telemetry drift") + + if signal_types: + recommendation = "Convert telemetry drift into eval, trust, or owner-review actions." + for candidate in candidates: + if not isinstance(candidate, dict): + continue + if str(candidate.get("signal", "")) in signal_types: + recommendation = str(candidate.get("recommendation") or recommendation) + break + signals.append( + { + "name": skill["name"], + "path": skill["path"], + "source": rel_report, + "risk_band": risk_band, + "signal_types": signal_types, + "recommendation": recommendation, + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), + "summary": { + "event_count": telemetry["event_count"], + "adoption_sample_count": telemetry["adoption_sample_count"], + "adoption_rate": telemetry["adoption_rate"], + "missed_trigger_count": int(summary.get("missed_trigger_count") or 0), + "wrong_trigger_count": int(summary.get("wrong_trigger_count") or 0), + "bad_output_count": int(summary.get("bad_output_count") or 0), + "missing_resource_count": int(summary.get("missing_resource_count") or 0), + "script_error_count": int(summary.get("script_error_count") or 0), + "review_overdue_count": int(summary.get("review_overdue_count") or 0), + }, + } + ) + return telemetry, signals + + def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: rows = [] collisions = [] @@ -351,22 +453,6 @@ def owner_review_gaps(skills: list[dict[str, Any]]) -> list[dict[str, Any]]: return gaps -def no_route_opportunities(workspace_root: Path) -> list[dict[str, Any]]: - opportunities = [] - for path in sorted(workspace_root.rglob("failure-cases.md")): - if should_skip(path, workspace_root): - continue - text = path.read_text(encoding="utf-8", errors="replace") - for line in text.splitlines(): - stripped = line.strip() - if not stripped.startswith("-"): - continue - lowered = stripped.casefold() - if "no_route" in lowered or "no route" in lowered or "missed" in lowered or "under-trigger" in lowered: - opportunities.append({"source": safe_rel(workspace_root, path), "note": stripped.lstrip("- ").strip()}) - return opportunities[:50] - - def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status", "actionable", "scope_a", "scope_b"] @@ -378,93 +464,35 @@ def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.write_text(buffer.getvalue(), encoding="utf-8") -def render_html(payload: dict[str, Any]) -> str: - summary = payload["summary"] - rows = [] - for skill in payload["catalog"]["skills"][:80]: - rows.append( - "" - f"{html.escape(skill['name'])}" - f"{html.escape(skill['path'])}" - f"{html.escape(skill.get('owner') or 'missing')}" - f"{html.escape(skill.get('maturity') or 'unknown')}" - f"{html.escape(skill.get('review_cadence') or 'missing')}" - f"{html.escape(skill.get('atlas_scope') or 'release')}" - "" - ) - blockers = payload["actionable_route_collisions"][:20] + payload["actionable_owner_review_gaps"][:20] + payload["actionable_stale_skills"][:20] - blocker_items = "".join( - f"
  • {html.escape(item.get('name', item.get('skill_a', 'issue')))} {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', [])))))}
    • " - for item in blockers - ) - return f""" - - - - - Skill Atlas - - - -
    -

    Skill Atlas

    -

    Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

    -
    -
    Skills{summary['skill_count']}
    -
    Actionable{summary['actionable_skill_count']}
    -
    Route Collisions{summary['actionable_route_collision_count']}
    -
    Owner Gaps{summary['actionable_owner_gap_count']}
    -
    Stale Skills{summary['actionable_stale_count']}
    -
    No-Route Opportunities{summary['no_route_opportunity_count']}
    -
    -
    -

    Actionable Issues

    -
      {blocker_items or '
    • No blocking portfolio issues detected.
      • '}
    -
    -
    -

    Full Portfolio Counts

    -

    All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, and {summary['stale_count']} total stale signals.

    -
    -
    -

    Catalog

    - - - {''.join(rows)} -
    NamePathOwnerMaturityReviewScope
    -
    -
    - - -""" - - def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, report_json: Path, threshold: float, today: date) -> dict[str, Any]: workspace_root = workspace_root.resolve() scope_policy = load_scope_policy(workspace_root) skill_dirs = find_skill_dirs(workspace_root) - skills = [collect_skill(workspace_root, skill_dir, scope_policy) for skill_dir in skill_dirs] + skills = [] + drift_signals: list[dict[str, Any]] = [] + telemetry_report_count = 0 + for skill_dir in skill_dirs: + skill = collect_skill(workspace_root, skill_dir, scope_policy) + telemetry, signals = load_telemetry_profile(workspace_root, skill_dir, skill) + skill["telemetry"] = telemetry + telemetry_report_count += 1 if telemetry["report_present"] else 0 + drift_signals.extend(signals) + skills.append(skill) overlap_rows, collisions = route_overlap(skills, threshold) graph = dependency_graph(skills) stale = stale_skills(skills, today) owner_gaps = owner_review_gaps(skills) - opportunities = no_route_opportunities(workspace_root) + opportunities = no_route_opportunities( + workspace_root, + drift_signals, + should_skip=should_skip, + safe_rel=safe_rel, + ) actionable_skills = [skill for skill in skills if skill.get("actionable")] actionable_collisions = [item for item in collisions if item.get("actionable")] actionable_stale = [item for item in stale if item.get("actionable")] actionable_owner_gaps = [item for item in owner_gaps if item.get("actionable")] + actionable_drift_signals = [item for item in drift_signals if item.get("actionable")] summary = { "skill_count": len(skills), "actionable_skill_count": len(actionable_skills), @@ -476,9 +504,13 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "actionable_stale_count": len(actionable_stale), "shared_resource_count": len(graph["shared_resources"]), "no_route_opportunity_count": len(opportunities), + "telemetry_report_count": telemetry_report_count, + "drift_signal_count": len(drift_signals), + "actionable_drift_signal_count": len(actionable_drift_signals), "non_actionable_issue_count": (len(collisions) - len(actionable_collisions)) + (len(owner_gaps) - len(actionable_owner_gaps)) - + (len(stale) - len(actionable_stale)), + + (len(stale) - len(actionable_stale)) + + (len(drift_signals) - len(actionable_drift_signals)), } catalog = { "workspace_root": display_path(workspace_root), @@ -499,6 +531,8 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "actionable_stale_skills": actionable_stale, "owner_review_gaps": owner_gaps, "actionable_owner_review_gaps": actionable_owner_gaps, + "drift_signals": drift_signals, + "actionable_drift_signals": actionable_drift_signals, "no_route_opportunities": opportunities, "artifacts": { "catalog": display_path(output_dir / "catalog.json"), @@ -506,6 +540,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "dependency_graph": display_path(output_dir / "dependency_graph.json"), "stale_skills": display_path(output_dir / "stale_skills.json"), "owner_review_gaps": display_path(output_dir / "owner_review_gaps.json"), + "drift_signals": display_path(output_dir / "drift_signals.json"), "no_route_opportunities": display_path(output_dir / "no_route_opportunities.json"), "report_json": display_path(report_json), "report_html": display_path(report_html), @@ -517,6 +552,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor (output_dir / "dependency_graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "stale_skills.json").write_text(json.dumps(stale, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "owner_review_gaps.json").write_text(json.dumps(owner_gaps, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output_dir / "drift_signals.json").write_text(json.dumps(drift_signals, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "no_route_opportunities.json").write_text(json.dumps(opportunities, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") report_json.parent.mkdir(parents=True, exist_ok=True) report_html.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/build_skill_atlas_layout.py b/scripts/build_skill_atlas_layout.py new file mode 100644 index 00000000..d992cfbf --- /dev/null +++ b/scripts/build_skill_atlas_layout.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""HTML layout helpers for the Skill Atlas report.""" + +import html +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by build_skill_atlas.py to render the static Skill Atlas HTML report." + + +def render_html(payload: dict[str, Any]) -> str: + summary = payload["summary"] + rows = [] + for skill in payload["catalog"]["skills"][:80]: + rows.append( + "" + f"{html.escape(skill['name'])}" + f"{html.escape(skill['path'])}" + f"{html.escape(skill.get('owner') or 'missing')}" + f"{html.escape(skill.get('maturity') or 'unknown')}" + f"{html.escape(skill.get('review_cadence') or 'missing')}" + f"{html.escape(skill.get('atlas_scope') or 'release')}" + "" + ) + blockers = ( + payload["actionable_route_collisions"][:20] + + payload["actionable_owner_review_gaps"][:20] + + payload["actionable_stale_skills"][:20] + + payload["actionable_drift_signals"][:20] + ) + blocker_items = "".join( + f"
  • {html.escape(item.get('name', item.get('skill_a', 'issue')))} {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', item.get('signal_types', []))))))}
    • " + for item in blockers + ) + opportunity_items = "".join( + ( + "
  • " + f"{html.escape(item.get('skill') or item.get('source_type', 'opportunity'))} " + f"{html.escape(item.get('note') or item.get('recommendation') or item.get('signal', 'no-route opportunity'))}" + f"
    {html.escape(item.get('source', ''))}" + "
    • " + ) + for item in payload["no_route_opportunities"][:20] + ) + return f""" + + + + + Skill Atlas + + + +
    +

    Skill Atlas

    +

    Portfolio-level review for route overlap, stale ownership, shared resources, telemetry drift, and no-route opportunities.

    +
    +
    Skills{summary['skill_count']}
    +
    Actionable{summary['actionable_skill_count']}
    +
    Route Collisions{summary['actionable_route_collision_count']}
    +
    Owner Gaps{summary['actionable_owner_gap_count']}
    +
    Stale Skills{summary['actionable_stale_count']}
    +
    Drift Signals{summary['actionable_drift_signal_count']}
    +
    No-Route Opportunities{summary['no_route_opportunity_count']}
    +
    +
    +

    Actionable Issues

    +
      {blocker_items or '
    • No blocking portfolio issues detected.
      • '}
    +
    +
    +

    No-Route Opportunities

    +

    Missed-trigger telemetry and explicit failure cases become candidate routing work without storing raw prompts or outputs.

    +
      {opportunity_items or '
    • No no-route opportunities detected.
      • '}
    +
    +
    +

    Full Portfolio Counts

    +

    All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, {summary['stale_count']} total stale signals, and {summary['drift_signal_count']} telemetry drift signals.

    +
    +
    +

    Catalog

    + + + {''.join(rows)} +
    NamePathOwnerMaturityReviewScope
    +
    +
    + + +""" diff --git a/scripts/build_skill_atlas_opportunities.py b/scripts/build_skill_atlas_opportunities.py new file mode 100644 index 00000000..d0d2b2ee --- /dev/null +++ b/scripts/build_skill_atlas_opportunities.py @@ -0,0 +1,82 @@ +from pathlib import Path +from typing import Any, Callable + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by build_skill_atlas.py to keep no-route opportunity detection out of the atlas CLI." + + +SkipPredicate = Callable[[Path, Path], bool] +RelFormatter = Callable[[Path, Path], str] + + +def failure_case_no_route_opportunities( + workspace_root: Path, + *, + should_skip: SkipPredicate, + safe_rel: RelFormatter, +) -> list[dict[str, Any]]: + opportunities = [] + for path in sorted(workspace_root.rglob("failure-cases.md")): + if should_skip(path, workspace_root): + continue + text = path.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + stripped = line.strip() + if not stripped.startswith("-"): + continue + lowered = stripped.casefold() + if "no_route" in lowered or "no route" in lowered or "missed" in lowered or "under-trigger" in lowered: + opportunities.append( + { + "source_type": "failure-case", + "source": safe_rel(workspace_root, path), + "note": stripped.lstrip("- ").strip(), + "actionable": True, + "privacy_contract": "source note only; raw prompts are not required", + } + ) + return opportunities[:50] + + +def telemetry_no_route_opportunities(drift_signals: list[dict[str, Any]]) -> list[dict[str, Any]]: + opportunities = [] + for signal in drift_signals: + signal_types = {str(item) for item in signal.get("signal_types", [])} + if not {"missed trigger", "under trigger"} & signal_types: + continue + summary = signal.get("summary", {}) if isinstance(signal.get("summary"), dict) else {} + opportunities.append( + { + "source_type": "telemetry", + "source": str(signal.get("source", "")), + "skill": str(signal.get("name", "")), + "path": str(signal.get("path", "")), + "signal": "missed trigger", + "missed_trigger_count": int(summary.get("missed_trigger_count") or 0), + "recommendation": str( + signal.get("recommendation") + or "Add missed prompts to trigger eval and evaluate whether a new skill route is needed." + ), + "actionable": bool(signal.get("actionable")), + "scope": str(signal.get("scope", "")), + "privacy_contract": "metadata-only telemetry; no raw prompt, output, transcript, or note is stored", + } + ) + return opportunities + + +def no_route_opportunities( + workspace_root: Path, + drift_signals: list[dict[str, Any]], + *, + should_skip: SkipPredicate, + safe_rel: RelFormatter, +) -> list[dict[str, Any]]: + opportunities = failure_case_no_route_opportunities( + workspace_root, + should_skip=should_skip, + safe_rel=safe_rel, + ) + opportunities.extend(telemetry_no_route_opportunities(drift_signals)) + return opportunities[:80] diff --git a/scripts/ci_test.py b/scripts/ci_test.py index fda84fe1..9ec03633 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -20,6 +20,7 @@ SAFE_ENV_KEYS = ( "TERM", "TMP", "TMPDIR", + "TZ", ) DEFAULT_TARGETS = [ "eval", @@ -29,14 +30,20 @@ DEFAULT_TARGETS = [ "description-optimization", "description-optimization-check", "promotion-check", + "python-compat-check", + "architecture-maintainability-check", "yao-cli-check", + "yao-cli-world-class-check", "skill-overview-check", + "skill-interpretation-check", "skill-report-metrics-check", "skill-report-charts-check", + "html-rendering-check", "skill-ir-check", "compiler-check", "output-eval-check", "output-execution-check", + "output-review-kit-check", "output-review-adjudication-check", "runtime-conformance-check", "runtime-permission-check", @@ -48,8 +55,28 @@ DEFAULT_TARGETS = [ "upgrade-check", "review-viewer-check", "review-studio-check", + "skill-os2-audit-check", + "skill-os2-coverage-check", + "world-class-evidence-check", + "world-class-ledger-check", + "world-class-intake-check", + "world-class-submission-kit-check", + "world-class-preflight-check", + "world-class-submission-review-check", + "world-class-runbook-check", + "world-class-claim-guard-check", + "benchmark-reproducibility-check", + "evidence-consistency-check", "feedback-check", + "adaptation-safety-check", + "skillops-opportunity-check", + "daily-skillops-check", + "weekly-curator-check", "adoption-drift-check", + "telemetry-import-check", + "telemetry-emit-check", + "telemetry-hooks-check", + "telemetry-native-host-check", "review-waivers-check", "review-annotations-check", "baseline-compare-check", diff --git a/scripts/compile_skill.py b/scripts/compile_skill.py index 62c59938..997e7223 100644 --- a/scripts/compile_skill.py +++ b/scripts/compile_skill.py @@ -10,96 +10,15 @@ try: except ImportError: # pragma: no cover yaml = None +from skill_ir_paths import find_skill_ir as find_skill_ir_document +from compile_skill_targets import TARGET_TRANSFORMS, target_native_contract, target_permission_contract + ROOT = Path(__file__).resolve().parent.parent COMPILER_SCHEMA_VERSION = "1.0" COMPILER_NAME = "yao-skill-ir-compiler" -TARGET_TRANSFORMS: dict[str, dict[str, Any]] = { - "openai": { - "adapter_mode": "metadata-adapter", - "generated_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], - "metadata_mapping": { - "display_name": "targets/openai/agents/openai.yaml::interface.display_name", - "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", - "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", - "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", - "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", - "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", - "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy", - }, - "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], - "unsupported_features": ["client-native script permission prompts are represented as permission contract metadata"], - }, - "claude": { - "adapter_mode": "neutral-source-plus-adapter", - "generated_files": ["targets/claude/adapter.json", "targets/claude/README.md"], - "metadata_mapping": { - "display_name": "targets/claude/adapter.json::display_name", - "default_prompt": "targets/claude/adapter.json::default_prompt", - "activation": "targets/claude/adapter.json::activation_mode", - "execution": "targets/claude/adapter.json::execution_context", - "trust": "targets/claude/adapter.json::trust_level", - "permissions": "targets/claude/adapter.json::target_permission_contract", - "degradation": "targets/claude/adapter.json::degradation_strategy", - }, - "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], - "unsupported_features": ["vendor-native metadata fields are carried as adapter JSON and README notes"], - }, - "generic": { - "adapter_mode": "agent-skills-compatible", - "generated_files": ["targets/generic/adapter.json"], - "metadata_mapping": { - "display_name": "targets/generic/adapter.json::display_name", - "default_prompt": "targets/generic/adapter.json::default_prompt", - "activation": "targets/generic/adapter.json::activation_mode", - "execution": "targets/generic/adapter.json::execution_context", - "trust": "targets/generic/adapter.json::trust_level", - "permissions": "targets/generic/adapter.json::target_permission_contract", - "degradation": "targets/generic/adapter.json::degradation_strategy", - }, - "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], - "unsupported_features": [], - }, - "agent-skills-compatible": { - "adapter_mode": "neutral-agent-skills-source", - "generated_files": ["SKILL.md", "agents/interface.yaml"], - "metadata_mapping": { - "name": "SKILL.md::frontmatter.name", - "description": "SKILL.md::frontmatter.description", - "interface": "agents/interface.yaml", - "manifest": "manifest.json", - }, - "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], - "unsupported_features": [], - }, - "agent-skills": { - "adapter_mode": "neutral-agent-skills-source", - "generated_files": ["SKILL.md", "agents/interface.yaml"], - "metadata_mapping": { - "name": "SKILL.md::frontmatter.name", - "description": "SKILL.md::frontmatter.description", - "interface": "agents/interface.yaml", - "manifest": "manifest.json", - }, - "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], - "unsupported_features": [], - }, - "vscode": { - "adapter_mode": "agent-skills-compatible-project-or-user-scope", - "generated_files": ["SKILL.md", "agents/interface.yaml"], - "metadata_mapping": { - "name": "folder-name-and-SKILL.md::frontmatter.name", - "description": "SKILL.md::frontmatter.description", - "interface": "agents/interface.yaml", - }, - "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], - "unsupported_features": ["VS Code installation scope is documented but not installed by this compiler"], - }, -} - - def display_path(path: Path, root: Path = ROOT) -> str: try: return str(path.resolve().relative_to(root.resolve())) @@ -146,20 +65,7 @@ def read_frontmatter(path: Path) -> dict[str, Any]: def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]: - candidates = [ - skill_dir / "reports" / "skill-ir.json", - skill_dir / "skill-ir" / "examples" / f"{name}.json", - skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", - ] - seen: set[Path] = set() - for path in candidates: - if path in seen: - continue - seen.add(path) - payload = load_json(path) - if payload: - return payload, display_path(path, skill_dir) - return {}, "frontmatter-fallback" + return find_skill_ir_document(skill_dir, name, fallback_source="frontmatter-fallback") def count_list(payload: dict[str, Any], key: str) -> int: @@ -246,191 +152,6 @@ def permission_contract(skill_dir: Path) -> dict[str, Any]: } -TARGET_PERMISSION_MODELS = { - "openai": { - "model": "metadata-only", - "native_enforcement": False, - "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", - "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package.", - }, - "claude": { - "model": "neutral-source-plus-adapter", - "native_enforcement": False, - "representation": "targets/claude/adapter.json::target_permission_contract and README notes", - "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review.", - }, - "generic": { - "model": "agent-skills-compatible-metadata", - "native_enforcement": False, - "representation": "targets/generic/adapter.json::target_permission_contract", - "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review.", - }, -} - - -TARGET_NATIVE_MODELS = { - "openai": { - "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", - "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", - "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", - "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "permission_enforcement": "metadata-only", - "install_scope": "plugin or skill package consumer", - "review_artifacts": ["targets/openai/agents/openai.yaml", "targets/openai/adapter.json", "reports/review-studio.html"], - "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", - "unsupported_native_features": [ - "client-native permission prompts", - "provider-executed scripts", - ], - }, - "claude": { - "native_surface": "Claude-compatible neutral source folder with adapter notes", - "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", - "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", - "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "permission_enforcement": "metadata-fallback", - "install_scope": "user or project skill directory", - "review_artifacts": ["targets/claude/README.md", "targets/claude/adapter.json", "reports/review-studio.html"], - "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", - "unsupported_native_features": [ - "vendor-native permission enforcement", - "provider-specific execution transforms", - ], - }, - "generic": { - "native_surface": "Agent Skills compatible neutral package", - "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", - "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", - "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "permission_enforcement": "consumer-enforced-or-metadata-only", - "install_scope": "generic Agent Skills compatible root", - "review_artifacts": ["targets/generic/adapter.json", "reports/review-studio.html"], - "fallback_behavior": "Neutral source is the runtime fallback.", - "unsupported_native_features": [], - }, - "agent-skills-compatible": { - "native_surface": "Agent Skills standard source tree", - "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", - "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "permission_enforcement": "consumer-enforced-or-metadata-only", - "install_scope": "Agent Skills source root", - "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], - "fallback_behavior": "The source tree itself is the target artifact.", - "unsupported_native_features": [], - }, - "agent-skills": { - "native_surface": "Agent Skills standard source tree", - "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", - "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", - "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "permission_enforcement": "consumer-enforced-or-metadata-only", - "install_scope": "Agent Skills source root", - "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], - "fallback_behavior": "The source tree itself is the target artifact.", - "unsupported_native_features": [], - }, - "vscode": { - "native_surface": "VS Code/Copilot Agent Skills project or user scope", - "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", - "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", - "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "permission_enforcement": "client-or-workspace-trust", - "install_scope": "VS Code user or project skills directory", - "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], - "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", - "unsupported_native_features": [ - "automatic VS Code installation", - ], - }, -} - - -def target_permission_contract(target: str, permissions: dict[str, Any]) -> dict[str, Any]: - model = TARGET_PERMISSION_MODELS.get( - target, - { - "model": "metadata-only", - "native_enforcement": False, - "representation": "adapter metadata", - "operator_note": "Permission semantics are preserved as metadata for reviewer visibility.", - }, - ) - return { - "schema_version": "1.0", - "target": target, - "permission_model": model["model"], - "native_enforcement": model["native_enforcement"], - "representation": model["representation"], - "review_required": bool(permissions.get("review_required")), - "declared_capabilities": permissions.get("declared_capabilities", []), - "capability_counts": { - name: item.get("script_count", 0) - for name, item in permissions.get("capabilities", {}).items() - }, - "evidence": permissions.get("source", ""), - "operator_note": model["operator_note"], - } - - -def target_native_contract( - target: str, - profile: dict[str, Any], - contract: dict[str, Any], - target_permissions: dict[str, Any], -) -> dict[str, Any]: - model = TARGET_NATIVE_MODELS.get( - target, - { - "native_surface": "adapter metadata", - "activation_policy": "Carry activation semantics as metadata for the target consumer.", - "resource_strategy": "Preserve neutral package resources.", - "script_strategy": "Expose script metadata for reviewer visibility.", - "permission_enforcement": "metadata-only", - "install_scope": "target consumer", - "review_artifacts": ["adapter.json", "reports/review-studio.html"], - "fallback_behavior": "Use neutral source package.", - "unsupported_native_features": [], - }, - ) - return { - "schema_version": "1.0", - "target": target, - "native_surface": model["native_surface"], - "activation": { - "policy": model["activation_policy"], - "trigger_description": contract.get("trigger", {}).get("description", ""), - "manual_activation_supported": True, - "automatic_activation_note": "Depends on the target client route/catalog implementation.", - }, - "resources": { - "strategy": model["resource_strategy"], - "counts": contract.get("resources", {}).get("counts", {}), - "generated_files": profile.get("generated_files", []), - }, - "scripts": { - "strategy": model["script_strategy"], - "script_count": contract.get("resources", {}).get("counts", {}).get("scripts", 0), - "help_smoke_failed_count": contract.get("permissions", {}).get("help_smoke", {}).get("failed_count", 0), - }, - "permissions": { - "enforcement": model["permission_enforcement"], - "native_enforcement": bool(target_permissions.get("native_enforcement")), - "declared_capabilities": target_permissions.get("declared_capabilities", []), - "review_required": bool(target_permissions.get("review_required")), - }, - "review": { - "artifacts": model["review_artifacts"], - "fallback_behavior": model["fallback_behavior"], - "unsupported_native_features": [ - *model.get("unsupported_native_features", []), - *profile.get("unsupported_features", []), - ], - }, - "install_scope": model["install_scope"], - } - - def load_sources(skill_dir: Path) -> dict[str, Any]: skill_dir = skill_dir.resolve() frontmatter = read_frontmatter(skill_dir / "SKILL.md") @@ -545,7 +266,7 @@ def compile_target_contract(skill_dir: Path, target: str) -> dict[str, Any]: "preserved_semantics": [], "unsupported_features": [], } - if target not in declared and not (target == "agent-skills" and "agent-skills-compatible" in declared): + if target not in declared and not (target in {"agent-skills", "vscode"} and "agent-skills-compatible" in declared): warnings.append(f"Target is not declared in Skill IR or interface metadata: {target}") if not sources["ir"]: warnings.append("Skill IR is missing; compiler used frontmatter fallback") diff --git a/scripts/compile_skill_targets.py b/scripts/compile_skill_targets.py new file mode 100644 index 00000000..414740f5 --- /dev/null +++ b/scripts/compile_skill_targets.py @@ -0,0 +1,282 @@ +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by compile_skill for target platform model and contract builders." + + +TARGET_TRANSFORMS: dict[str, dict[str, Any]] = { + "openai": { + "adapter_mode": "metadata-adapter", + "generated_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy", + }, + "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": ["client-native script permission prompts are represented as permission contract metadata"], + }, + "claude": { + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": ["targets/claude/adapter.json", "targets/claude/README.md"], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy", + }, + "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": ["vendor-native metadata fields are carried as adapter JSON and README notes"], + }, + "generic": { + "adapter_mode": "agent-skills-compatible", + "generated_files": ["targets/generic/adapter.json"], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy", + }, + "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": [], + }, + "agent-skills-compatible": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": ["SKILL.md", "agents/interface.yaml"], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json", + }, + "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": [], + }, + "agent-skills": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": ["SKILL.md", "agents/interface.yaml"], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json", + }, + "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": [], + }, + "vscode": { + "adapter_mode": "vscode-agent-skills-adapter", + "generated_files": ["targets/vscode/adapter.json", "targets/vscode/README.md"], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "permissions": "targets/vscode/adapter.json::target_permission_contract", + "install_scope": "targets/vscode/README.md", + }, + "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": ["VS Code installation scope is documented but not installed by this compiler"], + }, +} + + +TARGET_PERMISSION_MODELS = { + "openai": { + "model": "metadata-only", + "native_enforcement": False, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package.", + }, + "claude": { + "model": "neutral-source-plus-adapter", + "native_enforcement": False, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review.", + }, + "generic": { + "model": "agent-skills-compatible-metadata", + "native_enforcement": False, + "representation": "targets/generic/adapter.json::target_permission_contract", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review.", + }, + "vscode": { + "model": "vscode-workspace-trust-plus-metadata", + "native_enforcement": False, + "representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes", + "operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks.", + }, +} + + +TARGET_NATIVE_MODELS = { + "openai": { + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "permission_enforcement": "metadata-only", + "install_scope": "plugin or skill package consumer", + "review_artifacts": ["targets/openai/agents/openai.yaml", "targets/openai/adapter.json", "reports/review-studio.html"], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + ], + }, + "claude": { + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "permission_enforcement": "metadata-fallback", + "install_scope": "user or project skill directory", + "review_artifacts": ["targets/claude/README.md", "targets/claude/adapter.json", "reports/review-studio.html"], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + ], + }, + "generic": { + "native_surface": "Agent Skills compatible neutral package", + "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "install_scope": "generic Agent Skills compatible root", + "review_artifacts": ["targets/generic/adapter.json", "reports/review-studio.html"], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [], + }, + "agent-skills-compatible": { + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "install_scope": "Agent Skills source root", + "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [], + }, + "agent-skills": { + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "install_scope": "Agent Skills source root", + "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [], + }, + "vscode": { + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "permission_enforcement": "client-or-workspace-trust", + "install_scope": "VS Code user or project skills directory", + "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + ], + }, +} + + +def target_permission_contract(target: str, permissions: dict[str, Any]) -> dict[str, Any]: + model = TARGET_PERMISSION_MODELS.get( + target, + { + "model": "metadata-only", + "native_enforcement": False, + "representation": "adapter metadata", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility.", + }, + ) + return { + "schema_version": "1.0", + "target": target, + "permission_model": model["model"], + "native_enforcement": model["native_enforcement"], + "representation": model["representation"], + "review_required": bool(permissions.get("review_required")), + "declared_capabilities": permissions.get("declared_capabilities", []), + "capability_counts": { + name: item.get("script_count", 0) + for name, item in permissions.get("capabilities", {}).items() + }, + "evidence": permissions.get("source", ""), + "operator_note": model["operator_note"], + } + + +def target_native_contract( + target: str, + profile: dict[str, Any], + contract: dict[str, Any], + target_permissions: dict[str, Any], +) -> dict[str, Any]: + model = TARGET_NATIVE_MODELS.get( + target, + { + "native_surface": "adapter metadata", + "activation_policy": "Carry activation semantics as metadata for the target consumer.", + "resource_strategy": "Preserve neutral package resources.", + "script_strategy": "Expose script metadata for reviewer visibility.", + "permission_enforcement": "metadata-only", + "install_scope": "target consumer", + "review_artifacts": ["adapter.json", "reports/review-studio.html"], + "fallback_behavior": "Use neutral source package.", + "unsupported_native_features": [], + }, + ) + return { + "schema_version": "1.0", + "target": target, + "native_surface": model["native_surface"], + "activation": { + "policy": model["activation_policy"], + "trigger_description": contract.get("trigger", {}).get("description", ""), + "manual_activation_supported": True, + "automatic_activation_note": "Depends on the target client route/catalog implementation.", + }, + "resources": { + "strategy": model["resource_strategy"], + "counts": contract.get("resources", {}).get("counts", {}), + "generated_files": profile.get("generated_files", []), + }, + "scripts": { + "strategy": model["script_strategy"], + "script_count": contract.get("resources", {}).get("counts", {}).get("scripts", 0), + "help_smoke_failed_count": contract.get("permissions", {}).get("help_smoke", {}).get("failed_count", 0), + }, + "permissions": { + "enforcement": model["permission_enforcement"], + "native_enforcement": bool(target_permissions.get("native_enforcement")), + "declared_capabilities": target_permissions.get("declared_capabilities", []), + "review_required": bool(target_permissions.get("review_required")), + }, + "review": { + "artifacts": model["review_artifacts"], + "fallback_behavior": model["fallback_behavior"], + "unsupported_native_features": [ + *model.get("unsupported_native_features", []), + *profile.get("unsupported_features", []), + ], + }, + "install_scope": model["install_scope"], + } diff --git a/scripts/context_sizer.py b/scripts/context_sizer.py index a6e4f8d8..5abc5631 100644 --- a/scripts/context_sizer.py +++ b/scripts/context_sizer.py @@ -4,7 +4,7 @@ import json from pathlib import Path -TEXT_EXTS = {".md", ".txt", ".yaml", ".yml", ".json", ".py", ".sh", ".js", ".ts"} +TEXT_EXTS = {".css", ".md", ".txt", ".yaml", ".yml", ".json", ".py", ".sh", ".js", ".ts"} IGNORED_RELATIVE_DIRS = { Path("reports") / "release_snapshots", Path("tests") / "tmp", @@ -26,6 +26,27 @@ PACKAGE_PATHS = ( "input", "outputs", ) +IGNORED_FILE_PATTERNS = { + "reports/benchmark_reproducibility*.json", + "reports/benchmark_reproducibility*.md", + "reports/context_budget*.json", + "reports/context_budget*.md", + "reports/evidence_consistency*.json", + "reports/evidence_consistency*.md", + "reports/review-studio*.html", + "reports/review-studio*.json", + "reports/review-viewer*.html", + "reports/review-viewer*.json", + "reports/skill-interpretation*.html", + "reports/skill-interpretation*.json", + "reports/skill-overview*.html", + "reports/skill-overview*.json", + "reports/world_class_evidence_preflight*.json", + "reports/world_class_evidence_preflight*.md", + "reports/world_class_evidence_preflight*.html", + "reports/*pattern-analysis*.md", + "reports/*research-plan*.md", +} def estimate_tokens(text: str) -> int: @@ -61,6 +82,8 @@ def should_ignore(path: Path, skill_dir: Path) -> bool: rel = path.relative_to(skill_dir) if any(rel == ignored or ignored in rel.parents for ignored in IGNORED_RELATIVE_DIRS): return True + if any(rel.match(pattern) for pattern in IGNORED_FILE_PATTERNS): + return True return len(rel.parts) >= 2 and rel.parts[0] == "tests" and rel.parts[1].startswith("tmp_") diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index 6799afee..f71b7e7d 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -3,10 +3,12 @@ import argparse import json import shutil import zipfile -from pathlib import Path +from pathlib import Path, PurePosixPath import yaml from compile_skill import compile_target_contract +from cross_packager_contracts import PLATFORM_CONTRACTS +from skill_ir_paths import find_skill_ir as find_skill_ir_document def display_path(path: Path, root: Path) -> str: @@ -48,20 +50,15 @@ def read_interface(skill_dir: Path) -> dict: def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict, str]: - candidates = [ - skill_dir / "reports" / "skill-ir.json", - skill_dir / "skill-ir" / "examples" / f"{name}.json", - skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", - ] - seen = set() - for path in candidates: - if path in seen: - continue - seen.add(path) - payload = read_json(path) - if payload: - return payload, display_path(path, skill_dir) - return {}, "frontmatter-fallback" + return find_skill_ir_document(skill_dir, name, fallback_source="frontmatter-fallback") + + +def package_name_from_manifest(manifest: dict, skill_dir: Path) -> str: + name = str(manifest.get("name") or "").strip() + if name: + return name + frontmatter = read_frontmatter(skill_dir / "SKILL.md") + return str(frontmatter.get("name") or skill_dir.name) def require_fields(payload: dict, fields: list[str], label: str) -> None: @@ -147,13 +144,17 @@ def build_semantic_contract( "targets": target_values, "source_files_count": count_list(ir, "source_files") if ir else 0, } + alias_declared = ( + platform in {"agent-skills", "vscode"} and "agent-skills-compatible" in target_values + ) semantic_parity = { "source": "skill-ir" if ir else "frontmatter-fallback", "ir_source": ir_source, "name_matches_ir": bool(ir) and frontmatter_name == name, "description_matches_ir": bool(ir) and frontmatter_description == description, "platform_declared_in_ir": platform in target_values - or (platform == "generic" and "agent-skills-compatible" in target_values), + or (platform == "generic" and "agent-skills-compatible" in target_values) + or alias_declared, "platform_declared_in_interface": platform in adapter_targets, "display_name_present": bool(interface.get("display_name")), "default_prompt_present": bool(interface.get("default_prompt")), @@ -209,7 +210,7 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: "description": semantic["description"], "version": manifest.get("version") or frontmatter.get("version", "1.0.0"), "platform": platform, - "skill_root": skill_dir.name, + "skill_root": semantic["name"], "job_to_be_done": semantic["job_to_be_done"], "ir_source": semantic["ir_source"], "ir_schema_version": semantic["ir_schema_version"], @@ -250,115 +251,7 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: } -PLATFORM_CONTRACTS = { - "openai": { - "required_fields": [ - "name", - "description", - "version", - "display_name", - "short_description", - "default_prompt", - "job_to_be_done", - "ir_source", - "ir_schema_version", - "semantic_contract", - "semantic_parity", - "canonical_metadata", - "canonical_format", - "activation_mode", - "execution_context", - "shell", - "trust_level", - "remote_inline_execution", - "degradation_strategy", - "portability_profile", - "permission_contract", - "target_permission_contract", - "target_native_contract", - ], - "required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], - "field_mapping": { - "display_name": "interface.display_name", - "short_description": "interface.short_description", - "default_prompt": "interface.default_prompt", - "execution_context": "compatibility.execution.context", - "shell": "compatibility.execution.shell", - }, - }, - "claude": { - "required_fields": [ - "name", - "description", - "version", - "display_name", - "short_description", - "default_prompt", - "job_to_be_done", - "ir_source", - "ir_schema_version", - "semantic_contract", - "semantic_parity", - "canonical_metadata", - "canonical_format", - "activation_mode", - "execution_context", - "shell", - "trust_level", - "remote_inline_execution", - "degradation_strategy", - "portability_profile", - "permission_contract", - "target_permission_contract", - "target_native_contract", - ], - "required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], - "field_mapping": { - "display_name": "adapter.display_name", - "short_description": "adapter.short_description", - "default_prompt": "adapter.default_prompt", - "execution_context": "compatibility.execution.context", - "shell": "compatibility.execution.shell", - }, - }, - "generic": { - "required_fields": [ - "name", - "description", - "version", - "display_name", - "short_description", - "default_prompt", - "job_to_be_done", - "ir_source", - "ir_schema_version", - "semantic_contract", - "semantic_parity", - "canonical_metadata", - "canonical_format", - "activation_mode", - "execution_context", - "shell", - "trust_level", - "remote_inline_execution", - "degradation_strategy", - "portability_profile", - "permission_contract", - "target_permission_contract", - "target_native_contract", - ], - "required_files": ["targets/generic/adapter.json"], - "field_mapping": { - "display_name": "adapter.display_name", - "short_description": "adapter.short_description", - "default_prompt": "adapter.default_prompt", - "execution_context": "compatibility.execution.context", - "shell": "compatibility.execution.shell", - }, - }, -} - -EXCLUDED_ARCHIVE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"} +EXCLUDED_ARCHIVE_PARTS = {".git", ".previews", "__pycache__", ".venv", "venv", "node_modules", "dist"} def should_skip_archive_path(rel_path: Path) -> bool: @@ -454,7 +347,7 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: notes = target_dir / "README.md" native = payload["target_native_contract"] notes.write_text( - f"# Claude-Compatible Package\n\nUse `{skill_dir.name}` with its neutral source files. This target does not require vendor metadata by default.\n\n" + f"# Claude-Compatible Package\n\nUse `{payload['name']}` with its neutral source files. This target does not require vendor metadata by default.\n\n" f"Native surface: {native['native_surface']}.\n\n" f"Activation: {native['activation']['policy']}\n\n" f"Resources: {native['resources']['strategy']}\n\n" @@ -463,16 +356,34 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: encoding="utf-8", ) payload["install_hint"] = f"Use the packaged skill directly; this target relies on SKILL.md and optional neutral metadata." + elif platform == "vscode": + notes = target_dir / "README.md" + native = payload["target_native_contract"] + notes.write_text( + f"# VS Code / Copilot Agent Skills Package\n\n" + f"Install `{payload['name']}` as a VS Code user or project scoped Agent Skill. Keep the folder name aligned with `SKILL.md` frontmatter name.\n\n" + f"Native surface: {native['native_surface']}.\n\n" + f"Activation: {native['activation']['policy']}\n\n" + f"Resources: {native['resources']['strategy']}\n\n" + f"Scripts: {native['scripts']['strategy']}\n\n" + f"Permission model: {payload['target_permission_contract']['permission_model']}. " + "Review `target_permission_contract`, workspace trust, and `reports/security_trust_report.md` before running scripts.\n\n" + "This adapter does not perform automatic VS Code installation; it preserves the reviewed source package plus install notes.\n", + encoding="utf-8", + ) + payload["install_hint"] = ( + "Install the package as a VS Code user or project scoped Agent Skill; use targets/vscode/README.md for scope and trust notes." + ) else: - payload["install_hint"] = f"Use {skill_dir.name} as an Agent Skills compatible package." + payload["install_hint"] = f"Use {payload['name']} as an Agent Skills compatible package." path = target_dir / "adapter.json" payload["contract"] = PLATFORM_CONTRACTS.get(platform, PLATFORM_CONTRACTS["generic"]) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") return path -def make_zip(skill_dir: Path, out_dir: Path) -> Path: - zip_path = out_dir / f"{skill_dir.name}.zip" +def make_zip(skill_dir: Path, out_dir: Path, package_name: str) -> Path: + zip_path = out_dir / f"{package_name}.zip" skill_root = skill_dir.resolve() out_root = out_dir.resolve() with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf: @@ -487,17 +398,18 @@ def make_zip(skill_dir: Path, out_dir: Path) -> Path: rel_path = path.relative_to(skill_dir) if should_skip_archive_path(rel_path): continue - zf.write(path, arcname=str(path.relative_to(skill_dir.parent))) + zf.write(path, arcname=str(PurePosixPath(package_name, *rel_path.parts))) return zip_path -def copy_manifest(skill_dir: Path, out_dir: Path) -> Path: +def copy_manifest(skill_dir: Path, out_dir: Path) -> tuple[Path, str]: manifest_path = out_dir / "manifest.json" + manifest = build_manifest(skill_dir, "generic") manifest_path.write_text( - json.dumps(build_manifest(skill_dir, "generic"), ensure_ascii=False, indent=2), + json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8", ) - return manifest_path + return manifest_path, package_name_from_manifest(manifest, skill_dir) def load_expectations(path: Path | None) -> dict: @@ -511,9 +423,9 @@ def validate_exports(out_dir: Path, expectations: dict) -> dict: required_targets = expectations.get("required_targets", []) required_fields = expectations.get("required_fields", []) required_by_target = { - "openai": expectations.get("openai_required_files", []), - "claude": expectations.get("claude_required_files", []), - "generic": expectations.get("generic_required_files", []), + key[: -len("_required_files")]: value + for key, value in expectations.items() + if key.endswith("_required_files") } for target in required_targets: @@ -534,7 +446,7 @@ def validate_exports(out_dir: Path, expectations: dict) -> dict: def main() -> None: parser = argparse.ArgumentParser(description="Generate lightweight cross-platform packaging artifacts.") parser.add_argument("skill_dir", help="Path to the skill directory") - parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic") + parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic, vscode") parser.add_argument("--output-dir", default="dist", help="Output directory") parser.add_argument("--expectations", help="JSON file describing packaging expectations") parser.add_argument("--zip", action="store_true", help="Create a zip package") @@ -548,12 +460,12 @@ def main() -> None: if out_dir.exists(): shutil.rmtree(out_dir) out_dir.mkdir(parents=True) - manifest = copy_manifest(skill_dir, out_dir) + manifest, package_name = copy_manifest(skill_dir, out_dir) generated.append(str(manifest)) for platform in (args.platform or ["generic"]): generated.append(str(write_adapter(skill_dir, out_dir, platform))) if args.zip: - generated.append(str(make_zip(skill_dir, out_dir))) + generated.append(str(make_zip(skill_dir, out_dir, package_name))) except (FileNotFoundError, ValueError, yaml.YAMLError) as exc: failures.append(str(exc)) diff --git a/scripts/cross_packager_contracts.py b/scripts/cross_packager_contracts.py new file mode 100644 index 00000000..f5c2b8bf --- /dev/null +++ b/scripts/cross_packager_contracts.py @@ -0,0 +1,107 @@ +"""Platform contract definitions for cross-platform skill packages.""" + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by cross_packager.py to keep platform contract data separate from packaging flow." + + +COMMON_REQUIRED_FIELDS = [ + "name", + "description", + "version", + "display_name", + "short_description", + "default_prompt", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", + "canonical_metadata", + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile", + "permission_contract", + "target_permission_contract", + "target_native_contract", +] + +STANDARD_FIELD_MAPPING = { + "display_name": "adapter.display_name", + "short_description": "adapter.short_description", + "default_prompt": "adapter.default_prompt", + "execution_context": "compatibility.execution.context", + "shell": "compatibility.execution.shell", +} + + +def interface_field_mapping() -> dict[str, str]: + return { + "display_name": "interface.display_name", + "short_description": "interface.short_description", + "default_prompt": "interface.default_prompt", + "execution_context": "compatibility.execution.context", + "shell": "compatibility.execution.shell", + } + + +PLATFORM_CONTRACTS = { + "openai": { + "required_fields": list(COMMON_REQUIRED_FIELDS), + "required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], + "field_mapping": interface_field_mapping(), + }, + "claude": { + "required_fields": list(COMMON_REQUIRED_FIELDS), + "required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], + "field_mapping": dict(STANDARD_FIELD_MAPPING), + }, + "generic": { + "required_fields": list(COMMON_REQUIRED_FIELDS), + "required_files": ["targets/generic/adapter.json"], + "field_mapping": dict(STANDARD_FIELD_MAPPING), + }, + "vscode": { + "required_fields": [ + "name", + "description", + "version", + "display_name", + "short_description", + "default_prompt", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", + "compiler", + "compiled_contract", + "permission_contract", + "target_permission_contract", + "target_native_contract", + "target_transform", + "canonical_metadata", + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile", + ], + "required_files": ["targets/vscode/adapter.json", "targets/vscode/README.md"], + "field_mapping": { + "name": "SKILL.md::frontmatter.name and folder name", + "description": "SKILL.md::frontmatter.description", + "display_name": "agents/interface.yaml::interface.display_name", + "execution_context": "compatibility.execution.context", + "permissions": "adapter.target_permission_contract", + }, + }, +} diff --git a/scripts/description_optimizer_reporting.py b/scripts/description_optimizer_reporting.py new file mode 100644 index 00000000..5889c0b1 --- /dev/null +++ b/scripts/description_optimizer_reporting.py @@ -0,0 +1,141 @@ +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by optimize_description.py to render description optimization reports." + + +def render_markdown(report: dict, title: str) -> str: + lines = [ + f"# {title}", + "", + f"Winner: `{report['winner']['label']}`", + "", + f"- current tokens: `{report['current_candidate']['estimated_tokens']}`", + f"- winner tokens: `{report['winner']['estimated_tokens']}`", + ] + if report["baseline"]: + lines.append(f"- baseline tokens: `{report['baseline']['estimated_tokens']}`") + lines.extend( + [ + "", + "## Winner", + "", + report["winner"]["description"], + "", + "## Candidate Ranking", + "", + "| Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for candidate in report["candidates"]: + holdout = candidate.get("holdout", {}) + lines.append( + f"| `{candidate['label']}` | {candidate['estimated_tokens']} | {candidate['dev']['false_positives']} | {candidate['dev']['false_negatives']} | {candidate['dev']['near_neighbor_pass_rate']} | {holdout.get('false_positives', '-')} | {holdout.get('false_negatives', '-')} |" + ) + + lines.extend( + [ + "", + "## Acceptance Gates", + "", + "| Gate | Winner FP | Winner FN | Current FP | Current FN | Baseline FP | Baseline FN |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for gate_name, gate in ( + ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), + ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), + ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), + ): + winner_gate = gate.get("winner") or {} + current_gate = gate.get("current") or {} + baseline_gate = gate.get("baseline") or {} + if not winner_gate and not current_gate and not baseline_gate: + continue + lines.append( + f"| {gate_name} | {winner_gate.get('false_positives', '-')} | {winner_gate.get('false_negatives', '-')} | {current_gate.get('false_positives', '-')} | {current_gate.get('false_negatives', '-')} | {baseline_gate.get('false_positives', '-')} | {baseline_gate.get('false_negatives', '-')} |" + ) + + lines.extend( + [ + "", + "## Calibration", + "", + "| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |", + "| --- | ---: | --- | ---: | ---: | ---: |", + ] + ) + for gate_name, gate in ( + ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), + ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), + ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), + ): + winner_calibration = gate.get("winner_calibration") or {} + current_calibration = gate.get("current_calibration") or {} + baseline_calibration = gate.get("baseline_calibration") or {} + if not winner_calibration and not current_calibration and not baseline_calibration: + continue + lines.append( + f"| {gate_name} | {winner_calibration.get('score_gap', '-')} | {winner_calibration.get('risk_band', '-')} | {winner_calibration.get('boundary_case_rate', '-')} | {current_calibration.get('score_gap', '-')} | {baseline_calibration.get('score_gap', '-')} |" + ) + + lines.extend( + [ + "", + "## Judge Blind Summary", + "", + "| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement |", + "| --- | ---: | ---: | ---: | ---: |", + ] + ) + judge_gate = report["acceptance_gates"]["judge_blind_holdout_non_regression"] + judge_winner = (judge_gate.get("winner") or {}).get("judge_summary") or {} + judge_current = (judge_gate.get("current") or {}).get("judge_summary") or {} + judge_baseline = (judge_gate.get("baseline") or {}).get("judge_summary") or {} + if judge_winner or judge_current or judge_baseline: + lines.append( + f"| Judge Blind Holdout | {judge_winner.get('agreement_rate', '-')} | {judge_winner.get('mean_confidence', '-')} | {judge_current.get('agreement_rate', '-')} | {judge_baseline.get('agreement_rate', '-')} |" + ) + + lines.extend( + [ + "", + "## Family Health", + "", + "| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families |", + "| --- | --- | --- | --- | --- |", + ] + ) + for gate_name, gate in ( + ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), + ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), + ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), + ): + winner_health = gate.get("winner_family_health") or {} + current_health = gate.get("current_family_health") or {} + baseline_health = gate.get("baseline_family_health") or {} + if not winner_health and not current_health and not baseline_health: + continue + weakest = winner_health.get("weakest_family") or {} + weakest_label = ( + f"{weakest.get('family')} ({weakest.get('errors')} errors)" + if weakest.get("family") + else "-" + ) + lines.append( + f"| {gate_name} | {winner_health.get('clean_family_count', '-')}/{winner_health.get('family_count', '-')} | {weakest_label} | {current_health.get('clean_family_count', '-')}/{current_health.get('family_count', '-')} | {baseline_health.get('clean_family_count', '-')}/{baseline_health.get('family_count', '-')} |" + ) + + lines.extend( + [ + "", + "## Selection Logic", + "", + "Ordered by:", + ] + ) + for item in report["selection_logic"]["priority"]: + lines.append(f"- {item}") + return "\n".join(lines) + "\n" diff --git a/scripts/emit_telemetry_event.py b/scripts/emit_telemetry_event.py new file mode 100644 index 00000000..d737703c --- /dev/null +++ b/scripts/emit_telemetry_event.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse +import json +import shlex +from pathlib import Path +from typing import Any + +from render_adoption_drift_report import ( + ALLOWED_ACTIVATION_TYPES, + ALLOWED_EVENTS, + ALLOWED_FAILURE_TYPES, + ALLOWED_OUTCOMES, + ALLOWED_SOURCES, + display_path, + normalize_event, + skill_defaults, +) + + +def default_spool_path(skill_dir: Path) -> Path: + return skill_dir / ".yao" / "telemetry_spool" / "external_events.jsonl" + + +def append_event(path: Path, event: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n") + + +def import_command(skill_dir: Path, output_jsonl: Path) -> str: + return shlex.join( + [ + "python3", + "scripts/yao.py", + "telemetry-import", + display_path(skill_dir), + "--input-jsonl", + display_path(output_jsonl), + ] + ) + + +def emit_event( + skill_dir: Path, + output_jsonl: Path | None = None, + event_name: str = "script_run", + activation_type: str = "manual", + outcome: str = "unknown", + failure_type: str = "none", + source: str = "external", + command: str = "external-client", + timestamp: str | None = None, + skill_name: str | None = None, + version: str | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + output_jsonl = (output_jsonl or default_spool_path(skill_dir)).resolve() + raw_event: dict[str, Any] = { + "event": event_name, + "activation_type": activation_type, + "outcome": outcome, + "failure_type": failure_type, + "source": source, + "command": command, + } + if timestamp: + raw_event["timestamp"] = timestamp + if skill_name: + raw_event["skill"] = skill_name + if version: + raw_event["version"] = version + event, failures = normalize_event(raw_event, skill_defaults(skill_dir), "emit") + if event and not failures and not dry_run: + append_event(output_jsonl, event) + return { + "ok": not failures, + "schema_version": "1.0", + "skill_dir": display_path(skill_dir), + "output_jsonl": display_path(output_jsonl), + "dry_run": dry_run, + "emitted": bool(event and not failures and not dry_run), + "event": event or {}, + "failures": failures, + "artifacts": { + "spool_jsonl": display_path(output_jsonl), + "import_command": import_command(skill_dir, output_jsonl), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Emit one metadata-only telemetry event for later import.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-jsonl") + parser.add_argument("--event", choices=sorted(ALLOWED_EVENTS), default="script_run") + parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="manual") + parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown") + parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none") + parser.add_argument("--source", choices=sorted(ALLOWED_SOURCES), default="external") + parser.add_argument("--command", default="external-client") + parser.add_argument("--timestamp") + parser.add_argument("--skill-name") + parser.add_argument("--version") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + report = emit_event( + Path(args.skill_dir), + output_jsonl=Path(args.output_jsonl) if args.output_jsonl else None, + event_name=args.event, + activation_type=args.activation_type, + outcome=args.outcome, + failure_type=args.failure_type, + source=args.source, + command=args.command, + timestamp=args.timestamp, + skill_name=args.skill_name, + version=args.version, + dry_run=args.dry_run, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/evidence_consistency_artifact_roles.py b/scripts/evidence_consistency_artifact_roles.py new file mode 100644 index 00000000..12e82fe3 --- /dev/null +++ b/scripts/evidence_consistency_artifact_roles.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Compare artifact-role handoffs across world-class evidence reports.""" + +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts." + + +def _compare_check( + *, + key: str, + label: str, + expected: Any, + actual: Any, + paths: list[str], + detail: str, +) -> dict[str, Any]: + return { + "key": key, + "label": label, + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": paths, + "detail": detail, + } + + +def role_contract_signature(contract: dict[str, Any]) -> dict[str, Any]: + roles = { + str(item.get("role", "")): item + for item in contract.get("roles", []) + if isinstance(item, dict) and str(item.get("role", "")).strip() + } + return { + "role_source": contract.get("role_source"), + "counts_as_evidence": contract.get("counts_as_evidence"), + "artifact_prefill_counts_as_evidence": contract.get("artifact_prefill_counts_as_evidence"), + "submission_ref_total_count": contract.get("submission_ref_total_count"), + "submission_ref_ready_count": contract.get("submission_ref_ready_count"), + "supporting_evidence_total_count": contract.get("supporting_evidence_total_count"), + "supporting_evidence_ready_count": contract.get("supporting_evidence_ready_count"), + "submission_ref_copy_to_artifact_refs": roles.get("submission-ref", {}).get("copy_to_artifact_refs"), + "supporting_evidence_copy_to_artifact_refs": roles.get("supporting-evidence", {}).get( + "copy_to_artifact_refs" + ), + } + + +def preflight_role_signatures(world_class_preflight: dict[str, Any]) -> dict[str, dict[str, Any]]: + signatures: dict[str, dict[str, Any]] = {} + items = world_class_preflight.get("items", []) if isinstance(world_class_preflight, dict) else [] + for item in items: + if not isinstance(item, dict): + continue + key = str(item.get("evidence_key", "")).strip() + submission_kit = item.get("submission_kit", {}) if isinstance(item.get("submission_kit", {}), dict) else {} + contract = ( + submission_kit.get("artifact_role_contract", {}) + if isinstance(submission_kit.get("artifact_role_contract", {}), dict) + else {} + ) + if key and contract: + signatures[key] = role_contract_signature(contract) + return signatures + + +def review_studio_role_signatures(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]: + actions = review_studio.get("review_actions", []) if isinstance(review_studio, dict) else [] + for action in actions: + if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence": + continue + signatures: dict[str, dict[str, Any]] = {} + for step in action.get("evidence_steps", []): + if not isinstance(step, dict): + continue + key = str(step.get("key", "")).strip() + contract = ( + step.get("artifact_role_contract", {}) + if isinstance(step.get("artifact_role_contract", {}), dict) + else {} + ) + if key and contract: + signatures[key] = role_contract_signature(contract) + return signatures + return {} + + +def build_preflight_artifact_role_handoff_checks( + *, + skill_dir: Path, + world_class_preflight: dict[str, Any], + review_studio: dict[str, Any], + report_paths: dict[str, str], +) -> list[dict[str, Any]]: + preflight_submissions = ( + world_class_preflight.get("submissions", {}) + if isinstance(world_class_preflight.get("submissions", {}), dict) + else {} + ) + preflight_commands = ( + preflight_submissions.get("commands", {}) + if isinstance(preflight_submissions.get("commands", {}), dict) + else {} + ) + preflight_role_contract = ( + preflight_submissions.get("artifact_role_contract", {}) + if isinstance(preflight_submissions.get("artifact_role_contract", {}), dict) + else {} + ) + preflight_roles = { + str(item.get("role", "")): item for item in preflight_role_contract.get("roles", []) if isinstance(item, dict) + } + default_submissions_dir = "evidence/world_class/submissions" + expected_preflight_handoff = { + "directory": default_submissions_dir, + "drafts_count_as_evidence": False, + "preflight_counts_submission_as_completion": False, + "html_report": "reports/world_class_evidence_preflight.html", + "html_exists": True, + "prepare_submission": f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir}", + "prepare_prefilled_submission": ( + f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir} " + "--prefill-artifacts" + ), + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {default_submissions_dir}", + "submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {default_submissions_dir}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {default_submissions_dir}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + "artifact_prefill_counts_as_evidence": False, + "artifact_role_source": "world-class-submission-kit", + "artifact_role_counts_as_evidence": False, + "artifact_role_prefill_counts_as_evidence": False, + "submission_ref_role_present": True, + "supporting_evidence_role_present": True, + "submission_ref_copy_to_artifact_refs": True, + "supporting_evidence_copy_to_artifact_refs": False, + "submission_ref_total_present": True, + "supporting_evidence_total_present": True, + } + actual_preflight_handoff = { + "directory": preflight_submissions.get("directory"), + "drafts_count_as_evidence": preflight_submissions.get("drafts_count_as_evidence"), + "preflight_counts_submission_as_completion": preflight_submissions.get( + "preflight_counts_submission_as_completion" + ), + "html_report": world_class_preflight.get("artifacts", {}).get("html") + if isinstance(world_class_preflight.get("artifacts", {}), dict) + else None, + "html_exists": (skill_dir / "reports" / "world_class_evidence_preflight.html").exists(), + "prepare_submission": preflight_commands.get("prepare_submission"), + "prepare_prefilled_submission": preflight_commands.get("prepare_prefilled_submission"), + "validate_intake": preflight_commands.get("validate_intake"), + "submission_review": preflight_commands.get("submission_review"), + "refresh_ledger": preflight_commands.get("refresh_ledger"), + "guard_claim": preflight_commands.get("guard_claim"), + "artifact_prefill_counts_as_evidence": preflight_submissions.get("artifact_prefill_counts_as_evidence"), + "artifact_role_source": preflight_role_contract.get("role_source"), + "artifact_role_counts_as_evidence": preflight_role_contract.get("counts_as_evidence"), + "artifact_role_prefill_counts_as_evidence": preflight_role_contract.get( + "artifact_prefill_counts_as_evidence" + ), + "submission_ref_role_present": "submission-ref" in preflight_roles, + "supporting_evidence_role_present": "supporting-evidence" in preflight_roles, + "submission_ref_copy_to_artifact_refs": preflight_roles.get("submission-ref", {}).get( + "copy_to_artifact_refs" + ), + "supporting_evidence_copy_to_artifact_refs": preflight_roles.get("supporting-evidence", {}).get( + "copy_to_artifact_refs" + ), + "submission_ref_total_present": int(preflight_role_contract.get("submission_ref_total_count", 0)) > 0, + "supporting_evidence_total_present": int(preflight_role_contract.get("supporting_evidence_total_count", 0)) + > 0, + } + return [ + _compare_check( + key="preflight-submission-kit-handoff", + label="Preflight exposes a safe submission-kit handoff", + expected=expected_preflight_handoff, + actual=actual_preflight_handoff, + paths=[report_paths["world_class_preflight"], "reports/world_class_evidence_preflight.html"], + detail=( + "Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, " + "and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence." + ), + ), + _compare_check( + key="review-studio-preflight-artifact-role-handoff", + label="Review Studio mirrors preflight artifact roles", + expected=preflight_role_signatures(world_class_preflight), + actual=review_studio_role_signatures(review_studio), + paths=[report_paths["world_class_preflight"], report_paths["review_studio"]], + detail=( + "The Review Studio world-class action card must carry the same submission-ref versus " + "supporting-evidence contract as the preflight handoff." + ), + ), + ] diff --git a/scripts/evidence_consistency_core.py b/scripts/evidence_consistency_core.py new file mode 100644 index 00000000..9a0dc618 --- /dev/null +++ b/scripts/evidence_consistency_core.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Shared helpers for cross-report evidence consistency checks.""" + +import json +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py for shared report loading, comparison, and Markdown rendering helpers." + +REQUIRED_REPORTS = { + "benchmark": "reports/benchmark_reproducibility.json", + "overview": "reports/skill-overview.json", + "interpretation": "reports/skill-interpretation.json", + "adoption": "reports/adoption_drift_report.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + "world_class_plan": "reports/world_class_evidence_plan.json", + "world_class_intake": "reports/world_class_evidence_intake.json", + "world_class_preflight": "reports/world_class_evidence_preflight.json", + "world_class_submission_review": "reports/world_class_submission_review.json", + "world_class_operator_runbook": "reports/world_class_operator_runbook.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "review_studio": "reports/review-studio.json", + "package_verification": "reports/package_verification.json", + "install_simulation": "reports/install_simulation.json", + "trust": "reports/security_trust_report.json", + "context_budget": "reports/context_budget.json", + "world_class_claim_guard": "reports/world_class_claim_guard.json", +} +REQUIRED_TEXT_REPORTS = { + "skill_os2_review": "reports/skill-os-2-review.md", +} +BENCHMARK_SUMMARY_KEYS = [ + "release_lock_ready", + "required_artifact_count", + "missing_artifact_count", + "source_contract_sha256", + "archive_sha256", + "world_class_ledger_pending_count", + "world_class_source_check_count", + "world_class_source_pass_count", + "world_class_source_blocked_count", + "beta_test_ready", + "beta_test_blocker_count", + "beta_test_deferred_evidence_count", + "public_claim_ready", + "public_claim_blocker_count", +] +ADOPTION_SUMMARY_KEYS = [ + "event_count", + "adoption_sample_count", + "activation_count", + "accepted_count", + "adoption_rate", + "risk_band", + "event_types", + "source_types", +] +LEDGER_SUMMARY_KEYS = [ + "ledger_entry_count", + "accepted_count", + "pending_count", + "human_pending_count", + "external_pending_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "ready_to_claim_world_class", + "decision", +] +LOCKSTEP_SECTIONS = [ + "scorecard", + "capability_profile", + "principle_model", + "contract_boundary", + "quality_review", + "risk_governance", + "world_class_readiness", + "package_assets", + "iteration_roadmap", +] + + +def load_json(path: Path) -> tuple[dict[str, Any], str | None]: + if not path.exists(): + return {}, "missing" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {}, f"invalid-json: {exc}" + if not isinstance(payload, dict): + return {}, "json-root-not-object" + return payload, None + + +def load_text(path: Path) -> tuple[str, str | None]: + if not path.exists(): + return "", "missing" + return path.read_text(encoding="utf-8"), None + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def nested(payload: dict[str, Any], path: list[str], default: Any = None) -> Any: + current: Any = payload + for key in path: + if not isinstance(current, dict) or key not in current: + return default + current = current[key] + return current + + +def scanned_surface_paths(payload: dict[str, Any]) -> set[str]: + surfaces = payload.get("scanned_surfaces") + if not isinstance(surfaces, list): + return set() + paths: set[str] = set() + for item in surfaces: + if isinstance(item, dict) and isinstance(item.get("path"), str): + paths.add(item["path"]) + elif isinstance(item, str): + paths.add(item) + return paths + + +def as_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def add_check( + checks: list[dict[str, Any]], + *, + key: str, + label: str, + status: str, + expected: Any, + actual: Any, + paths: list[str], + detail: str, +) -> None: + checks.append( + { + "key": key, + "label": label, + "status": status, + "expected": expected, + "actual": actual, + "paths": paths, + "detail": detail, + } + ) + + +def compare_values( + checks: list[dict[str, Any]], + *, + key: str, + label: str, + expected: Any, + actual: Any, + paths: list[str], + detail: str, +) -> None: + add_check( + checks, + key=key, + label=label, + status="pass" if expected == actual else "fail", + expected=expected, + actual=actual, + paths=paths, + detail=detail, + ) + + +def compare_summary_keys( + checks: list[dict[str, Any]], + *, + key_prefix: str, + label: str, + source_summary: dict[str, Any], + embedded_summary: dict[str, Any], + keys: list[str], + paths: list[str], +) -> None: + expected = {key: source_summary.get(key) for key in keys} + actual = {key: embedded_summary.get(key) for key in keys} + compare_values( + checks, + key=key_prefix, + label=label, + expected=expected, + actual=actual, + paths=paths, + detail="Selected summary fields must match exactly across generated reports.", + ) + + +def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]: + gates = review_studio.get("gates") + if not isinstance(gates, list): + return {} + for item in gates: + if isinstance(item, dict) and item.get("key") == key: + return item + return {} + + +def beta_public_claim_split_values( + benchmark: dict[str, Any], + ledger: dict[str, Any], + review_studio: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any]]: + benchmark_summary = nested(benchmark, ["summary"], {}) + ledger_summary = nested(ledger, ["summary"], {}) + expected_beta_ready = ( + bool(benchmark_summary.get("reproducibility_ready")) + and bool(benchmark_summary.get("release_lock_ready")) + and bool(benchmark_summary.get("provider_evidence_complete")) + ) + output_review_summary = nested(review_studio, ["data", "output_review_adjudication", "summary"], {}) + review_pair_count = as_int(output_review_summary.get("pair_count")) if isinstance(output_review_summary, dict) else None + review_pending_count = as_int(output_review_summary.get("pending_count")) if isinstance(output_review_summary, dict) else None + expected_human_review_complete = ( + review_pair_count is not None and review_pair_count > 0 and review_pending_count == 0 + ) + ledger_entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] + pending_ledger_keys = sorted( + str(entry.get("key", "")) + for entry in ledger_entries + if isinstance(entry, dict) and entry.get("status") == "pending" and str(entry.get("key", "")) + ) + beta_boundary = { + "beta_test_ready": expected_beta_ready, + "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), + "human_review_complete": expected_human_review_complete, + "beta_release_ready": expected_beta_ready, + "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "deferred_evidence_keys": pending_ledger_keys, + "deferred_human_review": "human-adjudication" in pending_ledger_keys, + } + beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {} + deferred_keys = sorted( + str(item.get("key", "")) + for item in beta_release.get("allowed_deferred_evidence", []) + if isinstance(item, dict) and str(item.get("key", "")) + ) + actual_beta_boundary = { + "beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None, + "public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None, + "human_review_complete": benchmark_summary.get("human_review_complete") + if isinstance(benchmark_summary, dict) + else None, + "beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None, + "beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None, + "deferred_evidence_keys": deferred_keys, + "deferred_human_review": "human-adjudication" in deferred_keys, + } + return beta_boundary, actual_beta_boundary + + +def report_contract(payload: dict[str, Any]) -> dict[str, Any]: + contract = payload.get("report_contract") + return contract if isinstance(contract, dict) else {} + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Evidence Consistency", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- checks: `{summary['check_count']}`", + f"- pass: `{summary['pass_count']}`", + f"- warn: `{summary['warn_count']}`", + f"- fail: `{summary['fail_count']}`", + "", + "This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist.", + "", + "## Checks", + "", + "| Check | Status | Detail | Paths |", + "| --- | --- | --- | --- |", + ] + for check in report["checks"]: + paths = ", ".join(f"`{path}`" for path in check["paths"]) + lines.append( + "| " + + " | ".join( + [ + check["label"].replace("|", "\\|"), + f"`{check['status']}`", + check["detail"].replace("|", "\\|"), + paths.replace("|", "\\|"), + ] + ) + + " |" + ) + failures = [check for check in report["checks"] if check["status"] == "fail"] + if failures: + lines.extend(["", "## Failures", ""]) + for check in failures: + lines.extend( + [ + f"### {check['label']}", + "", + f"- key: `{check['key']}`", + f"- expected: `{json.dumps(check['expected'], ensure_ascii=False, sort_keys=True)}`", + f"- actual: `{json.dumps(check['actual'], ensure_ascii=False, sort_keys=True)}`", + "", + ] + ) + return "\n".join(lines).rstrip() + "\n" diff --git a/scripts/evidence_consistency_phase_queue.py b/scripts/evidence_consistency_phase_queue.py new file mode 100644 index 00000000..007cc454 --- /dev/null +++ b/scripts/evidence_consistency_phase_queue.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Compare world-class phase queues across generated reports.""" + +from typing import Any + +from evidence_consistency_world_class import world_class_review_action_steps +from world_class_phase_queue import build_phase_queue, summarize_phase_queue + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to prevent preflight and Review Studio phase-queue drift." + + +def phase_queue_signature(queue: list[dict[str, Any]]) -> list[dict[str, Any]]: + signature: list[dict[str, Any]] = [] + for item in queue: + if not isinstance(item, dict): + continue + signature.append( + { + "phase": item.get("phase"), + "priority": item.get("priority"), + "status": item.get("status"), + "blocked_count": item.get("blocked_count"), + "row_count": item.get("row_count"), + "owners": sorted(str(owner) for owner in item.get("owners", [])), + "evidence_keys": sorted(str(key) for key in item.get("evidence_keys", [])), + "next_action_id": item.get("next_action_id"), + "verification_command": item.get("verification_command"), + "counts_as_completion": item.get("counts_as_completion"), + } + ) + return signature + + +def summary_signature(summary: dict[str, Any]) -> dict[str, Any]: + keys = [ + "phase_queue_count", + "phase_queue_blocked_count", + "phase_queue_row_count", + "phase_queue_next_phase", + "phase_queue_next_action_id", + "phase_queue_next_command", + "phase_queue_counts_as_completion", + ] + return {key: summary.get(key) for key in keys} + + +def keyed_preflight_items(world_class_preflight: dict[str, Any]) -> dict[str, dict[str, Any]]: + items = world_class_preflight.get("items", []) if isinstance(world_class_preflight, dict) else [] + keyed: dict[str, dict[str, Any]] = {} + for item in items: + if not isinstance(item, dict): + continue + key = str(item.get("evidence_key", "")).strip() + if key: + keyed[key] = item + return keyed + + +def keyed_phase_queue_signatures_from_repair_rows(items: dict[str, dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + signatures: dict[str, list[dict[str, Any]]] = {} + for key, item in items.items(): + rows = item.get("repair_checklist", []) if isinstance(item.get("repair_checklist", []), list) else [] + signatures[key] = phase_queue_signature(build_phase_queue(rows)) + return signatures + + +def keyed_phase_queue_signatures_from_items(items: dict[str, dict[str, Any]], queue_key: str) -> dict[str, list[dict[str, Any]]]: + signatures: dict[str, list[dict[str, Any]]] = {} + for key, item in items.items(): + queue = item.get(queue_key, []) if isinstance(item.get(queue_key, []), list) else [] + signatures[key] = phase_queue_signature(queue) + return signatures + + +def any_phase_queue_counts_as_completion( + *queues: list[dict[str, Any]], + item_maps: dict[str, dict[str, Any]] | None = None, +) -> bool: + item_maps = item_maps or {} + for queue in queues: + for item in queue: + if isinstance(item, dict) and item.get("counts_as_completion") is True: + return True + for item in item_maps.values(): + queue = item.get("phase_queue", []) if isinstance(item.get("phase_queue", []), list) else [] + for row in queue: + if isinstance(row, dict) and row.get("counts_as_completion") is True: + return True + return False + + +def build_phase_queue_consistency_check( + *, + world_class_preflight: dict[str, Any], + world_class_operator_runbook: dict[str, Any], + review_studio: dict[str, Any], + report_paths: dict[str, str], +) -> dict[str, Any]: + repair_rows = ( + world_class_preflight.get("repair_checklist", []) + if isinstance(world_class_preflight.get("repair_checklist", []), list) + else [] + ) + expected_queue = build_phase_queue(repair_rows) + expected_summary = summarize_phase_queue(expected_queue) + preflight_items = keyed_preflight_items(world_class_preflight) + operator_runbook_items = keyed_preflight_items(world_class_operator_runbook) + review_steps = world_class_review_action_steps(review_studio) + expected = { + "summary": summary_signature(expected_summary), + "top_level_phase_queue": phase_queue_signature(expected_queue), + "item_phase_queues": keyed_phase_queue_signatures_from_repair_rows(preflight_items), + "phase_queue_counts_as_completion": False, + } + actual = { + "summary": summary_signature( + world_class_preflight.get("summary", {}) + if isinstance(world_class_preflight.get("summary", {}), dict) + else {} + ), + "top_level_phase_queue": phase_queue_signature( + world_class_preflight.get("phase_queue", []) + if isinstance(world_class_preflight.get("phase_queue", []), list) + else [] + ), + "operator_runbook_summary": summary_signature( + world_class_operator_runbook.get("summary", {}) + if isinstance(world_class_operator_runbook.get("summary", {}), dict) + else {} + ), + "operator_runbook_top_level_phase_queue": phase_queue_signature( + world_class_operator_runbook.get("phase_queue", []) + if isinstance(world_class_operator_runbook.get("phase_queue", []), list) + else [] + ), + "item_phase_queues": keyed_phase_queue_signatures_from_items(preflight_items, "phase_queue"), + "operator_runbook_phase_queues": keyed_phase_queue_signatures_from_items( + operator_runbook_items, + "phase_queue", + ), + "review_studio_phase_queues": keyed_phase_queue_signatures_from_items(review_steps, "phase_queue"), + "phase_queue_counts_as_completion": any_phase_queue_counts_as_completion( + world_class_preflight.get("phase_queue", []) + if isinstance(world_class_preflight.get("phase_queue", []), list) + else [], + world_class_operator_runbook.get("phase_queue", []) + if isinstance(world_class_operator_runbook.get("phase_queue", []), list) + else [], + item_maps={**preflight_items, **operator_runbook_items, **review_steps}, + ), + } + expected["operator_runbook_summary"] = expected["summary"] + expected["operator_runbook_top_level_phase_queue"] = expected["top_level_phase_queue"] + expected["operator_runbook_phase_queues"] = expected["item_phase_queues"] + expected["review_studio_phase_queues"] = expected["item_phase_queues"] + return { + "key": "world-class-phase-queue-consistency", + "label": "World-class phase queues mirror repair rows", + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": [ + report_paths["world_class_preflight"], + report_paths["world_class_operator_runbook"], + report_paths["review_studio"], + ], + "detail": ( + "Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook " + "and Review Studio without counting queue guidance as completion evidence." + ), + } diff --git a/scripts/evidence_consistency_release.py b/scripts/evidence_consistency_release.py new file mode 100644 index 00000000..72e16c42 --- /dev/null +++ b/scripts/evidence_consistency_release.py @@ -0,0 +1,132 @@ +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to verify release evidence refresh instructions." + +SOURCE_REFRESH_HEADER = "After source changes that affect scripts" +CLEAN_LOCK_HEADER = "For final release evidence" +CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`" + +SOURCE_REFRESH_REPORT_COMMANDS = [ + 'python3 scripts/run_output_execution.py --runner-command \'["python3","scripts/local_output_eval_runner.py"]\'', + 'python3 scripts/compile_skill.py . --generated-at "$GENERATED_AT"', + "python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip", + 'python3 scripts/simulate_install.py . --package-dir dist --install-root dist/install-simulation --output-json reports/install_simulation.json --output-md reports/install_simulation.md --generated-at "$GENERATED_AT"', + "python3 scripts/trust_check.py . --output-json reports/security_trust_report.json --output-md reports/security_trust_report.md", + 'python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at "$GENERATED_AT"', + 'python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at "$GENERATED_AT"', + 'python3 scripts/render_adoption_drift_report.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_architecture_maintainability.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/python_compat_check.py . --generated-at "$GENERATED_AT"', + "python3 scripts/probe_runtime_permissions.py . --package-dir dist", + 'python3 scripts/render_review_waivers.py . --generated-at "$GENERATED_AT"', + "python3 scripts/render_review_annotations.py .", + 'python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json --today "$GENERATED_AT"', + 'python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_daily_skillops_report.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_weekly_curator_report.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_skill_os2_audit.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"', + 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"', + "python3 scripts/render_skill_overview.py .", + "python3 scripts/render_skill_interpretation.py .", + "python3 scripts/render_review_viewer.py .", + 'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"', + "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json", + 'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"', +] +CLEAN_LOCK_REPORT_COMMANDS = [ + 'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"', + 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_daily_skillops_report.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_weekly_curator_report.py . --generated-at "$GENERATED_AT"', + "python3 scripts/render_skill_overview.py .", + "python3 scripts/render_skill_interpretation.py .", + "python3 scripts/render_review_viewer.py .", + 'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"', + "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json", + 'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"', +] + + +def section_between(text: str, start: str, end: str) -> str: + if start not in text: + return "" + section = text.split(start, 1)[1] + if end in section: + section = section.split(end, 1)[0] + return section + + +def command_presence(section: str, commands: list[str]) -> dict[str, bool]: + return {command: command in section for command in commands} + + +def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]: + agents_path = skill_dir / "AGENTS.md" + agents_text = agents_path.read_text(encoding="utf-8") if agents_path.exists() else "" + source_refresh = section_between(agents_text, SOURCE_REFRESH_HEADER, CLEAN_LOCK_HEADER) + clean_lock = section_between(agents_text, CLEAN_LOCK_HEADER, CLEAN_LOCK_END) + expected = { + "AGENTS.md": True, + "source_refresh_section": True, + "clean_lock_section": True, + "source_refresh_commands": {command: True for command in SOURCE_REFRESH_REPORT_COMMANDS}, + "clean_lock_commands": {command: True for command in CLEAN_LOCK_REPORT_COMMANDS}, + } + actual = { + "AGENTS.md": agents_path.exists(), + "source_refresh_section": bool(source_refresh), + "clean_lock_section": bool(clean_lock), + "source_refresh_commands": command_presence(source_refresh, SOURCE_REFRESH_REPORT_COMMANDS), + "clean_lock_commands": command_presence(clean_lock, CLEAN_LOCK_REPORT_COMMANDS), + } + return { + "key": "release-evidence-flow-covers-first-class-reports", + "label": "Release evidence flow covers first-class reports", + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": [ + "AGENTS.md", + "reports/output_execution_runs.json", + "reports/install_simulation.json", + "reports/security_trust_report.json", + "reports/registry_audit.json", + "reports/package_verification.json", + "reports/upgrade_check.json", + "reports/adoption_drift_report.json", + "reports/architecture_maintainability.json", + "reports/python_compatibility.json", + "reports/runtime_permission_probes.json", + "reports/review_waivers.json", + "reports/review_annotations.json", + "reports/skill_atlas.json", + "reports/skill_os2_audit.json", + "reports/skill_os2_coverage.json", + "reports/context_budget.json", + "reports/context_budget_summary.json", + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json", + "reports/skill-interpretation.json", + "reports/review-viewer.json", + "reports/world_class_evidence_preflight.json", + "reports/skillops/daily", + "reports/skillops/weekly", + "reports/review-studio.json", + "reports/evidence_consistency.json", + ], + "detail": ( + "Release refresh and clean-lock instructions must regenerate every first-class report " + "before evidence consistency can be trusted." + ), + } diff --git a/scripts/evidence_consistency_skill_os2_review.py b/scripts/evidence_consistency_skill_os2_review.py new file mode 100644 index 00000000..d483a217 --- /dev/null +++ b/scripts/evidence_consistency_skill_os2_review.py @@ -0,0 +1,83 @@ +import ast +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = ( + "Imported by render_evidence_consistency.py to keep Skill OS 2.0 review summary drift checks " + "out of the main consistency renderer." +) + + +def ci_default_target_count(path: Path) -> int | None: + if not path.exists(): + return None + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + if not any(isinstance(target, ast.Name) and target.id == "DEFAULT_TARGETS" for target in node.targets): + continue + if isinstance(node.value, (ast.List, ast.Tuple)): + return len(node.value.elts) + return None + + +def build_skill_os2_review_current_evidence_check( + *, + skill_dir: Path, + skill_os2_review: str, + studio_summary: dict[str, Any], + trust_summary: dict[str, Any], + package_summary: dict[str, Any], + install_summary: dict[str, Any], + benchmark_summary: dict[str, Any], + context_stats: dict[str, Any], + required_text_reports: dict[str, str], + required_reports: dict[str, str], +) -> dict[str, Any]: + ci_target_count = ci_default_target_count(skill_dir / "scripts" / "ci_test.py") + expected_review_snippets = [ + f"score `{studio_summary.get('world_class_score')}`", + f"`{studio_summary.get('gate_count')}` gates", + f"`{studio_summary.get('warning_count')}` warnings", + f"`{trust_summary.get('internal_module_count')}` declared internal modules", + ( + f"`{trust_summary.get('help_smoke_checked_count')} / {trust_summary.get('help_smoke_checked_count')}` " + f"CLI help smoke checks passing across `{trust_summary.get('script_count')}` scripts" + ), + f"`{package_summary.get('archive_entry_count')}` zip entries", + f"archive with `{package_summary.get('archive_entry_count')}` entries", + f"`{install_summary.get('installer_permission_enforced_count')}` installer permission checks enforced", + f"`{install_summary.get('installer_permission_failure_count')}` permission failures", + f"`{benchmark_summary.get('required_artifact_count')}` required artifacts", + f"`{benchmark_summary.get('command_count')}` reproduction commands", + ( + f"initial load `{context_stats.get('estimated_initial_load_tokens')}/" + f"{context_stats.get('context_budget_limit')}`" + ), + f"target count is `{ci_target_count}`", + ] + missing_review_snippets = [snippet for snippet in expected_review_snippets if snippet not in skill_os2_review] + return { + "key": "skill-os-2-review-current-evidence", + "label": "Skill OS 2.0 review summary mirrors current evidence", + "status": "pass" if not missing_review_snippets else "fail", + "expected": expected_review_snippets, + "actual": "all present" if not missing_review_snippets else {"missing": missing_review_snippets}, + "paths": [ + required_text_reports["skill_os2_review"], + required_reports["review_studio"], + required_reports["package_verification"], + required_reports["install_simulation"], + required_reports["trust"], + required_reports["context_budget"], + required_reports["benchmark"], + "scripts/ci_test.py", + ], + "detail": ( + "Manual 2.0 review summaries must not drift from generated gate, package, trust, " + "context, benchmark, or CI evidence." + ), + } diff --git a/scripts/evidence_consistency_world_class.py b/scripts/evidence_consistency_world_class.py new file mode 100644 index 00000000..10a6f969 --- /dev/null +++ b/scripts/evidence_consistency_world_class.py @@ -0,0 +1,222 @@ +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks." + + +def keyed_items(payload: dict[str, Any], collection_key: str) -> dict[str, dict[str, Any]]: + collection = payload.get(collection_key) + if not isinstance(collection, list): + return {} + keyed: dict[str, dict[str, Any]] = {} + for item in collection: + if not isinstance(item, dict): + continue + key = item.get("key") or item.get("evidence_key") + if isinstance(key, str): + keyed[key] = item + return keyed + + +def command_key_set(item: dict[str, Any]) -> set[str]: + commands = item.get("commands") + if isinstance(commands, dict): + return {key for key, value in commands.items() if isinstance(key, str) and value} + if isinstance(commands, list): + keys: set[str] = set() + for command in commands: + if isinstance(command, dict) and isinstance(command.get("key"), str) and command.get("command"): + keys.add(command["key"]) + return keys + return set() + + +def world_class_review_action_steps(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]: + actions = review_studio.get("review_actions") + if not isinstance(actions, list): + return {} + for action in actions: + if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence": + continue + steps = action.get("evidence_steps") + if not isinstance(steps, list): + return {} + keyed: dict[str, dict[str, Any]] = {} + for step in steps: + if isinstance(step, dict) and isinstance(step.get("key"), str): + keyed[step["key"]] = step + return keyed + return {} + + +def command_groups_present(command_keys: set[str]) -> dict[str, bool]: + return { + "prepare_submission": "prepare_submission" in command_keys, + "validate_intake": "validate_intake" in command_keys, + "submission_review": bool({"submission_review", "review_queue"} & command_keys), + "refresh_ledger": "refresh_ledger" in command_keys, + "guard_claim": "guard_claim" in command_keys, + } + + +def has_next_action(item: dict[str, Any]) -> bool: + for key in ["next_action", "audit_next_action"]: + value = item.get(key) + if isinstance(value, str) and value.strip(): + return True + return False + + +def command_expectations(keys: list[str]) -> dict[str, dict[str, dict[str, bool]]]: + expected_group = { + "prepare_submission": True, + "validate_intake": True, + "submission_review": True, + "refresh_ledger": True, + "guard_claim": True, + } + return { + key: { + "intake": dict(expected_group), + "operator_runbook": dict(expected_group), + "review_studio": dict(expected_group), + } + for key in keys + } + + +def build_world_class_workflow_check( + *, + ledger: dict[str, Any], + world_class_plan: dict[str, Any], + world_class_intake: dict[str, Any], + world_class_submission_review: dict[str, Any], + world_class_operator_runbook: dict[str, Any], + review_studio: dict[str, Any], + report_paths: dict[str, str], +) -> dict[str, Any]: + ledger_summary = ledger.get("summary") if isinstance(ledger.get("summary"), dict) else {} + plan_summary = world_class_plan.get("summary") if isinstance(world_class_plan.get("summary"), dict) else {} + intake_summary = world_class_intake.get("summary") if isinstance(world_class_intake.get("summary"), dict) else {} + submission_review_summary = ( + world_class_submission_review.get("summary") + if isinstance(world_class_submission_review.get("summary"), dict) + else {} + ) + operator_runbook_summary = ( + world_class_operator_runbook.get("summary") + if isinstance(world_class_operator_runbook.get("summary"), dict) + else {} + ) + + ledger_items = keyed_items(ledger, "entries") + plan_tasks = keyed_items(world_class_plan, "tasks") + intake_checklist = keyed_items(world_class_intake, "operator_checklist") + submission_review_items = keyed_items(world_class_submission_review, "items") + operator_runbook_items = keyed_items(world_class_operator_runbook, "items") + coordination_plan = ( + world_class_operator_runbook.get("coordination_plan") + if isinstance(world_class_operator_runbook.get("coordination_plan"), list) + else [] + ) + release_gate = ( + world_class_operator_runbook.get("release_gate") + if isinstance(world_class_operator_runbook.get("release_gate"), dict) + else {} + ) + review_action_steps = world_class_review_action_steps(review_studio) + pending_keys = sorted(key for key, item in ledger_items.items() if item.get("status") != "accepted") + operator_coordination_keys = sorted( + key + for key in {str(step.get("evidence_key", "")) for step in coordination_plan if isinstance(step, dict)} + if key + ) + + actual_command_groups = { + key: { + "intake": command_groups_present(command_key_set(intake_checklist.get(key, {}))), + "operator_runbook": command_groups_present(command_key_set(operator_runbook_items.get(key, {}))), + "review_studio": command_groups_present(command_key_set(review_action_steps.get(key, {}))), + } + for key in pending_keys + } + expected = { + "keys": pending_keys, + "pending_count": ledger_summary.get("pending_count"), + "human_pending_count": ledger_summary.get("human_pending_count"), + "external_pending_count": ledger_summary.get("external_pending_count"), + "source_check_count": ledger_summary.get("source_check_count"), + "source_pass_count": ledger_summary.get("source_pass_count"), + "source_blocked_count": ledger_summary.get("source_blocked_count"), + "plan_keys": pending_keys, + "intake_keys": pending_keys, + "submission_review_keys": pending_keys, + "operator_runbook_keys": pending_keys, + "operator_coordination_keys": pending_keys, + "operator_coordination_counts_as_completion": False, + "operator_release_gate_ready": ledger_summary.get("ready_to_claim_world_class") is True, + "operator_release_gate_counts_as_completion": False, + "review_studio_keys": pending_keys, + "intake_ready_to_claim_world_class": False, + "submission_review_ready_to_claim_world_class": False, + "submission_review_counts_as_completion": False, + "operator_runbook_ready_to_claim_world_class": False, + "operator_runbook_counts_as_completion": False, + "next_actions_present": {key: True for key in pending_keys}, + "commands": command_expectations(pending_keys), + } + actual = { + "keys": pending_keys, + "pending_count": plan_summary.get("task_count"), + "human_pending_count": plan_summary.get("human_task_count"), + "external_pending_count": plan_summary.get("external_task_count"), + "source_check_count": submission_review_summary.get("source_check_count"), + "source_pass_count": submission_review_summary.get("source_pass_count"), + "source_blocked_count": submission_review_summary.get("source_blocked_count"), + "plan_keys": sorted(plan_tasks), + "intake_keys": sorted(intake_checklist), + "submission_review_keys": sorted(submission_review_items), + "operator_runbook_keys": sorted(operator_runbook_items), + "operator_coordination_keys": operator_coordination_keys, + "operator_coordination_counts_as_completion": operator_runbook_summary.get( + "coordination_counts_as_completion" + ), + "operator_release_gate_ready": release_gate.get("ready"), + "operator_release_gate_counts_as_completion": release_gate.get("counts_as_completion"), + "review_studio_keys": sorted(review_action_steps), + "intake_ready_to_claim_world_class": intake_summary.get("ready_to_claim_world_class"), + "submission_review_ready_to_claim_world_class": submission_review_summary.get("ready_to_claim_world_class"), + "submission_review_counts_as_completion": submission_review_summary.get( + "review_counts_submission_as_completion" + ), + "operator_runbook_ready_to_claim_world_class": operator_runbook_summary.get("ready_to_claim_world_class"), + "operator_runbook_counts_as_completion": operator_runbook_summary.get("runbook_counts_as_completion"), + "next_actions_present": { + key: all( + has_next_action(collection.get(key, {})) + for collection in [plan_tasks, intake_checklist, submission_review_items, review_action_steps] + ) + for key in pending_keys + }, + "commands": actual_command_groups, + } + return { + "key": "world-class-evidence-workflow-coverage", + "label": "World-class evidence workflows cover every pending ledger entry", + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": [ + report_paths["world_class_ledger"], + report_paths["world_class_plan"], + report_paths["world_class_intake"], + report_paths["world_class_submission_review"], + report_paths["world_class_operator_runbook"], + report_paths["review_studio"], + ], + "detail": ( + "Every pending world-class evidence key must have matching plan, intake, submission review, " + "operator runbook, and Review Studio actions without counting planned work as completion." + ), + } diff --git a/scripts/export_skill_ir.py b/scripts/export_skill_ir.py index 08386a68..f02b913c 100644 --- a/scripts/export_skill_ir.py +++ b/scripts/export_skill_ir.py @@ -51,6 +51,8 @@ KEY_REPORTS = [ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -157,7 +159,7 @@ def report_list(skill_dir: Path) -> list[str]: return [rel for rel in KEY_REPORTS if (skill_dir / rel).exists()] -def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, limit: int = 80) -> list[str]: +def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, limit: int | None = None) -> list[str]: target = skill_dir / folder if not target.exists(): return [] @@ -168,7 +170,7 @@ def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, li if suffixes is not None and path.suffix not in suffixes: continue paths.append(str(path.relative_to(skill_dir))) - if len(paths) >= limit: + if limit is not None and len(paths) >= limit: break return paths diff --git a/scripts/html_rendering.py b/scripts/html_rendering.py new file mode 100644 index 00000000..afb7f750 --- /dev/null +++ b/scripts/html_rendering.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +"""Shared HTML rendering helpers for static report generators.""" + +import html +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Used by report renderers to escape HTML while preserving meaningful falsey values." + + +def html_text(value: Any) -> str: + """Escape text for HTML without dropping 0 or False values.""" + return html.escape("" if value is None else str(value), quote=True) diff --git a/scripts/import_output_review_decisions.py b/scripts/import_output_review_decisions.py new file mode 100644 index 00000000..2a4a8336 --- /dev/null +++ b/scripts/import_output_review_decisions.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +import argparse +import csv +import json +from datetime import date +from pathlib import Path +from typing import Any + +from adjudicate_output_review import ( + DEFAULT_ANSWER_KEY, + DEFAULT_BLIND_PACK, + DEFAULT_DECISIONS, + DEFAULT_OUTPUT_JSON, + DEFAULT_OUTPUT_MD, + adjudicate_output_review, + build_decision_template, + default_reviewer_attestation, + normalize_variant, + pair_index, + review_integrity, +) +from output_review_privacy import forbidden_decision_field_paths + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file." + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> tuple[Any, list[str]]: + try: + return json.loads(path.read_text(encoding="utf-8")), [] + except json.JSONDecodeError as exc: + return None, [f"Invalid JSON file {display_path(path)}: {exc}"] + + +def detect_format(path: Path, requested: str) -> str: + if requested != "auto": + return requested + suffix = path.suffix.lower() + if suffix == ".jsonl": + return "jsonl" + if suffix == ".csv": + return "csv" + return "json" + + +def read_decision_source(path: Path, source_format: str) -> tuple[dict[str, Any], list[dict[str, Any]], list[str]]: + if not path.exists(): + return {}, [], [f"Missing decision source: {display_path(path)}"] + if source_format == "json": + payload, failures = load_json(path) + if failures: + return {}, [], failures + if isinstance(payload, list): + return {}, payload, [] + if not isinstance(payload, dict): + return {}, [], ["JSON decision source must be an object or list"] + decisions = payload.get("decisions", []) + if not isinstance(decisions, list): + return payload, [], ["decisions must be a list"] + return payload, decisions, [] + if source_format == "jsonl": + decisions: list[dict[str, Any]] = [] + failures: list[str] = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + item = json.loads(line) + except json.JSONDecodeError as exc: + failures.append(f"JSONL line {line_number} is invalid: {exc}") + continue + if not isinstance(item, dict): + failures.append(f"JSONL line {line_number} must be an object") + continue + decisions.append(item) + return {}, decisions, failures + if source_format == "csv": + with path.open("r", encoding="utf-8", newline="") as handle: + return {}, [dict(row) for row in csv.DictReader(handle)], [] + return {}, [], [f"Unsupported decision source format: {source_format}"] + + +def known_case_ids(blind_pack_path: Path) -> tuple[set[str], list[str]]: + payload, failures = load_json(blind_pack_path) + if failures: + return set(), failures + if not isinstance(payload, dict): + return set(), [f"Blind pack root must be an object: {display_path(blind_pack_path)}"] + pairs = pair_index(payload) + if not pairs: + return set(), [f"Blind pack has no pairs: {display_path(blind_pack_path)}"] + return set(pairs), [] + + +def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None]: + if value is None or str(value).strip() == "": + return None, None + try: + parsed = float(value) + except (TypeError, ValueError): + return None, f"{case_id}: confidence must be numeric" + if parsed < 0 or parsed > 1: + return None, f"{case_id}: confidence must be between 0 and 1" + return round(parsed, 3), None + + +def normalize_decisions( + source_items: list[dict[str, Any]], + case_ids: set[str], +) -> tuple[list[dict[str, Any]], list[str]]: + failures: list[str] = [] + normalized: list[dict[str, Any]] = [] + seen: set[str] = set() + for index, item in enumerate(source_items, start=1): + if not isinstance(item, dict): + failures.append(f"decision #{index} must be an object") + continue + blocked_fields = forbidden_decision_field_paths(item, f"decision #{index}") + if blocked_fields: + failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}") + case_id = str(item.get("case_id", "")).strip() + if not case_id: + failures.append(f"decision #{index} is missing case_id") + continue + if case_id not in case_ids: + failures.append(f"decision #{index} references unknown case_id: {case_id}") + if case_id in seen: + failures.append(f"duplicate decision for case_id: {case_id}") + seen.add(case_id) + winner = normalize_variant(item.get("winner_variant", item.get("winner", item.get("reviewer_winner_variant", "")))) + if winner and winner not in {"A", "B"}: + failures.append(f"{case_id}: winner_variant must be A or B") + confidence, confidence_failure = parse_confidence(item.get("confidence"), case_id) + if confidence_failure: + failures.append(confidence_failure) + reason = str(item.get("reason", "")).strip() + if winner in {"A", "B"} and not reason: + failures.append(f"{case_id}: reason is required for imported human decisions") + normalized.append( + { + "case_id": case_id, + "winner_variant": winner, + "confidence": confidence, + "reason": reason, + } + ) + return normalized, failures + + +def canonical_payload( + blind_pack_path: Path, + source_path: Path, + source_format: str, + reviewer: str, + reviewed_at: str, + decisions: list[dict[str, Any]], + reviewer_attestation: dict[str, Any], +) -> dict[str, Any]: + blind_pack = json.loads(blind_pack_path.read_text(encoding="utf-8")) + template = build_decision_template(blind_pack) + completed = sum(1 for item in decisions if item.get("winner_variant") in {"A", "B"}) + return { + "schema_version": "1.0", + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "review_integrity": review_integrity(blind_pack), + "reviewer_attestation": reviewer_attestation, + "decision_contract": template["decision_contract"], + "import_contract": { + "schema_version": "1.0", + "source_path": display_path(source_path), + "source_format": source_format, + "raw_content_allowed": False, + "answer_key_fields_allowed": False, + "answer_key_opened_by_importer": False, + "all_or_nothing_write": True, + "decision_count": len(decisions), + "completed_decision_count": completed, + "pending_decision_count": len(decisions) - completed, + "blind_review_attestation_supported": True, + "blind_review_attested": reviewer_attestation.get("blind_review_completed_before_answer_key") is True + and reviewer_attestation.get("answer_key_not_opened_before_decisions") is True, + }, + "decisions": decisions, + } + + +def normalize_reviewer_attestation(source_meta: dict[str, Any], blind_review_attested: bool) -> dict[str, Any]: + attestation = default_reviewer_attestation() + source_attestation = source_meta.get("reviewer_attestation", {}) + if isinstance(source_attestation, dict): + for key in attestation: + if source_attestation.get(key) is True: + attestation[key] = True + elif source_attestation.get(key) is False: + attestation[key] = False + if blind_review_attested: + attestation["blind_review_completed_before_answer_key"] = True + attestation["answer_key_not_opened_before_decisions"] = True + return attestation + + +def import_output_review_decisions( + source_path: Path, + blind_pack_path: Path, + output_json: Path, + source_format: str = "auto", + reviewer: str = "", + reviewed_at: str = "", + run_adjudication: bool = False, + answer_key_path: Path = DEFAULT_ANSWER_KEY, + adjudication_json: Path = DEFAULT_OUTPUT_JSON, + adjudication_md: Path = DEFAULT_OUTPUT_MD, + blind_review_attested: bool = False, +) -> dict[str, Any]: + detected_format = detect_format(source_path, source_format) + source_meta, source_items, failures = read_decision_source(source_path, detected_format) + reviewer = reviewer or str(source_meta.get("reviewer", "")).strip() + reviewed_at = reviewed_at or str(source_meta.get("reviewed_at", "")).strip() + if not reviewer: + failures.append("reviewer is required for imported human decisions") + if not reviewed_at: + failures.append("reviewed_at is required for imported human decisions") + reviewer_attestation = normalize_reviewer_attestation(source_meta, blind_review_attested) + case_ids, case_failures = known_case_ids(blind_pack_path) + failures.extend(case_failures) + decisions, decision_failures = normalize_decisions(source_items, case_ids) + failures.extend(decision_failures) + payload: dict[str, Any] = { + "schema_version": "1.0", + "ok": not failures, + "summary": { + "decision_count": len(decisions), + "completed_decision_count": sum(1 for item in decisions if item.get("winner_variant") in {"A", "B"}), + "pending_decision_count": sum(1 for item in decisions if not item.get("winner_variant")), + "failure_count": len(failures), + "canonical_written": False, + "adjudication_run": False, + }, + "artifacts": { + "source": display_path(source_path), + "blind_pack": display_path(blind_pack_path), + "decisions": display_path(output_json), + "adjudication_json": display_path(adjudication_json), + "adjudication_markdown": display_path(adjudication_md), + }, + "failures": failures, + } + if failures: + return payload + canonical = canonical_payload( + blind_pack_path, + source_path, + detected_format, + reviewer, + reviewed_at, + decisions, + reviewer_attestation, + ) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(canonical, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + payload["summary"]["canonical_written"] = True + payload["canonical"] = canonical + if run_adjudication: + adjudication = adjudicate_output_review( + blind_pack_path=blind_pack_path, + answer_key_path=answer_key_path, + decisions_path=output_json, + output_json=adjudication_json, + output_md=adjudication_md, + ) + payload["summary"]["adjudication_run"] = True + payload["summary"]["adjudication_ok"] = adjudication["ok"] + payload["summary"]["adjudication_pending_count"] = adjudication["summary"]["pending_count"] + payload["summary"]["adjudication_invalid_decision_count"] = adjudication["summary"]["invalid_decision_count"] + payload["adjudication_summary"] = adjudication["summary"] + if not adjudication["ok"]: + payload["ok"] = False + payload["failures"] = list(adjudication.get("failures", [])) + payload["summary"]["failure_count"] = len(payload["failures"]) + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Import blind A/B reviewer decisions into the canonical output-review decision file.") + parser.add_argument("--input", required=True, help="Reviewer decision source in JSON, JSONL, or CSV format.") + parser.add_argument("--format", choices=["auto", "json", "jsonl", "csv"], default="auto") + parser.add_argument("--blind-pack", default=str(DEFAULT_BLIND_PACK)) + parser.add_argument("--output-json", default=str(DEFAULT_DECISIONS)) + parser.add_argument("--reviewer") + parser.add_argument("--reviewed-at", default=date.today().isoformat()) + parser.add_argument( + "--blind-review-attested", + action="store_true", + help="Attest that reviewer choices were completed before the answer key was opened.", + ) + parser.add_argument("--run-adjudication", action="store_true") + parser.add_argument("--answer-key", default=str(DEFAULT_ANSWER_KEY)) + parser.add_argument("--adjudication-json", default=str(DEFAULT_OUTPUT_JSON)) + parser.add_argument("--adjudication-md", default=str(DEFAULT_OUTPUT_MD)) + args = parser.parse_args() + payload = import_output_review_decisions( + source_path=Path(args.input).resolve(), + blind_pack_path=Path(args.blind_pack).resolve(), + output_json=Path(args.output_json).resolve(), + source_format=args.format, + reviewer=args.reviewer or "", + reviewed_at=args.reviewed_at or "", + run_adjudication=args.run_adjudication, + answer_key_path=Path(args.answer_key).resolve(), + adjudication_json=Path(args.adjudication_json).resolve(), + adjudication_md=Path(args.adjudication_md).resolve(), + blind_review_attested=args.blind_review_attested, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/import_telemetry_events.py b/scripts/import_telemetry_events.py new file mode 100644 index 00000000..d2802fe8 --- /dev/null +++ b/scripts/import_telemetry_events.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from typing import Any + +from render_adoption_drift_report import display_path, normalize_event, render_report, skill_defaults + + +def load_candidate_events( + input_jsonl: Path, + defaults: dict[str, str], + default_source: str, + default_command: str, +) -> tuple[list[dict[str, str]], list[str]]: + events: list[dict[str, str]] = [] + failures: list[str] = [] + for index, line in enumerate(input_jsonl.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + raw = json.loads(line) + except json.JSONDecodeError as exc: + failures.append(f"line {index}: invalid JSONL event: {exc.msg}") + continue + if not isinstance(raw, dict): + failures.append(f"line {index}: telemetry event must be a JSON object") + continue + raw.setdefault("source", default_source) + raw.setdefault("command", default_command) + event, event_failures = normalize_event(raw, defaults, f"line {index}") + failures.extend(event_failures) + if event: + events.append(event) + return events, failures + + +def append_events(events_jsonl: Path, events: list[dict[str, str]]) -> None: + events_jsonl.parent.mkdir(parents=True, exist_ok=True) + with events_jsonl.open("a", encoding="utf-8") as handle: + for event in events: + handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n") + + +def import_events( + skill_dir: Path, + input_jsonl: Path, + events_jsonl: Path | None = None, + output_json: Path | None = None, + output_md: Path | None = None, + generated_at: str | None = None, + default_source: str = "external", + default_command: str = "external-client", + dry_run: bool = False, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + input_jsonl = input_jsonl.resolve() + reports_dir = skill_dir / "reports" + events_jsonl = (events_jsonl or reports_dir / "telemetry_events.jsonl").resolve() + output_json = (output_json or reports_dir / "adoption_drift_report.json").resolve() + output_md = (output_md or reports_dir / "adoption_drift_report.md").resolve() + failures: list[str] = [] + if not input_jsonl.exists(): + failures.append(f"Input telemetry JSONL does not exist: {display_path(input_jsonl)}") + candidate_events: list[dict[str, str]] = [] + else: + candidate_events, load_failures = load_candidate_events( + input_jsonl, + skill_defaults(skill_dir), + default_source, + default_command, + ) + failures.extend(load_failures) + + adoption_report: dict[str, Any] | None = None + if not failures and not dry_run: + append_events(events_jsonl, candidate_events) + adoption_report = render_report( + skill_dir, + events_jsonl=events_jsonl, + output_json=output_json, + output_md=output_md, + generated_at=generated_at, + ) + + return { + "ok": not failures, + "schema_version": "1.0", + "skill_dir": display_path(skill_dir), + "input_jsonl": display_path(input_jsonl), + "events_jsonl": display_path(events_jsonl), + "dry_run": dry_run, + "imported_count": 0 if failures or dry_run else len(candidate_events), + "candidate_count": len(candidate_events), + "failures": failures, + "imported_preview": candidate_events[:5], + "artifacts": { + "events_jsonl": display_path(events_jsonl), + "adoption_drift_json": display_path(output_json), + "adoption_drift_md": display_path(output_md), + }, + "adoption_drift": { + "summary": adoption_report.get("summary", {}) if adoption_report else {}, + "artifacts": adoption_report.get("artifacts", {}) if adoption_report else {}, + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Import external metadata-only telemetry into a local skill event stream.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--input-jsonl", required=True) + parser.add_argument("--events-jsonl") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--generated-at") + parser.add_argument("--source", choices=["external", "manual", "unknown", "yao_cli"], default="external") + parser.add_argument("--command", default="external-client") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + report = import_events( + Path(args.skill_dir), + Path(args.input_jsonl), + events_jsonl=Path(args.events_jsonl) if args.events_jsonl else None, + output_json=Path(args.output_json) if args.output_json else None, + output_md=Path(args.output_md) if args.output_md else None, + generated_at=args.generated_at, + default_source=args.source, + default_command=args.command, + dry_run=args.dry_run, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_skill.py b/scripts/init_skill.py index be9b33e8..ff67e8fc 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -21,6 +21,7 @@ from render_reference_scan import parse_reference, render_reference_scan from render_reference_synthesis import render_reference_synthesis from render_review_studio import render_review_studio from render_review_viewer import render_review_viewer +from render_skill_interpretation import render_skill_interpretation from render_skill_overview import render_skill_overview from render_system_model import render_system_model @@ -73,19 +74,20 @@ README_TEMPLATE = """# {title} 4. Review `reports/intent-confidence.md` to see whether the real job, inputs, outputs, and exclusions are clear enough yet. 5. Open `reports/reference-synthesis.md` to see the GitHub benchmarks plus curated official, research, and principle tracks in one place. 6. Follow the workflow steps in `SKILL.md`. -7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right. -8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view. -9. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up. -10. Open `reports/review-viewer.html` for a compact visual review of the package. -11. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. -12. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. -13. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. -14. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. -15. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts. -16. Review `reports/iteration-directions.md` for the three most valuable next moves. -17. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. -18. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals. -19. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates. +7. Open `reports/skill-interpretation.html` first for the first-class bilingual interpretation report: role, principle, scenarios, trigger, inputs, outputs, highlights, risks, assets, and upgrade directions. It defaults to Simplified Chinese and includes an English switch in the top right. +8. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. +9. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view. +10. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up. +11. Open `reports/review-viewer.html` for a compact visual review of the package. +12. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. +13. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. +14. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. +15. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. +16. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts. +17. Review `reports/iteration-directions.md` for the three most valuable next moves. +18. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. +19. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals. +20. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates. ## Honest Boundaries @@ -109,6 +111,7 @@ README_TEMPLATE = """# {title} - `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points - `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance - `reports/compiled_targets.md`: target compiler report showing generated contracts, adapter modes, preserved semantics, warnings, and unsupported features +- `reports/skill-interpretation.html`: first-class bilingual interpretation report for role, principle, scenarios, trigger, inputs, outputs, quality evidence, risks, assets, highlights, and upgrade directions - `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap - `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps @@ -129,6 +132,7 @@ compatibility: - "openai" - "claude" - "generic" + - "vscode" activation: mode: "manual" paths: [] @@ -143,6 +147,7 @@ compatibility: openai: "metadata-adapter" claude: "neutral-source-plus-adapter" generic: "neutral-source" + vscode: "agent-skills-source-with-vscode-notes" """ @@ -213,6 +218,7 @@ def build_manifest(name: str, mode: str, archetype: str) -> dict: "claude", "generic", "agent-skills-compatible", + "vscode", ], "factory_components": [ "references", @@ -225,23 +231,26 @@ def build_manifest(name: str, mode: str, archetype: str) -> dict: def build_report_view(artifacts: dict) -> dict: html_report = artifacts.get("skill_overview_html", "") json_report = artifacts.get("skill_overview_json", "") + interpretation_report = artifacts.get("skill_interpretation_html", "") system_model = artifacts.get("system_model_md", "") review_studio = artifacts.get("review_studio_html", "") return { "title": "Skill 总结报告", "html_report": html_report, "json_report": json_report, + "interpretation_report": interpretation_report, "system_model": system_model, "review_studio": review_studio, "message": ( - f"Skill 已创建完成。建议先打开总结报告：{html_report}。" - "它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线；" + f"Skill 已创建完成。建议先打开解读报告：{interpretation_report}；再查看总结报告：{html_report}。" + "解读报告会用中文简体默认展示这个 Skill 的作用、原理、使用场景、触发方式、输入输出、亮点和后续升级方向；" + "总结报告会展示概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线；" f"然后打开 Review Studio 2.0：{review_studio}，检查意图、触发、输出评测、运行一致性、信任和发布闸门。" "后续 reviewer 的文件级或行级意见可以记录到 reports/review_annotations.md。" "如需审查平台适配细节，请打开 reports/compiled_targets.md。" "报告默认使用中文简体，右上角可以切换英文版。" ), - "next_action": "Open reports/skill-overview.html before editing more files.", + "next_action": "Open reports/skill-interpretation.html before editing more files.", } @@ -356,6 +365,7 @@ def initialize_skill( review_waivers = render_review_waivers(root) review_annotations = render_review_annotations(root) overview = render_skill_overview(root) + interpretation = render_skill_interpretation(root) review_viewer = render_review_viewer(root) review_studio = render_review_studio(root) artifacts = { @@ -366,6 +376,8 @@ def initialize_skill( "intent_context_json": intent_confidence["artifacts"]["context_json"], "skill_overview_html": overview["artifacts"]["html"], "skill_overview_json": overview["artifacts"]["json"], + "skill_interpretation_html": interpretation["artifacts"]["html"], + "skill_interpretation_json": interpretation["artifacts"]["json"], "intent_dialogue_md": intent_dialogue["artifacts"]["markdown"], "intent_dialogue_json": intent_dialogue["artifacts"]["json"], "reference_scan_md": reference_scan["artifacts"]["markdown"], diff --git a/scripts/optimize_description.py b/scripts/optimize_description.py index 560626e5..9bfee6d5 100644 --- a/scripts/optimize_description.py +++ b/scripts/optimize_description.py @@ -4,6 +4,7 @@ import json from pathlib import Path from context_sizer import estimate_tokens +from description_optimizer_reporting import render_markdown from judge_blind_eval import evaluate_judge from trigger_eval import ( compare_reports, @@ -539,145 +540,6 @@ def optimize( return report -def render_markdown(report: dict, title: str) -> str: - lines = [ - f"# {title}", - "", - f"Winner: `{report['winner']['label']}`", - "", - f"- current tokens: `{report['current_candidate']['estimated_tokens']}`", - f"- winner tokens: `{report['winner']['estimated_tokens']}`", - ] - if report["baseline"]: - lines.append(f"- baseline tokens: `{report['baseline']['estimated_tokens']}`") - lines.extend( - [ - "", - "## Winner", - "", - report["winner"]["description"], - "", - "## Candidate Ranking", - "", - "| Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", - ] - ) - for candidate in report["candidates"]: - holdout = candidate.get("holdout", {}) - lines.append( - f"| `{candidate['label']}` | {candidate['estimated_tokens']} | {candidate['dev']['false_positives']} | {candidate['dev']['false_negatives']} | {candidate['dev']['near_neighbor_pass_rate']} | {holdout.get('false_positives', '-')} | {holdout.get('false_negatives', '-')} |" - ) - - lines.extend( - [ - "", - "## Acceptance Gates", - "", - "| Gate | Winner FP | Winner FN | Current FP | Current FN | Baseline FP | Baseline FN |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", - ] - ) - for gate_name, gate in ( - ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), - ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), - ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), - ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), - ): - winner_gate = gate.get("winner") or {} - current_gate = gate.get("current") or {} - baseline_gate = gate.get("baseline") or {} - if not winner_gate and not current_gate and not baseline_gate: - continue - lines.append( - f"| {gate_name} | {winner_gate.get('false_positives', '-')} | {winner_gate.get('false_negatives', '-')} | {current_gate.get('false_positives', '-')} | {current_gate.get('false_negatives', '-')} | {baseline_gate.get('false_positives', '-')} | {baseline_gate.get('false_negatives', '-')} |" - ) - - lines.extend( - [ - "", - "## Calibration", - "", - "| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |", - "| --- | ---: | --- | ---: | ---: | ---: |", - ] - ) - for gate_name, gate in ( - ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), - ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), - ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), - ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), - ): - winner_calibration = gate.get("winner_calibration") or {} - current_calibration = gate.get("current_calibration") or {} - baseline_calibration = gate.get("baseline_calibration") or {} - if not winner_calibration and not current_calibration and not baseline_calibration: - continue - lines.append( - f"| {gate_name} | {winner_calibration.get('score_gap', '-')} | {winner_calibration.get('risk_band', '-')} | {winner_calibration.get('boundary_case_rate', '-')} | {current_calibration.get('score_gap', '-')} | {baseline_calibration.get('score_gap', '-')} |" - ) - - lines.extend( - [ - "", - "## Judge Blind Summary", - "", - "| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement |", - "| --- | ---: | ---: | ---: | ---: |", - ] - ) - judge_gate = report["acceptance_gates"]["judge_blind_holdout_non_regression"] - judge_winner = (judge_gate.get("winner") or {}).get("judge_summary") or {} - judge_current = (judge_gate.get("current") or {}).get("judge_summary") or {} - judge_baseline = (judge_gate.get("baseline") or {}).get("judge_summary") or {} - if judge_winner or judge_current or judge_baseline: - lines.append( - f"| Judge Blind Holdout | {judge_winner.get('agreement_rate', '-')} | {judge_winner.get('mean_confidence', '-')} | {judge_current.get('agreement_rate', '-')} | {judge_baseline.get('agreement_rate', '-')} |" - ) - - lines.extend( - [ - "", - "## Family Health", - "", - "| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families |", - "| --- | --- | --- | --- | --- |", - ] - ) - for gate_name, gate in ( - ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), - ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), - ("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]), - ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), - ): - winner_health = gate.get("winner_family_health") or {} - current_health = gate.get("current_family_health") or {} - baseline_health = gate.get("baseline_family_health") or {} - if not winner_health and not current_health and not baseline_health: - continue - weakest = winner_health.get("weakest_family") or {} - weakest_label = ( - f"{weakest.get('family')} ({weakest.get('errors')} errors)" - if weakest.get("family") - else "-" - ) - lines.append( - f"| {gate_name} | {winner_health.get('clean_family_count', '-')}/{winner_health.get('family_count', '-')} | {weakest_label} | {current_health.get('clean_family_count', '-')}/{current_health.get('family_count', '-')} | {baseline_health.get('clean_family_count', '-')}/{baseline_health.get('family_count', '-')} |" - ) - - lines.extend( - [ - "", - "## Selection Logic", - "", - "Ordered by:", - ] - ) - for item in report["selection_logic"]["priority"]: - lines.append(f"- {item}") - return "\n".join(lines) + "\n" - - def main() -> None: parser = argparse.ArgumentParser( description="Generate and score description candidates on dev, holdout, blind, and adversarial suites." diff --git a/scripts/output_review_privacy.py b/scripts/output_review_privacy.py new file mode 100644 index 00000000..a19011cc --- /dev/null +++ b/scripts/output_review_privacy.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively." + +RAW_CONTENT_FIELDS = { + "api_key", + "assistant_message", + "assistant_messages", + "baseline_output", + "credential", + "credentials", + "input", + "inputs", + "message", + "messages", + "model_output", + "output", + "outputs", + "prompt", + "prompts", + "raw_content", + "raw_output", + "raw_prompt", + "raw_provider_prompt", + "raw_user_content", + "secret", + "secrets", + "token", + "transcript", + "transcripts", + "user_message", + "user_messages", + "with_skill_output", +} + +ANSWER_KEY_FIELDS = { + "answer_key", + "baseline_label", + "expected", + "expected_winner", + "expected_winner_role", + "expected_winner_variant", + "label", + "variant_label", + "with_skill_label", +} + +BLOCKED_DECISION_FIELDS = RAW_CONTENT_FIELDS | ANSWER_KEY_FIELDS + + +def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]: + found: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + key_text = str(key).strip() + child_path = f"{prefix}.{key_text}" + if key_text.casefold() in BLOCKED_DECISION_FIELDS: + found.append(child_path) + found.extend(forbidden_decision_field_paths(child, child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]")) + return found + + +def forbidden_raw_content_field_paths(value: Any, prefix: str) -> list[str]: + found: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + key_text = str(key).strip() + child_path = f"{prefix}.{key_text}" + if key_text.casefold() in RAW_CONTENT_FIELDS: + found.append(child_path) + found.extend(forbidden_raw_content_field_paths(child, child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + found.extend(forbidden_raw_content_field_paths(child, f"{prefix}[{index}]")) + return found diff --git a/scripts/prepare_output_review_kit.py b/scripts/prepare_output_review_kit.py new file mode 100644 index 00000000..26c972ad --- /dev/null +++ b/scripts/prepare_output_review_kit.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""Prepare a reviewer-facing blind A/B output review kit.""" + +import argparse +import json +from pathlib import Path +from typing import Any + +from adjudicate_output_review import ( + build_decision_template, + confidence_value, + decision_index, + display_path, + load_json, + normalize_variant, +) +from html_rendering import html_text + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_BLIND_PACK_JSON = ROOT / "reports" / "output_blind_review_pack.json" +DEFAULT_BLIND_PACK_MD = ROOT / "reports" / "output_blind_review_pack.md" +DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json" +DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_kit.json" +DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_kit.md" +DEFAULT_OUTPUT_HTML = ROOT / "reports" / "output_review_kit.html" + + +def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]: + if not path.exists(): + return {"schema_version": "1.0", "decisions": []}, [] + return load_json(path) + + +def pairs_from_pack(blind_pack: dict[str, Any]) -> list[dict[str, Any]]: + pairs = blind_pack.get("pairs", []) + if not isinstance(pairs, list): + return [] + return [item for item in pairs if isinstance(item, dict) and item.get("case_id")] + + +def decision_state(decision: dict[str, Any] | None) -> dict[str, Any]: + if not decision: + return { + "status": "awaiting-decision", + "winner_variant_recorded": False, + "confidence_recorded": False, + "reason_recorded": False, + "blocking_reason": "No reviewer choice has been recorded yet.", + } + winner = normalize_variant(decision.get("winner_variant", decision.get("reviewer_winner_variant", decision.get("winner", "")))) + confidence, confidence_failure = confidence_value(decision.get("confidence")) + reason = str(decision.get("reason", "")).strip() + if not winner and confidence is None and not reason: + return { + "status": "awaiting-decision", + "winner_variant_recorded": False, + "confidence_recorded": False, + "reason_recorded": False, + "blocking_reason": "Decision template exists but this row is still blank.", + } + failures = [] + if winner not in {"A", "B"}: + failures.append("winner_variant must be A or B") + if confidence_failure: + failures.append(confidence_failure) + if not reason: + failures.append("reason should explain the reviewer rationale") + if failures: + return { + "status": "needs-fix", + "winner_variant_recorded": winner in {"A", "B"}, + "confidence_recorded": confidence is not None, + "reason_recorded": bool(reason), + "blocking_reason": "; ".join(failures), + } + return { + "status": "ready-for-adjudication", + "winner_variant_recorded": True, + "confidence_recorded": confidence is not None, + "reason_recorded": True, + "blocking_reason": "Reviewer choice is complete enough for adjudication.", + } + + +def build_cases(blind_pack: dict[str, Any], decisions: dict[str, dict[str, Any]]) -> list[dict[str, Any]]: + cases = [] + for pair in pairs_from_pack(blind_pack): + case_id = str(pair.get("case_id", "")) + rubric = [] + for item in pair.get("rubric", []) if isinstance(pair.get("rubric", []), list) else []: + if isinstance(item, dict): + rubric.append( + { + "id": str(item.get("id", "")), + "description": str(item.get("description", "")), + "weight": float(item.get("weight", 1) or 0), + } + ) + cases.append( + { + "case_id": case_id, + "prompt": str(pair.get("prompt", "")), + "input_files": pair.get("input_files", []) if isinstance(pair.get("input_files", []), list) else [], + "review_instruction": str(pair.get("review_instruction", "")), + "rubric": rubric, + "variant_a": { + "blind_id": str(pair.get("variant_a", {}).get("blind_id", "")) if isinstance(pair.get("variant_a"), dict) else "", + "output": str(pair.get("variant_a", {}).get("output", "")) if isinstance(pair.get("variant_a"), dict) else "", + }, + "variant_b": { + "blind_id": str(pair.get("variant_b", {}).get("blind_id", "")) if isinstance(pair.get("variant_b"), dict) else "", + "output": str(pair.get("variant_b", {}).get("output", "")) if isinstance(pair.get("variant_b"), dict) else "", + }, + "decision_state": decision_state(decisions.get(case_id)), + } + ) + return cases + + +def build_summary(cases: list[dict[str, Any]], failures: list[str], reviewer: str, reviewed_at: str) -> dict[str, Any]: + status_counts: dict[str, int] = {} + for case in cases: + status = str(case.get("decision_state", {}).get("status", "awaiting-decision")) + status_counts[status] = status_counts.get(status, 0) + 1 + ready_count = status_counts.get("ready-for-adjudication", 0) + pending_count = status_counts.get("awaiting-decision", 0) + invalid_count = status_counts.get("needs-fix", 0) + return { + "case_count": len(cases), + "ready_for_adjudication_count": ready_count, + "pending_decision_count": pending_count, + "invalid_decision_count": invalid_count, + "reviewer_metadata_present": bool(str(reviewer).strip() and str(reviewed_at).strip()), + "answer_key_hidden": True, + "answer_key_path_exposed": False, + "ready_to_run_adjudication": bool(cases) and ready_count == len(cases) and invalid_count == 0, + "failure_count": len(failures), + } + + +def build_contract(blind_pack_md: Path, decisions_path: Path) -> dict[str, Any]: + return { + "reviewer_steps": [ + f"Open {display_path(blind_pack_md)} or this kit and compare Variant A vs Variant B for each case.", + f"Record choices in {display_path(decisions_path)} without opening the answer key.", + "Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.", + "Set reviewer_attestation truthfully after choices are recorded and before adjudication.", + "Run python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication after choices are recorded.", + "Refresh python3 scripts/yao.py review-studio . before asking for release approval.", + ], + "required_fields": { + "reviewer": "Human reviewer name or review group.", + "reviewed_at": "Review date or timestamp.", + "winner_variant": "A or B for every case.", + "confidence": "Optional numeric confidence from 0 to 1.", + "reason": "Required rationale based on the rubric, not on hidden labels.", + "reviewer_attestation": "Blind-review and answer-key-order attestation after decisions are complete.", + "review_integrity": "Blind pack and prompt fingerprints generated by the template.", + }, + "privacy_contract": [ + "The answer key is intentionally withheld from this kit.", + "Do not inspect hidden labels or expected winners before decisions are recorded.", + "Do not paste private user data into decision reasons.", + "Pending decisions must stay pending instead of being counted as human agreement.", + ], + } + + +def render_rubric(rubric: list[dict[str, Any]]) -> list[str]: + if not rubric: + return ["- No rubric items found."] + return [f"- `{item['id']}` ({item['weight']}): {item['description']}" for item in rubric] + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + contract = payload["review_contract"] + lines = [ + "# Output Review Kit", + "", + "This reviewer-facing packet contains the blind A/B cases, decision fields, and command flow. It intentionally does not expose the answer key.", + "", + "## Summary", + "", + f"- cases: `{summary['case_count']}`", + f"- ready for adjudication: `{summary['ready_for_adjudication_count']}`", + f"- pending decisions: `{summary['pending_decision_count']}`", + f"- invalid decisions: `{summary['invalid_decision_count']}`", + f"- reviewer metadata present: `{str(summary['reviewer_metadata_present']).lower()}`", + f"- answer key hidden: `{str(summary['answer_key_hidden']).lower()}`", + f"- answer key path exposed: `{str(summary['answer_key_path_exposed']).lower()}`", + "", + "## Review Flow", + "", + ] + lines.extend(f"{index}. {step}" for index, step in enumerate(contract["reviewer_steps"], start=1)) + lines.extend(["", "## Required Fields", ""]) + lines.extend(f"- `{key}`: {value}" for key, value in contract["required_fields"].items()) + lines.extend(["", "## Privacy Contract", ""]) + lines.extend(f"- {item}" for item in contract["privacy_contract"]) + lines.extend( + [ + "", + "## Decision States", + "", + "| Case | State | Winner | Confidence | Reason | Blocking Reason |", + "| --- | --- | --- | --- | --- | --- |", + ] + ) + for case in payload["cases"]: + state = case["decision_state"] + lines.append( + f"| `{case['case_id']}` | `{state['status']}` | `{str(state['winner_variant_recorded']).lower()}` | " + f"`{str(state['confidence_recorded']).lower()}` | `{str(state['reason_recorded']).lower()}` | {state['blocking_reason']} |" + ) + lines.extend(["", "## Blind Cases", ""]) + for case in payload["cases"]: + lines.extend( + [ + f"### {case['case_id']}", + "", + f"Prompt: {case['prompt']}", + "", + "Rubric:", + *render_rubric(case["rubric"]), + "", + "#### Variant A", + "", + case["variant_a"]["output"] or "_No output found._", + "", + "#### Variant B", + "", + case["variant_b"]["output"] or "_No output found._", + "", + ] + ) + if payload.get("failures"): + lines.extend(["## Failures", ""]) + lines.extend(f"- {failure}" for failure in payload["failures"]) + return "\n".join(lines).strip() + "\n" + + +def status_label(status: str) -> str: + return { + "awaiting-decision": "Awaiting", + "needs-fix": "Needs fix", + "ready-for-adjudication": "Ready", + }.get(status, status) + + +def render_html_rubric(rubric: list[dict[str, Any]]) -> str: + if not rubric: + return "
  • No rubric items found.
    • " + return "".join( + "
  • {id}

    {description}

    {weight}
    • ".format( + id=html_text(item.get("id", "")), + description=html_text(item.get("description", "")), + weight=html_text(item.get("weight", "")), + ) + for item in rubric + ) + + +def render_html_cases(cases: list[dict[str, Any]]) -> str: + cards = [] + for index, case in enumerate(cases, start=1): + state = case.get("decision_state", {}) + status = str(state.get("status", "awaiting-decision")) + cards.append( + f""" +
    +
    +
    + Case {index:02d} +

    {html_text(case.get('case_id', ''))}

    +
    + {html_text(status_label(status))} +
    +

    {html_text(case.get('prompt', ''))}

    +

    Rubric

      {render_html_rubric(case.get('rubric', []))}
    +
    +

    Variant A

    {html_text(case.get('variant_a', {}).get('output', ''))}
    +

    Variant B

    {html_text(case.get('variant_b', {}).get('output', ''))}
    +
    +
    + Winner recorded: {html_text(str(state.get('winner_variant_recorded', False)).lower())} + Confidence: {html_text(str(state.get('confidence_recorded', False)).lower())} + Reason: {html_text(str(state.get('reason_recorded', False)).lower())} + {html_text(state.get('blocking_reason', ''))} +
    +
    """ + ) + return "\n".join(cards) + + +def decision_template_json(cases: list[dict[str, Any]]) -> str: + template = { + "schema_version": "1.0", + "reviewer": "", + "reviewed_at": "", + "decisions": [ + {"case_id": case.get("case_id", ""), "winner_variant": "", "confidence": None, "reason": ""} + for case in cases + ], + } + return json.dumps(template, ensure_ascii=False, indent=2) + + +def render_html(payload: dict[str, Any]) -> str: + summary = payload["summary"] + contract = payload["review_contract"] + stats = [ + ("Cases", summary["case_count"]), + ("Ready", summary["ready_for_adjudication_count"]), + ("Pending", summary["pending_decision_count"]), + ("Invalid", summary["invalid_decision_count"]), + ] + stat_html = "".join(f"
    {html_text(label)}{html_text(value)}
    " for label, value in stats) + flow_html = "".join(f"
  • {html_text(step)}
    • " for step in contract["reviewer_steps"]) + privacy_html = "".join(f"
  • {html_text(item)}
    • " for item in contract["privacy_contract"]) + decision_json = html_text(decision_template_json(payload["cases"])) + return f""" + + + + + Output Review Kit + + + + +
    +
    + Blind A/B Human Review +

    Reviewer cockpit for output quality decisions

    +

    Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.

    +
    {stat_html}
    +
    +
    +

    Review Flow

      {flow_html}
    + +
    +
    {render_html_cases(payload["cases"])}
    +

    Decision Template

    Use this shape in {html_text(payload['artifacts']['decisions'])}; leave a case blank when the reviewer is not ready.

    {decision_json}
    +
    + + +""" + + +def prepare_output_review_kit( + blind_pack_json: Path, + blind_pack_md: Path, + decisions_path: Path, + output_json: Path, + output_md: Path, + output_html: Path | None = None, + write_template: bool = False, +) -> dict[str, Any]: + blind_pack, failures = load_json(blind_pack_json) + decisions_payload, decision_failures = load_optional_decisions(decisions_path) + failures.extend(decision_failures) + template_written = False + if write_template and blind_pack and not decisions_path.exists(): + decisions_path.parent.mkdir(parents=True, exist_ok=True) + decisions_path.write_text( + json.dumps(build_decision_template(blind_pack), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + template_written = True + decisions_payload, decision_failures = load_optional_decisions(decisions_path) + failures.extend(decision_failures) + decisions, index_failures = decision_index(decisions_payload) + failures.extend(index_failures) + case_ids = {str(pair.get("case_id", "")) for pair in pairs_from_pack(blind_pack)} + for case_id in decisions: + if case_id not in case_ids: + failures.append(f"decision references unknown case_id: {case_id}") + cases = build_cases(blind_pack, decisions) + summary = build_summary(cases, failures, str(decisions_payload.get("reviewer", "")), str(decisions_payload.get("reviewed_at", ""))) + payload = { + "schema_version": "1.0", + "ok": not failures, + "summary": summary, + "artifacts": { + "reviewer_kit_json": display_path(output_json), + "reviewer_kit_markdown": display_path(output_md), + "reviewer_kit_html": display_path(output_html) if output_html else "", + "blind_pack_json": display_path(blind_pack_json), + "blind_pack_markdown": display_path(blind_pack_md), + "decisions": display_path(decisions_path), + "adjudication_json": "reports/output_review_adjudication.json", + "adjudication_markdown": "reports/output_review_adjudication.md", + }, + "template_written": template_written, + "review_contract": build_contract(blind_pack_md, decisions_path), + "cases": cases, + "failures": failures, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + if output_html: + output_html.parent.mkdir(parents=True, exist_ok=True) + output_html.write_text(render_html(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prepare a reviewer-facing blind A/B output review kit.") + parser.add_argument("--blind-pack-json", default=str(DEFAULT_BLIND_PACK_JSON)) + parser.add_argument("--blind-pack-md", default=str(DEFAULT_BLIND_PACK_MD)) + parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS)) + parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON)) + parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD)) + parser.add_argument("--output-html", default=str(DEFAULT_OUTPUT_HTML)) + parser.add_argument("--write-template", action="store_true") + args = parser.parse_args() + + payload = prepare_output_review_kit( + Path(args.blind_pack_json).resolve(), + Path(args.blind_pack_md).resolve(), + Path(args.decisions).resolve(), + Path(args.output_json).resolve(), + Path(args.output_md).resolve(), + Path(args.output_html).resolve() if args.output_html else None, + write_template=args.write_template, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py new file mode 100644 index 00000000..66c3412c --- /dev/null +++ b/scripts/prepare_world_class_submission_kit.py @@ -0,0 +1,560 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import shlex +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_intake import build_intake +from world_class_evidence_contract import DISALLOWED_REAL_ARTIFACTS +from world_class_phase_queue import build_phase_queue, summarize_phase_queue +from world_class_repair_checklist import build_repair_checklist, summarize_repair_checklist +from world_class_source_checks import build_source_checklist, summarize_source_checklist +from world_class_submission_matrix import build_evidence_matrix, summarize_evidence_matrix +from world_class_submission_kit_rendering import render_html, render_readme + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Prepares editable world-class evidence intake packets without counting drafts as accepted evidence." + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def shell_path(path: Path, root: Path) -> str: + return shlex.quote(rel_path(path, root)) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def has_glob_pattern(value: str) -> bool: + return any(token in value for token in ("*", "?", "[")) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def requested_checklist_items(intake: dict[str, Any], evidence_keys: list[str]) -> list[dict[str, Any]]: + items = intake.get("operator_checklist", []) + if not evidence_keys: + return items + requested = set(evidence_keys) + return [item for item in items if item.get("evidence_key") in requested] + + +def template_result_by_key(intake: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {str(item.get("evidence_key")): item for item in intake.get("templates", [])} + + +def artifact_rows_by_key_and_path(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, Any]]: + ready: dict[tuple[str, str], dict[str, Any]] = {} + for row in rows: + key = str(row.get("evidence_key", "")) + path = str(row.get("path", "")) + if key and path and row.get("artifact_ref_ready"): + ready[(key, path)] = row + return ready + + +def template_artifact_ref_paths(skill_dir: Path, item: dict[str, Any]) -> set[str]: + template_path = skill_dir / str(item.get("template_path", "")) + errors: list[str] = [] + payload = load_template_payload(template_path, errors) + if payload is None: + return set() + refs = payload.get("artifact_refs", []) + if not isinstance(refs, list): + return set() + paths: set[str] = set() + for ref in refs: + if not isinstance(ref, dict): + continue + path = str(ref.get("path", "")).strip() + if path: + paths.add(path) + return paths + + +def load_template_payload(path: Path, errors: list[str]) -> dict[str, Any] | None: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + errors.append("template file is not valid JSON") + return None + return payload if isinstance(payload, dict) else None + + +def prefill_artifact_refs( + payload: dict[str, Any], + evidence_key: str, + ready_rows: dict[tuple[str, str], dict[str, Any]], +) -> dict[str, int]: + refs = payload.get("artifact_refs", []) + stats = { + "prefilled_artifact_ref_count": 0, + "unfilled_artifact_ref_count": 0, + } + if not isinstance(refs, list): + return stats + for ref in refs: + if not isinstance(ref, dict): + stats["unfilled_artifact_ref_count"] += 1 + continue + path = str(ref.get("path", "")).strip() + row = ready_rows.get((evidence_key, path)) + if row and row.get("sha256"): + ref["sha256"] = row["sha256"] + ref["contains_raw_content"] = False + stats["prefilled_artifact_ref_count"] += 1 + else: + stats["unfilled_artifact_ref_count"] += 1 + return stats + + +def copy_template( + skill_dir: Path, + output_dir: Path, + item: dict[str, Any], + template_results: dict[str, dict[str, Any]], + ready_rows: dict[tuple[str, str], dict[str, Any]], + overwrite: bool, + prefill_artifacts: bool, +) -> dict[str, Any]: + key = str(item.get("evidence_key", "")) + template_result = template_results.get(key, {}) + template_path = skill_dir / str(item.get("template_path", "")) + output_path = output_dir / f"{key}.json" + errors: list[str] = [] + + if template_result.get("status") != "pass": + errors.append("template failed intake validation") + if not template_path.exists(): + errors.append("template file is missing") + + if errors: + return { + "evidence_key": key, + "status": "skipped", + "template_path": rel_path(template_path, skill_dir), + "output_path": rel_path(output_path, skill_dir), + "prefilled_artifact_ref_count": 0, + "unfilled_artifact_ref_count": 0, + "errors": errors, + } + + if output_path.exists() and not overwrite: + return { + "evidence_key": key, + "status": "exists", + "template_path": rel_path(template_path, skill_dir), + "output_path": rel_path(output_path, skill_dir), + "prefilled_artifact_ref_count": 0, + "unfilled_artifact_ref_count": 0, + "errors": [], + } + + payload = load_template_payload(template_path, errors) + if errors or payload is None: + return { + "evidence_key": key, + "status": "skipped", + "template_path": rel_path(template_path, skill_dir), + "output_path": rel_path(output_path, skill_dir), + "prefilled_artifact_ref_count": 0, + "unfilled_artifact_ref_count": 0, + "errors": errors or ["template file is not a JSON object"], + } + + prefill_stats = ( + prefill_artifact_refs(payload, key, ready_rows) + if prefill_artifacts + else {"prefilled_artifact_ref_count": 0, "unfilled_artifact_ref_count": 0} + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + write_json(output_path, payload) + return { + "evidence_key": key, + "status": "written", + "template_path": rel_path(template_path, skill_dir), + "output_path": rel_path(output_path, skill_dir), + **prefill_stats, + "errors": [], + } + + +def artifact_row(skill_dir: Path, evidence_key: str, pattern: str, path: Path, status: str) -> dict[str, Any]: + exists = path.exists() + is_file = exists and path.is_file() + relative = rel_path(path, skill_dir) if exists else pattern + contains_raw_content = relative in DISALLOWED_REAL_ARTIFACTS + digest = sha256_file(path) if is_file and not contains_raw_content else "" + row_status = status if status else ("ready" if is_file else "missing") + if contains_raw_content: + row_status = "raw-content-disallowed" + return { + "evidence_key": evidence_key, + "source_pattern": pattern, + "path": relative, + "status": row_status, + "exists": exists, + "is_file": is_file, + "sha256": digest, + "artifact_ref_ready": bool(is_file and digest and not contains_raw_content), + "copy_path": relative if is_file else "", + "copy_sha256": digest, + "contains_raw_content": contains_raw_content, + "concrete_reference_required": has_glob_pattern(pattern), + } + + +def with_artifact_role(row: dict[str, Any], submission_ref_paths: set[str]) -> dict[str, Any]: + is_submission_ref = str(row.get("path", "")) in submission_ref_paths + return { + **row, + "artifact_role": "submission-ref" if is_submission_ref else "supporting-evidence", + "submission_ref_required": is_submission_ref, + } + + +def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[dict[str, Any]]: + key = str(item.get("evidence_key", "")) + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + artifacts = must_collect.get("evidence_artifacts", []) + if not isinstance(artifacts, list): + return [] + submission_ref_paths = template_artifact_ref_paths(skill_dir, item) + rows: list[dict[str, Any]] = [] + for artifact in artifacts: + pattern = str(artifact or "").strip() + if not pattern: + continue + if Path(pattern).is_absolute() or ".." in Path(pattern).parts: + rows.append( + with_artifact_role( + { + "evidence_key": key, + "source_pattern": pattern, + "path": pattern, + "status": "unsafe-path", + "exists": False, + "is_file": False, + "sha256": "", + "artifact_ref_ready": False, + "copy_path": "", + "copy_sha256": "", + "contains_raw_content": False, + "concrete_reference_required": True, + }, + submission_ref_paths, + ) + ) + continue + if has_glob_pattern(pattern): + matches = sorted(path for path in skill_dir.glob(pattern) if path.is_file()) + if not matches: + rows.append( + with_artifact_role( + { + "evidence_key": key, + "source_pattern": pattern, + "path": pattern, + "status": "glob-no-match", + "exists": False, + "is_file": False, + "sha256": "", + "artifact_ref_ready": False, + "copy_path": "", + "copy_sha256": "", + "contains_raw_content": False, + "concrete_reference_required": True, + }, + submission_ref_paths, + ) + ) + continue + for match in matches: + rows.append(with_artifact_role(artifact_row(skill_dir, key, pattern, match, "ready"), submission_ref_paths)) + continue + rows.append(with_artifact_role(artifact_row(skill_dir, key, pattern, skill_dir / pattern, ""), submission_ref_paths)) + return rows + + +def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for item in items: + rows.extend(artifact_checklist_for_item(skill_dir, item)) + return rows + + +def build_handoff_steps( + commands: dict[str, str], + *, + files: list[dict[str, Any]], + source_blocked_count: int, + invalid_draft_count: int, +) -> list[dict[str, Any]]: + drafts_ready = bool(files) and invalid_draft_count == 0 + return [ + { + "step_id": "prepare-drafts", + "label": "Prepare editable drafts", + "status": "ready" if drafts_ready else "fix-required", + "command": commands["prepare_submission"], + "completion_signal": "JSON drafts, submission_manifest.json, README.md, and index.html are present.", + "counts_as_completion": False, + "blocking_condition": "One or more drafts were skipped or failed template validation." if invalid_draft_count else "", + }, + { + "step_id": "collect-source", + "label": "Collect real external or human evidence", + "status": "blocked" if source_blocked_count else "ready", + "command": "", + "completion_signal": "Source aggregate reports satisfy the required provider, human, native, or telemetry checks.", + "counts_as_completion": False, + "blocking_condition": ( + f"{source_blocked_count} source check(s) still block ledger review." + if source_blocked_count + else "" + ), + }, + { + "step_id": "edit-submission", + "label": "Edit submission packet", + "status": "manual", + "command": "", + "completion_signal": "template_only is false only after real evidence exists; ledger_reviewer_approved, ledger_reviewer, and ledger_reviewed_at stay unset until reviewer approval.", + "counts_as_completion": False, + "blocking_condition": "Raw prompts, outputs, transcripts, credentials, or private content must not be included.", + }, + { + "step_id": "validate-intake", + "label": "Validate intake contract", + "status": "pending", + "command": commands["validate_intake"], + "completion_signal": "world_class_evidence_intake reports valid submissions and no invalid packets before reviewer approval.", + "counts_as_completion": False, + "blocking_condition": "A valid packet is ready for ledger review but is not accepted evidence by itself.", + }, + { + "step_id": "review-submission", + "label": "Review submission queue", + "status": "pending", + "command": commands["review_submission"], + "completion_signal": "world_class_submission_review shows ready-for-ledger-review before reviewer identity, timestamp, and approval are set.", + "counts_as_completion": False, + "blocking_condition": "Reviewer queue output is advisory and cannot accept evidence.", + }, + { + "step_id": "refresh-ledger", + "label": "Refresh evidence ledger", + "status": "pending", + "command": commands["refresh_ledger"], + "completion_signal": "world_class_evidence_ledger accepts the evidence entry with valid source checks.", + "counts_as_completion": False, + "blocking_condition": "Ledger remains the only world-class readiness source of truth.", + }, + { + "step_id": "guard-claim", + "label": "Guard public claim", + "status": "pending", + "command": commands["guard_claim"], + "completion_signal": "world_class_claim_guard allows the public readiness claim.", + "counts_as_completion": False, + "blocking_condition": "Public world-class claims stay blocked until every ledger entry is accepted.", + }, + ] + + +def build_submission_kit( + skill_dir: Path, + output_dir: Path, + generated_at: str, + evidence_keys: list[str] | None = None, + overwrite: bool = False, + prefill_artifacts: bool = False, + output_html: Path | None = None, +) -> dict[str, Any]: + intake = build_intake(skill_dir, generated_at, submissions_dir=output_dir) + items = requested_checklist_items(intake, evidence_keys or []) + valid_keys = {str(item.get("evidence_key")) for item in intake.get("operator_checklist", [])} + unknown_keys = sorted(set(evidence_keys or []) - valid_keys) + template_results = template_result_by_key(intake) + artifact_checklist = build_artifact_checklist(skill_dir, items) + ready_rows = artifact_rows_by_key_and_path(artifact_checklist) + files = [ + copy_template( + skill_dir, + output_dir, + item, + template_results, + ready_rows, + overwrite, + prefill_artifacts, + ) + for item in items + ] + source_checklist = build_source_checklist(items) + evidence_matrix = build_evidence_matrix(items, files, artifact_checklist, source_checklist) + repair_checklist = build_repair_checklist(items, files, artifact_checklist, source_checklist, unknown_keys) + phase_queue = build_phase_queue(repair_checklist) + manifest_path = output_dir / "submission_manifest.json" + readme_path = output_dir / "README.md" + output_html = output_html or (output_dir / "index.html") + written_count = sum(1 for item in files if item["status"] == "written") + existing_count = sum(1 for item in files if item["status"] == "exists") + skipped_count = sum(1 for item in files if item["status"] == "skipped") + prefilled_artifact_ref_count = sum(item.get("prefilled_artifact_ref_count", 0) for item in files) + unfilled_artifact_ref_count = sum(item.get("unfilled_artifact_ref_count", 0) for item in files) + artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready")) + artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready")) + artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required")) + submission_ref_rows = [item for item in artifact_checklist if item.get("submission_ref_required")] + supporting_rows = [item for item in artifact_checklist if not item.get("submission_ref_required")] + submission_ref_ready_count = sum(1 for item in submission_ref_rows if item.get("artifact_ref_ready")) + supporting_artifact_ready_count = sum(1 for item in supporting_rows if item.get("artifact_ref_ready")) + source_summary = summarize_source_checklist(source_checklist) + matrix_summary = summarize_evidence_matrix(evidence_matrix) + repair_summary = summarize_repair_checklist(repair_checklist) + phase_queue_summary = summarize_phase_queue(phase_queue) + ok = not unknown_keys and skipped_count == 0 + output_dir_arg = shell_path(output_dir, skill_dir) + requested_key_args = " ".join(f"--evidence-key {shlex.quote(key)}" for key in (evidence_keys or [])) + prepare_command = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir_arg}" + if requested_key_args: + prepare_command = f"{prepare_command} {requested_key_args}" + if prefill_artifacts: + prepare_command = f"{prepare_command} --prefill-artifacts" + commands = { + "prepare_submission": prepare_command, + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir_arg}", + "review_submission": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir_arg}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir_arg}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + } + handoff_steps = build_handoff_steps( + commands, + files=files, + source_blocked_count=source_summary["source_blocked_count"], + invalid_draft_count=skipped_count + len(unknown_keys), + ) + report = { + "schema_version": "1.0", + "ok": ok, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "output_dir": rel_path(output_dir, skill_dir), + "summary": { + "requested_count": len(items) + len(unknown_keys), + "prepared_count": len(files), + "written_count": written_count, + "existing_count": existing_count, + "skipped_count": skipped_count, + "unknown_key_count": len(unknown_keys), + "artifact_checklist_count": len(artifact_checklist), + "artifact_ready_count": artifact_ready_count, + "artifact_missing_count": artifact_missing_count, + "artifact_glob_expansion_count": artifact_glob_count, + "submission_ref_count": len(submission_ref_rows), + "submission_ref_ready_count": submission_ref_ready_count, + "submission_ref_missing_count": len(submission_ref_rows) - submission_ref_ready_count, + "supporting_artifact_count": len(supporting_rows), + "supporting_artifact_ready_count": supporting_artifact_ready_count, + "supporting_artifact_missing_count": len(supporting_rows) - supporting_artifact_ready_count, + "artifact_prefill_enabled": prefill_artifacts, + "artifact_ref_prefill_count": prefilled_artifact_ref_count, + "artifact_ref_unfilled_count": unfilled_artifact_ref_count, + **source_summary, + **matrix_summary, + **repair_summary, + **phase_queue_summary, + "handoff_step_count": len(handoff_steps), + "handoff_blocked_count": sum(1 for item in handoff_steps if item["status"] == "blocked"), + "handoff_fix_required_count": sum(1 for item in handoff_steps if item["status"] == "fix-required"), + "handoff_counts_as_completion": False, + "drafts_count_as_evidence": False, + "ledger_counts_submission_as_completion": False, + "decision": "submission-kit-ready" if ok else "fix-submission-kit", + }, + "unknown_evidence_keys": unknown_keys, + "files": files, + "artifact_checklist": artifact_checklist, + "source_checklist": source_checklist, + "evidence_matrix": evidence_matrix, + "repair_checklist": repair_checklist, + "phase_queue": phase_queue, + "operator_handoff": handoff_steps, + "evidence_items": items, + "commands": commands, + "safety": { + "template_only_drafts": True, + "artifact_prefill_counts_as_evidence": False, + "operator_handoff_counts_as_evidence": False, + "real_evidence_required_before_template_only_false": True, + "raw_content_allowed": False, + "credentials_allowed": False, + "drafts_count_as_evidence": False, + }, + "artifacts": { + "manifest": rel_path(manifest_path, skill_dir), + "readme": rel_path(readme_path, skill_dir), + "html": rel_path(output_html, skill_dir), + }, + } + output_dir.mkdir(parents=True, exist_ok=True) + write_json(manifest_path, report) + readme_path.write_text(render_readme(report), encoding="utf-8") + output_html.parent.mkdir(parents=True, exist_ok=True) + output_html.write_text(render_html(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Prepare editable world-class evidence submission drafts.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-dir", default="evidence/world_class/submission-kit") + parser.add_argument("--evidence-key", action="append", default=[]) + parser.add_argument("--overwrite", action="store_true") + parser.add_argument( + "--prefill-artifacts", + action="store_true", + help="Insert SHA-256 digests for currently available aggregate artifacts while keeping drafts template-only.", + ) + parser.add_argument("--generated-at", default=date.today().isoformat()) + parser.add_argument("--output-html") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + output_dir = Path(args.output_dir) + if not output_dir.is_absolute(): + output_dir = skill_dir / output_dir + report = build_submission_kit( + skill_dir, + output_dir.resolve(), + args.generated_at, + evidence_keys=args.evidence_key, + overwrite=args.overwrite, + prefill_artifacts=args.prefill_artifacts, + output_html=Path(args.output_html).resolve() if args.output_html else None, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/probe_runtime_permissions.py b/scripts/probe_runtime_permissions.py index 6ea25045..6a1ddbd9 100644 --- a/scripts/probe_runtime_permissions.py +++ b/scripts/probe_runtime_permissions.py @@ -10,7 +10,7 @@ except ImportError: # pragma: no cover yaml = None -DEFAULT_TARGETS = ["openai", "claude", "generic"] +DEFAULT_TARGETS = ["openai", "claude", "generic", "vscode"] def display_path(path: Path, root: Path) -> str: @@ -69,6 +69,103 @@ def sorted_strings(value: Any) -> list[str]: return sorted(str(item) for item in value) if isinstance(value, list) else [] +def check_index(checks: list[Any]) -> dict[str, str]: + indexed: dict[str, str] = {} + for item in checks: + if not isinstance(item, dict): + continue + check_id = str(item.get("id") or item.get("key") or "").strip() + status = str(item.get("status") or ("pass" if item.get("passed") is True else "")).strip() + if check_id: + indexed[check_id] = status + return indexed + + +def installer_enforcement_source( + skill_dir: Path, + package_dir: Path, + targets: list[str], + expected: list[str], + install_simulation_path: Path | None, +) -> dict[str, Any]: + path = install_simulation_path or skill_dir / "reports" / "install_simulation.json" + report = load_json(path) + source_display = display_path(path, skill_dir) + if not path.exists() or not report: + return { + "source": source_display, + "source_status": "missing", + "package_dir_matches": False, + "summary": {}, + "targets": { + target: { + "target": target, + "source_status": "missing", + "enforced": False, + "enforced_capabilities": [], + "missing_capabilities": expected, + "failure_details": ["install simulation report is missing"], + } + for target in targets + }, + } + + expected_package_dir = display_path(package_dir, skill_dir) + observed_package_dir = str(report.get("package_dir", "")).strip() + package_matches = observed_package_dir in {expected_package_dir, str(package_dir)} + summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {} + if not package_matches: + source_status = "package-mismatch" + else: + source_status = "present" + + indexed = check_index(report.get("checks", []) if isinstance(report.get("checks"), list) else []) + target_results: dict[str, Any] = {} + for target in targets: + enforced_capabilities = [] + missing_capabilities = [] + failure_details = [] + for capability in expected: + approval_id = f"permission-{target}-{capability}-approved" + enforcement_id = f"permission-{target}-{capability}-target-enforcement" + approval_ok = indexed.get(approval_id) == "pass" + enforcement_ok = indexed.get(enforcement_id) == "pass" + if source_status == "present" and approval_ok and enforcement_ok: + enforced_capabilities.append(capability) + else: + missing_capabilities.append(capability) + if source_status == "present" and not approval_ok: + failure_details.append(f"{approval_id} did not pass") + if source_status == "present" and not enforcement_ok: + failure_details.append(f"{enforcement_id} did not pass") + if source_status == "package-mismatch": + failure_details.append( + f"install simulation package_dir {observed_package_dir or 'missing'} does not match probed package_dir {expected_package_dir}" + ) + target_results[target] = { + "target": target, + "source_status": source_status, + "enforced": bool(expected) and source_status == "present" and not missing_capabilities, + "enforced_capabilities": enforced_capabilities, + "missing_capabilities": missing_capabilities, + "failure_details": failure_details, + } + + return { + "source": source_display, + "source_status": source_status, + "package_dir_matches": package_matches, + "summary": { + "installer_permission_enforced_count": int(summary.get("installer_permission_enforced_count", 0) or 0), + "installer_permission_failure_count": int(summary.get("installer_permission_failure_count", 0) or 0), + "permission_target_count": int(summary.get("permission_target_count", 0) or 0), + "permission_capability_count": int(summary.get("permission_capability_count", 0) or 0), + "failure_count": int(summary.get("failure_count", 0) or 0), + }, + "targets": target_results, + } + + def probe_openai_yaml(package_dir: Path, expected: list[str]) -> tuple[list[dict[str, Any]], list[str]]: checks: list[dict[str, Any]] = [] failures: list[str] = [] @@ -93,7 +190,13 @@ def probe_openai_yaml(package_dir: Path, expected: list[str]) -> tuple[list[dict return checks, failures -def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list[str]) -> dict[str, Any]: +def probe_target( + skill_dir: Path, + package_dir: Path, + target: str, + expected: list[str], + installer_targets: dict[str, Any], +) -> dict[str, Any]: adapter_path = package_dir / "targets" / target / "adapter.json" checks: list[dict[str, Any]] = [] failures: list[str] = [] @@ -167,6 +270,17 @@ def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list residual_risks = [] if native is False: residual_risks.append("Client-native permission enforcement is not provided by this target; installer or operator must honor metadata.") + installer_enforcement = installer_targets.get( + target, + { + "target": target, + "source_status": "missing", + "enforced": False, + "enforced_capabilities": [], + "missing_capabilities": expected, + "failure_details": ["install simulation report is missing"], + }, + ) return { "target": target, "status": "pass" if not failures else "fail", @@ -174,6 +288,7 @@ def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list "permission_model": str(target_contract.get("permission_model", "")), "native_enforcement": bool(native) if isinstance(native, bool) else None, "metadata_fallback_explicit": metadata_fallback, + "installer_enforcement": installer_enforcement, "assurance": assurance, "declared_capabilities": sorted_strings(target_contract.get("declared_capabilities")), "checks": checks, @@ -184,10 +299,11 @@ def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list def render_markdown(report: dict[str, Any]) -> str: summary = report["summary"] + installer = report.get("installer_enforcement", {}) lines = [ "# Runtime Permission Probes", "", - "Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit.", + "Runtime permission probes verify that generated target adapters expose high-permission capabilities, make native-enforcement limits explicit, and link installer enforcement evidence when available.", "", "## Summary", "", @@ -197,17 +313,35 @@ def render_markdown(report: dict[str, Any]) -> str: f"- Failed: `{summary['fail_count']}`", f"- Native enforcement targets: `{summary['native_enforcement_count']}`", f"- Explicit metadata fallbacks: `{summary['metadata_fallback_count']}`", + f"- Installer enforcement source: `{summary['installer_enforcement_source_status']}`", + f"- Installer-enforced targets: `{summary['installer_enforcement_pass_count']}`", + f"- Installer permission failures: `{summary['installer_permission_failure_count']}`", + f"- World-class native evidence ready: `{summary['world_class_native_evidence_ready']}`", f"- Required capabilities: `{', '.join(report['expected_capabilities']) or 'none'}`", "", - "| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk |", - "| --- | --- | --- | --- | --- | --- |", + "| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Installer Enforcement | Residual Risk |", + "| --- | --- | --- | --- | --- | --- | --- |", ] for target in report["targets"]: residual = "
    ".join(target["residual_risks"]) if target["residual_risks"] else "None" + installer_status = target.get("installer_enforcement", {}) + installer_label = "pass" if installer_status.get("enforced") else installer_status.get("source_status", "missing") lines.append( f"| `{target['target']}` | `{target['status']}` | `{target['assurance']}` | " - f"`{target['native_enforcement']}` | `{target['metadata_fallback_explicit']}` | {residual} |" + f"`{target['native_enforcement']}` | `{target['metadata_fallback_explicit']}` | `{installer_label}` | {residual} |" ) + lines.extend( + [ + "", + "## Installer Enforcement", + "", + f"- Source: `{installer.get('source', '')}`", + f"- Source status: `{installer.get('source_status', 'missing')}`", + f"- Package dir matches probe: `{installer.get('package_dir_matches', False)}`", + "", + "Installer enforcement means the package install simulation blocks missing capability approvals or target enforcement notes. It is supporting local distribution evidence, not proof of target-client native enforcement.", + ] + ) lines.extend(["", "## Failures", ""]) lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) lines.extend( @@ -227,12 +361,21 @@ def probe_runtime_permissions( targets: list[str], output_json: Path, output_md: Path, + install_simulation_json: Path | None = None, ) -> dict[str, Any]: skill_dir = skill_dir.resolve() package_dir = package_dir.resolve() expected = expected_capabilities(skill_dir) - target_results = [probe_target(skill_dir, package_dir, target, expected) for target in targets] + installer = installer_enforcement_source(skill_dir, package_dir, targets, expected, install_simulation_json) + installer_targets = installer.get("targets", {}) if isinstance(installer.get("targets"), dict) else {} + target_results = [probe_target(skill_dir, package_dir, target, expected, installer_targets) for target in targets] failures = [failure for target in target_results for failure in target["failures"]] + installer_summary = installer.get("summary", {}) if isinstance(installer.get("summary"), dict) else {} + installer_pass_count = sum( + 1 + for item in installer_targets.values() + if isinstance(item, dict) and item.get("enforced") is True + ) summary = { "target_count": len(target_results), "pass_count": sum(1 for item in target_results if item["status"] == "pass"), @@ -242,6 +385,20 @@ def probe_runtime_permissions( "residual_risk_count": sum(len(item["residual_risks"]) for item in target_results), "required_capability_count": len(expected), "failure_count": len(failures), + "installer_enforcement_source_status": installer.get("source_status", "missing"), + "installer_enforcement_target_count": len(installer_targets), + "installer_enforcement_pass_count": installer_pass_count, + "installer_permission_enforced_count": int(installer_summary.get("installer_permission_enforced_count", 0) or 0), + "installer_permission_failure_count": int(installer_summary.get("installer_permission_failure_count", 0) or 0), + "installer_permission_capability_count": int(installer_summary.get("permission_capability_count", 0) or 0), + "world_class_native_evidence_ready": sum(1 for item in target_results if item["native_enforcement"] is True) > 0 and not failures, + "installer_enforcement_ready": ( + installer.get("source_status") == "present" + and installer_pass_count == len(target_results) + and bool(expected) + and int(installer_summary.get("failure_count", 0) or 0) == 0 + and int(installer_summary.get("installer_permission_failure_count", 0) or 0) == 0 + ), } report = { "schema_version": "1.0", @@ -250,6 +407,12 @@ def probe_runtime_permissions( "package_dir": display_path(package_dir, skill_dir), "expected_capabilities": expected, "summary": summary, + "installer_enforcement": { + "source": installer.get("source", ""), + "source_status": installer.get("source_status", "missing"), + "package_dir_matches": installer.get("package_dir_matches", False), + "summary": installer_summary, + }, "targets": target_results, "failures": failures, "artifacts": { @@ -271,6 +434,7 @@ def main() -> None: parser.add_argument("--target", action="append", choices=DEFAULT_TARGETS) parser.add_argument("--output-json") parser.add_argument("--output-md") + parser.add_argument("--install-simulation-json") args = parser.parse_args() skill_dir = Path(args.skill_dir).resolve() @@ -280,6 +444,7 @@ def main() -> None: args.target or DEFAULT_TARGETS, Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "runtime_permission_probes.json", Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "runtime_permission_probes.md", + Path(args.install_simulation_json).resolve() if args.install_simulation_json else None, ) print(json.dumps(report, ensure_ascii=False, indent=2)) raise SystemExit(0 if report["ok"] else 2) diff --git a/scripts/propose_adaptation.py b/scripts/propose_adaptation.py new file mode 100644 index 00000000..67903e93 --- /dev/null +++ b/scripts/propose_adaptation.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Turns redacted repeated preference patterns into proposal-only adaptation plans." + +PROPOSAL_LIBRARY = { + "language_default": { + "title": "Keep reports Chinese-first with optional English", + "change_type": "report-default-language", + "risk_level": "low", + "target_files": ["scripts/render_skill_overview.py", "references/artifact-design-doctrine.md"], + "suggested_changes": [ + "Keep user-facing report copy Simplified Chinese by default.", + "Expose English through the existing language switch instead of mixing languages in the default view.", + ], + "verification_commands": ["python3 tests/verify_skill_overview.py"], + "rollback_plan": "Revert report language template changes and rerun the overview verifier.", + }, + "report_ui": { + "title": "Improve report layout, visual hierarchy, and chart readability", + "change_type": "artifact-ui-polish", + "risk_level": "medium", + "target_files": ["scripts/render_skill_overview.py", "references/artifact-design-doctrine.md", "tests/verify_skill_overview.py"], + "suggested_changes": [ + "Prefer vertical narrative sections with limited two-column layouts only when content has enough width.", + "Keep charts inline SVG, with captions and stable responsive constraints.", + ], + "verification_commands": ["python3 tests/verify_skill_overview.py", "python3 tests/verify_skill_report_charts.py"], + "rollback_plan": "Restore the previous report renderer and regenerate the demo report.", + }, + "approval_safety": { + "title": "Keep adaptive iteration approval-gated", + "change_type": "privacy-governance", + "risk_level": "low", + "target_files": ["references/user-memory-policy.md", "references/autonomous-adaptation.md", "schemas/adaptation-proposal.schema.json"], + "suggested_changes": [ + "Require explicit source paths for memory scans.", + "Generate proposals before any source patching.", + "Reserve automatic apply for a future approval ledger and rollback implementation.", + ], + "verification_commands": ["python3 tests/verify_adaptation_safety.py"], + "rollback_plan": "Remove the adaptive proposal artifacts and keep feedback/adoption drift as the only iteration inputs.", + }, + "delivery_format": { + "title": "Make generated artifact paths explicit in CLI output", + "change_type": "artifact-discoverability", + "risk_level": "low", + "target_files": ["scripts/yao.py", "README.md"], + "suggested_changes": [ + "Include stable report paths in command output.", + "Document which artifacts are meant for human review.", + ], + "verification_commands": ["python3 tests/verify_yao_cli.py"], + "rollback_plan": "Revert CLI copy/documentation changes and keep artifact paths unchanged.", + }, + "evidence_testing": { + "title": "Attach tests and evidence refresh to each upgrade", + "change_type": "quality-gate", + "risk_level": "medium", + "target_files": ["tests/verify_adaptation_safety.py", "scripts/render_skill_os2_coverage.py", "reports/skill_os2_coverage.json"], + "suggested_changes": [ + "Add focused verifier coverage for every new adaptive behavior.", + "Refresh Skill OS 2.0 coverage so planned, partial, and covered states remain visible.", + ], + "verification_commands": ["python3 tests/verify_adaptation_safety.py", "python3 tests/verify_skill_os2_coverage.py"], + "rollback_plan": "Revert the new verifier and coverage status updates, then regenerate coverage reports.", + }, +} + +TOP_LEVEL_SUMMARY_FIELDS = [ + "pattern_count", + "proposal_count", + "apply_supported", + "failure_count", +] +TOP_LEVEL_PROPOSAL_CONTRACT_FIELDS = [ + "proposal_only", + "approval_required", + "writes_repository_files", + "allowlisted_targets_required", + "target_file_sha256_required_for_apply", + "approval_draft_supported", + "rollback_required_for_apply", + "apply_command_available", +] + + +def top_level_mirrors(summary: dict[str, Any], proposal_contract: dict[str, Any]) -> dict[str, Any]: + mirrored = {key: summary[key] for key in TOP_LEVEL_SUMMARY_FIELDS if key in summary} + mirrored.update({key: proposal_contract[key] for key in TOP_LEVEL_PROPOSAL_CONTRACT_FIELDS if key in proposal_contract}) + return mirrored + + +def report_contract() -> dict[str, Any]: + return { + "schema_version": "1.0", + "contract": "adaptation-proposals", + "top_level_mirrors_summary": True, + "top_level_mirrors_proposal_contract": True, + "summary_fields": TOP_LEVEL_SUMMARY_FIELDS, + "proposal_contract_fields": TOP_LEVEL_PROPOSAL_CONTRACT_FIELDS, + "source_of_truth": ["summary", "proposal_contract"], + } + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def display_path(path: Path, skill_dir: Path) -> str: + try: + return str(path.resolve().relative_to(skill_dir.resolve())) + except ValueError: + return str(path.resolve()) + + +def resolve_output(skill_dir: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else skill_dir / path + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def proposal_id(pattern_id: str, support_count: int, target_files: list[str]) -> str: + digest = hashlib.sha1(f"{pattern_id}:{support_count}:{','.join(target_files)}".encode("utf-8")).hexdigest() + return f"adapt-{digest[:10]}" + + +def proposal_from_pattern(pattern: dict[str, Any]) -> dict[str, Any]: + pattern_id = str(pattern.get("pattern_id") or "generic") + spec = PROPOSAL_LIBRARY.get( + pattern_id, + { + "title": "Review repeated preference pattern", + "change_type": "manual-review", + "risk_level": "medium", + "target_files": ["README.md"], + "suggested_changes": ["Review this repeated signal manually before changing skill behavior."], + "verification_commands": ["make ci-test"], + "rollback_plan": "Revert the approved patch and rerun the relevant verifier.", + }, + ) + support_count = int(pattern.get("support_count") or 0) + target_files = list(spec["target_files"]) + return { + "proposal_id": proposal_id(pattern_id, support_count, target_files), + "pattern_id": pattern_id, + "title": spec["title"], + "change_type": spec["change_type"], + "status": "proposal-only", + "requires_approval": True, + "write_allowed_without_approval": False, + "risk_level": spec["risk_level"], + "reason": pattern.get("reason", "Repeated signal detected."), + "support_count": support_count, + "target_files": target_files, + "suggested_changes": spec["suggested_changes"], + "verification_commands": spec["verification_commands"], + "rollback_plan": spec["rollback_plan"], + "evidence_refs": [ + { + "record_id": item.get("record_id", "unknown"), + "excerpt": item.get("excerpt", ""), + } + for item in pattern.get("evidence", [])[:3] + if isinstance(item, dict) + ], + } + + +def build_report(skill_dir: Path, patterns_json: Path, generated_at: str) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + if not patterns_json.is_absolute(): + patterns_json = skill_dir / patterns_json + patterns_payload = load_json(patterns_json) + failures: list[str] = [] + if not patterns_payload: + failures.append(f"Pattern report does not exist or is invalid: {display_path(patterns_json, skill_dir)}") + elif patterns_payload.get("ok") is not True: + failures.append("Pattern report is not ok; fix scan failures before proposal generation.") + patterns = patterns_payload.get("patterns", []) if isinstance(patterns_payload.get("patterns"), list) else [] + proposals = [proposal_from_pattern(pattern) for pattern in patterns if isinstance(pattern, dict)] if not failures else [] + summary = { + "pattern_count": len(patterns), + "proposal_count": len(proposals), + "apply_supported": (ROOT / "scripts" / "apply_adaptation.py").exists(), + "failure_count": len(failures), + } + proposal_contract = { + "proposal_only": True, + "approval_required": True, + "writes_repository_files": False, + "allowlisted_targets_required": True, + "target_file_sha256_required_for_apply": True, + "approval_draft_supported": True, + "rollback_required_for_apply": True, + "apply_command_available": (ROOT / "scripts" / "apply_adaptation.py").exists(), + } + return { + "schema_version": "1.0", + "ok": not failures, + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + "source_patterns": display_path(patterns_json, skill_dir), + **top_level_mirrors(summary, proposal_contract), + "summary": summary, + "proposal_contract": proposal_contract, + "report_contract": report_contract(), + "proposals": proposals, + "failures": failures, + "artifacts": { + "json": "reports/adaptation_proposals.json", + "markdown": "reports/adaptation_proposals.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + lines = [ + "# Adaptation Proposals", + "", + f"- Generated at: `{report['generated_at']}`", + f"- Pattern report: `{report['source_patterns']}`", + f"- Proposal only: `{str(report['proposal_contract']['proposal_only']).lower()}`", + f"- Writes repository files: `{str(report['proposal_contract']['writes_repository_files']).lower()}`", + f"- Proposals: `{report['summary']['proposal_count']}`", + "", + ] + if not report["proposals"]: + lines.append("No proposals were generated.") + for proposal in report["proposals"]: + lines.extend( + [ + f"## {proposal['title']}", + "", + f"- ID: `{proposal['proposal_id']}`", + f"- Status: `{proposal['status']}`", + f"- Pattern: `{proposal['pattern_id']}`", + f"- Risk: `{proposal['risk_level']}`", + f"- Requires approval: `{str(proposal['requires_approval']).lower()}`", + f"- Reason: {proposal['reason']}", + "- Target files:", + ] + ) + lines.extend(f" - `{path}`" for path in proposal["target_files"]) + lines.append("- Suggested changes:") + lines.extend(f" - {item}" for item in proposal["suggested_changes"]) + lines.append("- Verification:") + lines.extend(f" - `{item}`" for item in proposal["verification_commands"]) + lines.append(f"- Rollback: {proposal['rollback_plan']}") + if proposal["evidence_refs"]: + lines.append("- Redacted evidence refs:") + lines.extend(f" - `{item['record_id']}`: {item['excerpt']}" for item in proposal["evidence_refs"]) + lines.append("") + if report["failures"]: + lines.extend(["## Failures", ""]) + lines.extend(f"- {failure}" for failure in report["failures"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Create proposal-only adaptation plans from summarized user signal patterns.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--patterns-json", default="reports/user_patterns.json") + parser.add_argument("--output-json", default="reports/adaptation_proposals.json") + parser.add_argument("--output-md", default="reports/adaptation_proposals.md") + parser.add_argument("--generated-at", default=utc_now()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, Path(args.patterns_json), args.generated_at) + if report["ok"]: + output_json = resolve_output(skill_dir, args.output_json) + output_md = resolve_output(skill_dir, args.output_md) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + report["artifacts"] = { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + } + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/provider_output_eval_runner.py b/scripts/provider_output_eval_runner.py new file mode 100644 index 00000000..4423a7a8 --- /dev/null +++ b/scripts/provider_output_eval_runner.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1/responses" +DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com/chat/completions" +DEFAULT_BASE_URL = DEFAULT_OPENAI_BASE_URL +DEFAULT_HOST = urlparse(DEFAULT_OPENAI_BASE_URL).hostname or "api.openai.com" +DEFAULT_PROVIDER_CONFIGS = { + "openai": { + "base_url": DEFAULT_OPENAI_BASE_URL, + "api_format": "responses", + "api_key_env": "OPENAI_API_KEY", + "thinking": "", + }, + "deepseek": { + "base_url": DEFAULT_DEEPSEEK_BASE_URL, + "api_format": "chat-completions", + "api_key_env": "DEEPSEEK_API_KEY", + "thinking": "disabled", + }, +} +ALLOWED_PATH_PREFIX = "/v1/responses" +CHAT_COMPLETIONS_PATHS = {"/chat/completions", "/v1/chat/completions"} +LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"} + + +def fail(message: str) -> None: + print(message, file=sys.stderr) + raise SystemExit(2) + + +def validate_base_url( + base_url: str, + allow_insecure_localhost: bool, + allow_custom_base_url: bool, + api_format: str, +) -> None: + parsed = urlparse(base_url) + host = parsed.hostname or "" + if api_format == "responses" and not parsed.path.startswith(ALLOWED_PATH_PREFIX): + fail(f"provider endpoint path must start with {ALLOWED_PATH_PREFIX}") + if api_format == "chat-completions" and parsed.path not in CHAT_COMPLETIONS_PATHS: + fail("chat-completions provider endpoint path must be /chat/completions or /v1/chat/completions") + if parsed.scheme == "https" and ( + base_url in {DEFAULT_OPENAI_BASE_URL, DEFAULT_DEEPSEEK_BASE_URL} + or host == DEFAULT_HOST + or allow_custom_base_url + ): + return + if parsed.scheme == "https": + fail("custom provider host requires --allow-custom-base-url") + if parsed.scheme == "http" and allow_insecure_localhost and (parsed.hostname or "") in LOCAL_HOSTS: + return + fail("provider runner requires HTTPS; use --allow-insecure-localhost only for local test servers") + + +def provider_defaults(provider: str) -> dict[str, str]: + return DEFAULT_PROVIDER_CONFIGS.get(provider, DEFAULT_PROVIDER_CONFIGS["openai"]) + + +def load_request() -> dict[str, Any]: + raw = sys.stdin.read() + if not raw.strip(): + fail("provider runner requires a JSON request on stdin") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + fail(f"invalid JSON request: {exc}") + if not isinstance(payload, dict): + fail("runner request must be a JSON object") + return payload + + +def safe_relative(path_value: str) -> Path | None: + path = Path(path_value) + if path.is_absolute() or ".." in path.parts: + return None + return path + + +def read_input_files(paths: Any, input_root: Path, max_chars: int) -> list[dict[str, str]]: + if not isinstance(paths, list): + return [] + files: list[dict[str, str]] = [] + for item in paths: + rel = safe_relative(str(item)) + if rel is None: + files.append({"path": str(item), "status": "skipped-unsafe-path", "content": ""}) + continue + path = input_root / rel + if not path.exists() or not path.is_file(): + files.append({"path": str(item), "status": "missing", "content": ""}) + continue + text = path.read_text(encoding="utf-8", errors="replace") + files.append({"path": str(item), "status": "loaded", "content": text[:max_chars]}) + return files + + +def read_skill_instructions(path: Path, max_chars: int) -> str: + if not path.exists() or not path.is_file(): + return "" + return path.read_text(encoding="utf-8", errors="replace")[:max_chars] + + +def build_provider_input(request: dict[str, Any], skill_text: str, input_files: list[dict[str, str]]) -> str: + variant = str(request.get("variant", "")) + lines = [ + "You are producing one output for a Yao Meta Skill output-eval case.", + f"Case id: {request.get('case_id', '')}", + f"Variant: {variant}", + "", + "User task:", + str(request.get("prompt", "")), + "", + ] + if variant == "with_skill": + lines.extend( + [ + "Use the skill instructions below as the operating guidance. Preserve concrete evidence paths and boundaries.", + "", + "Skill instructions:", + skill_text or "(Skill instructions were not available.)", + "", + ] + ) + else: + lines.extend( + [ + "Produce a direct baseline answer without using the Yao Meta Skill guidance.", + "Do not invent files, reports, governance evidence, or hidden review artifacts.", + "", + ] + ) + if input_files: + lines.append("Input files:") + for item in input_files: + lines.append(f"--- {item['path']} ({item['status']}) ---") + if item["content"]: + lines.append(item["content"]) + lines.append("") + lines.extend( + [ + "Return only the final user-facing answer for this variant.", + "Do not mention or copy any fixture output from the eval case.", + ] + ) + return "\n".join(lines) + + +def response_text(payload: dict[str, Any]) -> str: + if isinstance(payload.get("output_text"), str): + return str(payload["output_text"]) + parts: list[str] = [] + output = payload.get("output") + if isinstance(output, list): + for item in output: + if not isinstance(item, dict): + continue + content = item.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if isinstance(block.get("text"), str): + parts.append(str(block["text"])) + if parts: + return "\n".join(part for part in parts if part).strip() + choices = payload.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + message = first.get("message", {}) + if isinstance(message, dict) and isinstance(message.get("content"), str): + return str(message["content"]) + return "" + + +def observed_usage(payload: dict[str, Any]) -> dict[str, Any]: + usage = payload.get("usage", {}) + if not isinstance(usage, dict): + return {} + input_tokens = usage.get("input_tokens", usage.get("prompt_tokens")) + output_tokens = usage.get("output_tokens", usage.get("completion_tokens")) + total_tokens = usage.get("total_tokens") + result: dict[str, Any] = {} + if input_tokens is not None: + result["input_tokens"] = int(input_tokens) + if output_tokens is not None: + result["output_tokens"] = int(output_tokens) + if total_tokens is not None: + result["total_tokens"] = int(total_tokens) + if result: + result["estimated"] = False + return result + + +def request_body( + model: str, + provider_input: str, + api_format: str, + thinking_type: str, + temperature: float | None, +) -> dict[str, Any]: + if api_format == "chat-completions": + body: dict[str, Any] = { + "model": model, + "messages": [{"role": "user", "content": provider_input}], + } + if thinking_type: + body["thinking"] = {"type": thinking_type} + if temperature is not None: + body["temperature"] = temperature + return body + body = {"model": model, "input": provider_input} + if temperature is not None: + body["temperature"] = temperature + return body + + +def call_provider( + base_url: str, + api_key: str, + model: str, + provider_input: str, + timeout_seconds: float, + api_format: str, + thinking_type: str, + temperature: float | None, +) -> dict[str, Any]: + body = json.dumps( + request_body(model, provider_input, api_format, thinking_type, temperature), + ensure_ascii=False, + ).encode("utf-8") + request = Request( + base_url, + data=body, + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urlopen(request, timeout=timeout_seconds) as response: + response_body = response.read().decode("utf-8", errors="replace") + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:500] + fail(f"provider request failed with HTTP {exc.code}: {detail}") + except URLError as exc: + fail(f"provider request failed: {exc.reason}") + try: + payload = json.loads(response_body) + except json.JSONDecodeError as exc: + fail(f"provider returned invalid JSON: {exc}") + if not isinstance(payload, dict): + fail("provider response must be a JSON object") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Provider-backed output-eval runner for run_output_execution.py. " + "Requires real model credentials and reports execution_kind=model only after an HTTP provider call." + ) + ) + parser.add_argument("--provider", default="openai", help="Provider label to write into execution evidence.") + parser.add_argument( + "--base-url", + help="Override the provider endpoint. Defaults are selected from --provider.", + ) + parser.add_argument( + "--api-format", + choices=["responses", "chat-completions"], + help="Provider API shape. Defaults are selected from --provider.", + ) + parser.add_argument("--thinking", choices=["enabled", "disabled"], help="Optional chat-completions thinking mode.") + parser.add_argument("--temperature", type=float, default=0.0, help="Sampling temperature for provider requests.") + parser.add_argument("--model", default=os.environ.get("YAO_OUTPUT_EVAL_MODEL", "")) + parser.add_argument("--api-key-env", help="Environment variable that contains the provider API key.") + parser.add_argument("--input-root", default=str(ROOT / "evals" / "output")) + parser.add_argument("--skill-file", default=str(ROOT / "SKILL.md")) + parser.add_argument("--timeout-seconds", type=float, default=60.0) + parser.add_argument("--max-input-file-chars", type=int, default=6000) + parser.add_argument("--max-skill-chars", type=int, default=8000) + parser.add_argument("--allow-insecure-localhost", action="store_true") + parser.add_argument("--allow-custom-base-url", action="store_true") + args = parser.parse_args() + + defaults = provider_defaults(args.provider) + base_url = args.base_url or defaults["base_url"] + api_format = args.api_format or defaults["api_format"] + thinking = args.thinking if args.thinking is not None else defaults["thinking"] + api_key_env = args.api_key_env or defaults["api_key_env"] + + validate_base_url(base_url, args.allow_insecure_localhost, args.allow_custom_base_url, api_format) + if args.temperature is not None and not 0 <= args.temperature <= 2: + fail("--temperature must be between 0 and 2") + if not args.model: + fail("missing model; pass --model or set YAO_OUTPUT_EVAL_MODEL") + api_key = os.environ.get(api_key_env, "") + if not api_key: + fail(f"missing API key env: {api_key_env}") + + request = load_request() + input_files = read_input_files(request.get("input_files", []), Path(args.input_root).resolve(), args.max_input_file_chars) + skill_text = read_skill_instructions(Path(args.skill_file).resolve(), args.max_skill_chars) + provider_input = build_provider_input(request, skill_text, input_files) + response = call_provider( + base_url, + api_key, + args.model, + provider_input, + args.timeout_seconds, + api_format, + thinking, + args.temperature, + ) + output = response_text(response) + if not output: + fail("provider response did not contain output text") + result = { + "output": output, + "execution_kind": "model", + "provider": args.provider, + "model": args.model, + "usage": observed_usage(response), + "response_id": str(response.get("id", "")), + } + print(json.dumps(result, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/python_compat_check.py b/scripts/python_compat_check.py new file mode 100644 index 00000000..b6afaa76 --- /dev/null +++ b/scripts/python_compat_check.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +import argparse +import io +import json +import token +import tokenize +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Checks repository Python source for syntax that can pass locally but fail on the supported CI interpreter." + +EXCLUDED_DIRS = { + ".git", + ".mypy_cache", + ".previews", + ".pytest_cache", + ".ruff_cache", + ".venv", + "__pycache__", + "dist", + "node_modules", + "venv", +} +MAX_FILE_BYTES = 1_000_000 + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def resolve_path(raw_path: str, root: Path) -> Path: + path = Path(raw_path) + if not path.is_absolute(): + path = root / path + return path.resolve() + + +def is_ignored(path: Path, root: Path) -> bool: + try: + parts = path.resolve().relative_to(root.resolve()).parts + except ValueError: + return True + if any(part in EXCLUDED_DIRS for part in parts): + return True + if len(parts) >= 2 and parts[0] == "tests" and parts[1].startswith("tmp"): + return True + return False + + +def expand_scan_paths(root: Path, raw_paths: list[str]) -> list[Path]: + candidates: list[Path] = [] + explicit_paths = bool(raw_paths) + if raw_paths: + for raw_path in raw_paths: + path = resolve_path(raw_path, root) + if path.is_dir(): + candidates.extend(path.rglob("*.py")) + elif path.exists(): + candidates.append(path) + else: + candidates.extend(root.rglob("*.py")) + files = [] + for path in candidates: + if not path.is_file() or path.is_symlink() or path.suffix != ".py": + continue + if not explicit_paths and is_ignored(path, root): + continue + try: + if path.stat().st_size > MAX_FILE_BYTES: + continue + except OSError: + continue + files.append(path) + return sorted(set(files), key=lambda item: rel_path(item, root)) + + +def issue(path: Path, root: Path, line: int, column: int, rule: str, message: str, excerpt: str = "") -> dict[str, Any]: + return { + "path": rel_path(path, root), + "line": line, + "column": column, + "rule": rule, + "message": message, + "excerpt": excerpt.strip()[:220], + } + + +def compile_issues(path: Path, root: Path, source: str) -> list[dict[str, Any]]: + try: + compile(source, str(path), "exec") + except SyntaxError as exc: + return [ + issue( + path, + root, + exc.lineno or 0, + exc.offset or 0, + "current-python-syntax", + str(exc.msg), + exc.text or "", + ) + ] + return [] + + +def token_name(token_type: int) -> str: + return token.tok_name.get(token_type, "") + + +def scan_modern_fstring_tokens(path: Path, root: Path, source: str) -> tuple[bool, list[dict[str, Any]]]: + findings: list[dict[str, Any]] = [] + try: + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + except tokenize.TokenError: + return False, findings + has_modern_fstring_tokens = any(token_name(item.type) == "FSTRING_START" for item in tokens) + if not has_modern_fstring_tokens: + return False, findings + fstring_depth = 0 + expression_depth = 0 + for item in tokens: + name = token_name(item.type) + if name == "FSTRING_START": + fstring_depth += 1 + expression_depth = 0 + continue + if name == "FSTRING_END": + fstring_depth = max(0, fstring_depth - 1) + expression_depth = 0 + continue + if not fstring_depth: + continue + if item.type == token.OP and item.string == "{": + expression_depth += 1 + continue + if item.type == token.OP and item.string == "}": + expression_depth = max(0, expression_depth - 1) + continue + if expression_depth > 0 and "\\" in item.string: + findings.append( + issue( + path, + root, + item.start[0], + item.start[1] + 1, + "fstring-expression-backslash", + "Python 3.11 rejects backslashes inside f-string expressions.", + item.line, + ) + ) + return True, findings + + +def split_string_token(token_text: str) -> tuple[str, str] | None: + index = 0 + while index < len(token_text) and token_text[index].isalpha(): + index += 1 + prefix = token_text[:index].lower() + if "f" not in prefix: + return None + quote = "" + for candidate in ('"""', "'''", '"', "'"): + if token_text[index:].startswith(candidate): + quote = candidate + break + if not quote or not token_text.endswith(quote): + return None + return prefix, token_text[index + len(quote) : -len(quote)] + + +def scan_legacy_fstring_body(path: Path, root: Path, token_text: str, start_line: int, start_column: int) -> list[dict[str, Any]]: + parsed = split_string_token(token_text) + if not parsed: + return [] + _, body = parsed + findings: list[dict[str, Any]] = [] + expression_depth = 0 + line = start_line + column = start_column + index = 0 + while index < len(body): + char = body[index] + next_char = body[index + 1] if index + 1 < len(body) else "" + if expression_depth == 0: + if char == "{" and next_char == "{": + index += 2 + column += 2 + continue + if char == "{" and next_char != "{": + expression_depth = 1 + else: + if char == "\\": + findings.append( + issue( + path, + root, + line, + column + 1, + "fstring-expression-backslash", + "Python 3.11 rejects backslashes inside f-string expressions.", + token_text, + ) + ) + elif char == "{": + expression_depth += 1 + elif char == "}": + expression_depth = max(0, expression_depth - 1) + if char == "\n": + line += 1 + column = 0 + else: + column += 1 + index += 1 + return findings + + +def fstring_compat_issues(path: Path, root: Path, source: str) -> list[dict[str, Any]]: + handled, findings = scan_modern_fstring_tokens(path, root, source) + if handled: + return findings + try: + tokens = tokenize.generate_tokens(io.StringIO(source).readline) + for item in tokens: + if item.type == token.STRING: + findings.extend(scan_legacy_fstring_body(path, root, item.string, item.start[0], item.start[1])) + except tokenize.TokenError: + return findings + return findings + + +def check_file(path: Path, root: Path) -> dict[str, Any]: + try: + source = path.read_text(encoding="utf-8", errors="replace") + except OSError as exc: + issues = [issue(path, root, 0, 0, "read-error", str(exc))] + else: + issues = compile_issues(path, root, source) + issues.extend(fstring_compat_issues(path, root, source)) + return { + "path": rel_path(path, root), + "ok": not issues, + "issue_count": len(issues), + "issues": issues, + } + + +def build_report(root: Path, raw_paths: list[str], target_python: str, generated_at: str) -> dict[str, Any]: + files = expand_scan_paths(root, raw_paths) + checked = [check_file(path, root) for path in files] + issues = [item for file_report in checked for item in file_report["issues"]] + syntax_error_count = sum(1 for item in issues if item["rule"] == "current-python-syntax") + fstring_count = sum(1 for item in issues if item["rule"] == "fstring-expression-backslash") + return { + "schema_version": "1.0", + "ok": not issues, + "generated_at": generated_at, + "root": rel_path(root, ROOT), + "summary": { + "target_python": target_python, + "file_count": len(checked), + "issue_count": len(issues), + "syntax_error_count": syntax_error_count, + "fstring_311_violation_count": fstring_count, + "decision": "pass" if not issues else "block-python-compat", + }, + "rules": [ + { + "key": "current-python-syntax", + "reason": "Every scanned Python source file must compile under the running interpreter.", + }, + { + "key": "fstring-expression-backslash", + "reason": "Python 3.11 rejects backslashes inside f-string expressions; keep escaping outside the expression.", + }, + ], + "files": checked, + "issues": issues, + "artifacts": { + "json": "reports/python_compatibility.json", + "markdown": "reports/python_compatibility.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Python Compatibility", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- target python: `{summary['target_python']}`", + f"- files scanned: `{summary['file_count']}`", + f"- issues: `{summary['issue_count']}`", + f"- syntax errors: `{summary['syntax_error_count']}`", + f"- f-string 3.11 violations: `{summary['fstring_311_violation_count']}`", + "", + "This report catches Python syntax and compatibility hazards that can pass on a newer local interpreter but fail in the supported CI/runtime interpreter.", + "", + "## Issues", + "", + "| Path | Line | Rule | Message |", + "| --- | ---: | --- | --- |", + ] + if report["issues"]: + for item in report["issues"]: + message = str(item["message"]).replace("|", "\\|") + lines.append(f"| `{item['path']}` | {item['line']} | `{item['rule']}` | {message} |") + else: + lines.append("| `none` | 0 | `none` | none |") + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check Python source compatibility for supported CI/runtime versions.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--path", action="append", default=[], help="Optional file or directory to scan relative to skill_dir.") + parser.add_argument("--target-python", default="3.11") + parser.add_argument("--output-json", default="reports/python_compatibility.json") + parser.add_argument("--output-md", default="reports/python_compatibility.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + root = Path(args.skill_dir).resolve() + report = build_report(root, args.path, args.target_python, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = root / output_json + if not output_md.is_absolute(): + output_md = root / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/reference_synthesis_markdown.py b/scripts/reference_synthesis_markdown.py new file mode 100644 index 00000000..ab44b66f --- /dev/null +++ b/scripts/reference_synthesis_markdown.py @@ -0,0 +1,99 @@ +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_reference_synthesis.py to keep reference synthesis modeling separate from Markdown rendering." + + +def render_markdown(summary: dict[str, Any]) -> str: + lines = [ + "# Reference Synthesis", + "", + f"Skill: `{summary['skill_name']}`", + f"- Description: {summary['description']}", + f"- Intent confidence: `{summary['intent_confidence']['score']}/100` (`{summary['intent_confidence']['band']}`)", + "", + "## Live GitHub Benchmarks", + "", + ] + if summary["github_benchmarks"]: + for repo in summary["github_benchmarks"]: + lines.extend( + [ + f"### {repo['name']}", + f"- URL: {repo['url']}", + f"- Stars: `{repo['stars']}`", + ] + ) + for item in repo.get("borrow", []): + lines.append(f"- Borrow: {item}") + lines.append("") + else: + lines.append("- No live GitHub benchmarks are attached yet.") + lines.append("") + + lines.extend(["## Curated World-Class Pattern Tracks", ""]) + for track in summary["source_tracks"]: + lines.extend( + [ + f"### {track['name']}", + f"- Type: `{track['source_type']}`", + f"- Evidence mode: `{track['evidence_mode']}`", + f"- Why relevant: {track['why_relevant']}", + f"- Borrow: {track['borrow']}", + f"- Avoid: {track['avoid']}", + "", + ] + ) + + lines.extend(["## Borrow Now", ""]) + for item in summary["synthesis"]["borrow_now"]: + lines.append(f"- {item}") + + lines.extend(["", "## Avoid Now", ""]) + for item in summary["synthesis"]["avoid_now"]: + lines.append(f"- {item}") + + lines.extend(["", "## Pattern Gate", ""]) + pattern_gate = summary["synthesis"]["pattern_gate"] + lines.append(f"- Summary: {pattern_gate['summary']}") + lines.append(f"- Acceptance threshold: `{pattern_gate['threshold']}/4`") + if pattern_gate["accepted"]: + lines.append("- Accepted patterns:") + for item in pattern_gate["accepted"][:5]: + lines.append( + f" - **{item['name']}**: {item['score']}/4 " + f"({', '.join(item['passed'])})" + ) + if pattern_gate["deferred"]: + lines.append("- Deferred patterns:") + for item in pattern_gate["deferred"][:5]: + lines.append( + f" - **{item['name']}**: missing {', '.join(item['missing']) or 'none'}" + ) + + lines.extend(["", "## Default Recommendation", ""]) + lines.append(f"- Summary: {summary['synthesis']['recommendation']['summary']}") + lines.append(f"- Why: {summary['synthesis']['recommendation']['why']}") + lines.append(f"- User decision required: `{summary['synthesis']['recommendation']['user_decision_required']}`") + + lines.extend(["", "## Visibility Mode", ""]) + lines.append(f"- Mode: `{summary['synthesis']['visibility']['mode']}`") + if summary["synthesis"]["visibility"]["reasons"]: + lines.append(f"- Reasons: {', '.join(summary['synthesis']['visibility']['reasons'])}") + lines.append(f"- User note: {summary['synthesis']['visibility']['user_note']}") + lines.append(f"- Reviewer note: {summary['synthesis']['visibility']['reviewer_note']}") + + lines.extend(["", "## Conflict Check", ""]) + if summary["synthesis"]["conflicts"]: + for conflict in summary["synthesis"]["conflicts"]: + lines.append(f"- **{conflict['key']}**: {conflict['summary']}") + else: + lines.append("- No material design conflict detected. Keep the synthesis silent for the user.") + + lines.extend(["", "## Quality Lift Thesis", ""]) + for item in summary["synthesis"]["quality_risers"]: + lines.append(f"- {item}") + + lines.extend(["", "## Decision Prompt", "", summary["synthesis"]["decision_prompt"], ""]) + return "\n".join(lines).strip() + "\n" diff --git a/scripts/registry_audit.py b/scripts/registry_audit.py index 8f197d2d..922a2f66 100644 --- a/scripts/registry_audit.py +++ b/scripts/registry_audit.py @@ -11,6 +11,8 @@ try: except ImportError: # pragma: no cover yaml = None +from skill_ir_paths import find_skill_ir as find_skill_ir_document + ROOT = Path(__file__).resolve().parent.parent DEFAULT_REGISTRY_DIR = ROOT / "registry" @@ -73,16 +75,7 @@ def read_frontmatter(path: Path) -> dict[str, Any]: def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]: - candidates = [ - skill_dir / "reports" / "skill-ir.json", - skill_dir / "skill-ir" / "examples" / f"{name}.json", - skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", - ] - for path in candidates: - payload = load_json(path) - if payload: - return payload, display_path(path, skill_dir) - return {}, "missing" + return find_skill_ir_document(skill_dir, name, fallback_source="missing") def license_id(skill_dir: Path) -> str: diff --git a/scripts/render_adoption_drift_report.py b/scripts/render_adoption_drift_report.py index 9ba81aaf..c1bf6d49 100644 --- a/scripts/render_adoption_drift_report.py +++ b/scripts/render_adoption_drift_report.py @@ -10,6 +10,7 @@ from typing import Any ROOT = Path(__file__).resolve().parent.parent ALLOWED_EVENTS = {"skill_activation", "skill_output", "script_run", "review_event"} +ADOPTION_EVENTS = {"skill_activation", "skill_output"} ALLOWED_ACTIVATION_TYPES = {"implicit", "explicit", "manual", "unknown"} ALLOWED_OUTCOMES = {"accepted", "edited", "rejected", "missed", "failed", "reviewed", "unknown"} ALLOWED_FAILURE_TYPES = { @@ -22,14 +23,17 @@ ALLOWED_FAILURE_TYPES = { "review_overdue", } ALLOWED_FIELDS = { + "command", "event", "skill", + "source", "version", "activation_type", "outcome", "failure_type", "timestamp", } +ALLOWED_SOURCES = {"manual", "yao_cli", "external", "unknown"} SENSITIVE_FIELDS = { "prompt", "content", @@ -111,6 +115,8 @@ def normalize_event(raw: dict[str, Any], defaults: dict[str, str], line_label: s timestamp = str(raw.get("timestamp") or utc_now()) skill = str(raw.get("skill") or defaults["skill"]) version = str(raw.get("version") or defaults["version"]) + source = str(raw.get("source") or "manual") + command = str(raw.get("command") or "unknown") if event not in ALLOWED_EVENTS: failures.append(f"{line_label}: unsupported event `{event}`") @@ -120,12 +126,18 @@ def normalize_event(raw: dict[str, Any], defaults: dict[str, str], line_label: s failures.append(f"{line_label}: unsupported outcome `{outcome}`") if failure_type not in ALLOWED_FAILURE_TYPES: failures.append(f"{line_label}: unsupported failure_type `{failure_type}`") + if source not in ALLOWED_SOURCES: + failures.append(f"{line_label}: unsupported source `{source}`") + if not command.replace("-", "").replace("_", "").isalnum() or len(command) > 64: + failures.append(f"{line_label}: command must use only letters, numbers, hyphens, or underscores and stay under 64 chars") if failures: return None, failures return { + "command": command, "event": event, "skill": skill, + "source": source, "version": version, "activation_type": activation_type, "outcome": outcome, @@ -165,21 +177,27 @@ def append_event(path: Path, event: dict[str, str]) -> None: def adoption_by_skill(events: list[dict[str, str]]) -> list[dict[str, Any]]: grouped: dict[str, Counter[str]] = defaultdict(Counter) + adoption_grouped: dict[str, Counter[str]] = defaultdict(Counter) for event in events: - grouped[event["skill"]][event["outcome"]] += 1 + skill = event["skill"] + grouped[skill]["events"] += 1 + if event["event"] in ADOPTION_EVENTS: + adoption_grouped[skill][event["outcome"]] += 1 rows = [] for skill, counts in sorted(grouped.items()): - total = sum(counts.values()) - adopted = counts["accepted"] + counts["edited"] + adoption_counts = adoption_grouped[skill] + adoption_total = sum(adoption_counts.values()) + adopted = adoption_counts["accepted"] + adoption_counts["edited"] rows.append( { "skill": skill, - "events": total, - "accepted": counts["accepted"], - "edited": counts["edited"], - "rejected": counts["rejected"], - "missed": counts["missed"], - "adoption_rate": round(adopted / total * 100, 1) if total else 0, + "events": counts["events"], + "adoption_events": adoption_total, + "accepted": adoption_counts["accepted"], + "edited": adoption_counts["edited"], + "rejected": adoption_counts["rejected"], + "missed": adoption_counts["missed"], + "adoption_rate": round(adopted / adoption_total * 100, 1) if adoption_total else 0, } ) return rows @@ -243,11 +261,15 @@ def next_candidates(summary: dict[str, Any]) -> list[dict[str, str]]: def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[str, Any]: - outcomes = Counter(event["outcome"] for event in events) + adoption_events = [event for event in events if event["event"] in ADOPTION_EVENTS] + outcomes = Counter(event["outcome"] for event in adoption_events) failures = Counter(event["failure_type"] for event in events if event["failure_type"] != "none") event_types = Counter(event["event"] for event in events) + source_types = Counter(event.get("source", "manual") for event in events) + command_counts = Counter(event.get("command", "unknown") for event in events if event.get("command", "unknown") != "unknown") adopted = outcomes["accepted"] + outcomes["edited"] event_count = len(events) + adoption_sample_count = len(adoption_events) missed_trigger = outcomes["missed"] + failures["under_trigger"] bad_output = failures["bad_output"] script_error = failures["script_error"] @@ -263,13 +285,14 @@ def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[s risk_band = "low" return { "event_count": event_count, + "adoption_sample_count": adoption_sample_count, "activation_count": event_types["skill_activation"], "accepted_count": outcomes["accepted"], "edited_count": outcomes["edited"], "rejected_count": outcomes["rejected"], "missed_count": outcomes["missed"], "failed_count": outcomes["failed"], - "adoption_rate": round(adopted / event_count * 100, 1) if event_count else 0, + "adoption_rate": round(adopted / adoption_sample_count * 100, 1) if adoption_sample_count else 0, "missed_trigger_count": missed_trigger, "wrong_trigger_count": wrong_trigger, "bad_output_count": bad_output, @@ -279,6 +302,8 @@ def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[s "risk_band": risk_band, "event_types": dict(sorted(event_types.items())), "failure_types": dict(sorted(failures.items())), + "source_types": dict(sorted(source_types.items())), + "command_counts": dict(sorted(command_counts.items())), } @@ -292,6 +317,7 @@ def render_markdown(report: dict[str, Any]) -> str: "## Summary", "", f"- Events: `{summary['event_count']}`", + f"- Adoption samples: `{summary['adoption_sample_count']}`", f"- Activation events: `{summary['activation_count']}`", f"- Adoption rate: `{summary['adoption_rate']}`", f"- Missed trigger signals: `{summary['missed_trigger_count']}`", @@ -309,16 +335,16 @@ def render_markdown(report: dict[str, Any]) -> str: "", "## Adoption By Skill", "", - "| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + "| Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", ] for row in report["adoption_by_skill"]: lines.append( - f"| `{row['skill']}` | {row['events']} | {row['accepted']} | {row['edited']} | " + f"| `{row['skill']}` | {row['events']} | {row['adoption_events']} | {row['accepted']} | {row['edited']} | " f"{row['rejected']} | {row['missed']} | {row['adoption_rate']} |" ) if not report["adoption_by_skill"]: - lines.append("| `none` | 0 | 0 | 0 | 0 | 0 | 0 |") + lines.append("| `none` | 0 | 0 | 0 | 0 | 0 | 0 | 0 |") lines.extend(["", "## Next Iteration Candidates", ""]) for item in report["next_iteration_candidates"]: lines.append(f"- `{item['signal']}`: {item['recommendation']}") @@ -328,6 +354,7 @@ def render_markdown(report: dict[str, Any]) -> str: for event in report["recent_events"]: lines.append( f"- `{event['timestamp']}` `{event['skill']}` event=`{event['event']}` " + f"source=`{event.get('source', 'manual')}` command=`{event.get('command', 'unknown')}` " f"activation=`{event['activation_type']}` outcome=`{event['outcome']}` failure=`{event['failure_type']}`" ) if not report["recent_events"]: @@ -404,6 +431,8 @@ def main() -> None: parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="unknown") parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown") parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none") + parser.add_argument("--source", choices=sorted(ALLOWED_SOURCES), default="manual") + parser.add_argument("--command", default="unknown") parser.add_argument("--timestamp") parser.add_argument("--skill-name") parser.add_argument("--version") @@ -416,6 +445,8 @@ def main() -> None: "activation_type": args.activation_type, "outcome": args.outcome, "failure_type": args.failure_type, + "source": args.source, + "command": args.command, } if args.timestamp: record_event["timestamp"] = args.timestamp diff --git a/scripts/render_architecture_maintainability.py b/scripts/render_architecture_maintainability.py new file mode 100644 index 00000000..d2cc0a76 --- /dev/null +++ b/scripts/render_architecture_maintainability.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Render a maintainability audit for the skill code surface.""" + +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent + + +EXCLUDED_PARTS = { + ".git", + ".mypy_cache", + ".pytest_cache", + "__pycache__", + "dist", + ".previews", +} + + +def iter_python_files(skill_dir: Path) -> list[Path]: + roots = [skill_dir / "scripts", skill_dir / "tests"] + files: list[Path] = [] + for root in roots: + if not root.exists(): + continue + for path in root.rglob("*.py"): + rel_parts = path.relative_to(skill_dir).parts + if any(part in EXCLUDED_PARTS for part in rel_parts): + continue + if len(rel_parts) >= 2 and rel_parts[0] == "tests" and rel_parts[1].startswith("tmp"): + continue + files.append(path) + return sorted(files) + + +def rel(skill_dir: Path, path: Path) -> str: + try: + return str(path.resolve().relative_to(skill_dir.resolve())) + except ValueError: + return str(path) + + +def classify_python_file(path: Path, text: str) -> str: + if 'SCRIPT_INTERFACE = "internal-module"' in text or "SCRIPT_INTERFACE = 'internal-module'" in text: + return "internal-module" + if "argparse.ArgumentParser" in text or ".add_argument(" in text or "argparse" in text: + return "cli-script" + if path.parts[-2:-1] == ("tests",) or "/tests/" in path.as_posix(): + return "test" + return "module" + + +def recommendation_for(path: str) -> str: + if path == "scripts/yao.py": + return "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." + if path == "scripts/render_review_studio.py": + return "Move data loading and large section renderers into focused review_studio_* modules." + if path == "scripts/render_review_viewer.py": + return "Split viewer data assembly from HTML section rendering." + if path.startswith("tests/"): + return "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + return "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + + +def count_handlers_in_file(path: Path) -> int: + if not path.exists(): + return 0 + return sum(1 for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.startswith("def command_")) + + +def command_module_paths(skill_dir: Path) -> list[Path]: + scripts_dir = skill_dir / "scripts" + if not scripts_dir.exists(): + return [] + paths = [scripts_dir / "yao.py"] + paths.extend(sorted(scripts_dir.glob("yao_cli_*commands.py"))) + return [path for path in paths if path.exists()] + + +def count_cli_command_handlers(skill_dir: Path) -> int: + return sum(count_handlers_in_file(path) for path in command_module_paths(skill_dir)) + + +def build_report(skill_dir: Path, warn_lines: int, block_lines: int, trend_lines: int, generated_at: str) -> dict[str, Any]: + files = iter_python_files(skill_dir) + watch_lines = max(1, int(warn_lines * 0.8)) + early_watch_lines = max(1, min(trend_lines, watch_lines)) + records: list[dict[str, Any]] = [] + internal_count = 0 + cli_count = 0 + test_count = 0 + script_count = 0 + for path in files: + text = path.read_text(encoding="utf-8", errors="replace") + line_count = len(text.splitlines()) + kind = classify_python_file(path, text) + rel_path = rel(skill_dir, path) + if kind == "internal-module": + internal_count += 1 + if kind == "cli-script": + cli_count += 1 + if rel_path.startswith("tests/"): + test_count += 1 + if rel_path.startswith("scripts/"): + script_count += 1 + severity = "pass" + if line_count >= block_lines: + severity = "block" + elif line_count >= warn_lines: + severity = "warn" + early_watch = severity == "pass" and line_count >= early_watch_lines + records.append( + { + "path": rel_path, + "lines": line_count, + "kind": kind, + "severity": severity, + "early_watch": early_watch, + "recommendation": recommendation_for(rel_path), + } + ) + records.sort(key=lambda item: (-int(item["lines"]), str(item["path"]))) + hotspots = [item for item in records if item["severity"] in {"warn", "block"}] + watchlist = [item for item in records if item["severity"] == "pass" and int(item["lines"]) >= watch_lines] + early_watchlist = [ + item + for item in records + if item["severity"] == "pass" and int(item["lines"]) >= early_watch_lines and item not in watchlist + ] + blockers = [item for item in records if item["severity"] == "block"] + summary = { + "python_file_count": len(records), + "script_file_count": script_count, + "test_file_count": test_count, + "internal_module_count": internal_count, + "cli_script_count": cli_count, + "command_handler_count": count_cli_command_handlers(skill_dir), + "entrypoint_command_handler_count": count_handlers_in_file(skill_dir / "scripts" / "yao.py"), + "command_module_count": len(command_module_paths(skill_dir)), + "warn_line_threshold": warn_lines, + "watch_line_threshold": watch_lines, + "early_watch_line_threshold": early_watch_lines, + "block_line_threshold": block_lines, + "largest_file_lines": records[0]["lines"] if records else 0, + "watchlist_count": len(watchlist), + "early_watchlist_count": len(early_watchlist), + "hotspot_count": len(hotspots), + "blocker_count": len(blockers), + "decision": "block-maintainability" if blockers else ("watch-maintainability-hotspots" if hotspots else "pass"), + } + return { + "schema_version": "1.0", + "ok": not blockers, + "generated_at": generated_at, + "skill_dir": ".", + "summary": summary, + "largest_files": records[:12], + "watchlist": watchlist[:12], + "early_watchlist": early_watchlist[:12], + "hotspots": hotspots, + "actions": [ + { + "path": item["path"], + "severity": item["severity"], + "action": item["recommendation"], + } + for item in hotspots[:8] + ], + "artifacts": { + "json": "reports/architecture_maintainability.json", + "markdown": "reports/architecture_maintainability.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Architecture Maintainability", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- python files: `{summary['python_file_count']}`", + f"- scripts: `{summary['script_file_count']}`", + f"- tests: `{summary['test_file_count']}`", + f"- internal modules: `{summary['internal_module_count']}`", + f"- CLI scripts: `{summary['cli_script_count']}`", + f"- Yao CLI command handlers: `{summary['command_handler_count']}`", + f"- entrypoint command handlers: `{summary['entrypoint_command_handler_count']}`", + f"- command modules: `{summary['command_module_count']}`", + f"- largest file lines: `{summary['largest_file_lines']}`", + f"- early watch threshold lines: `{summary['early_watch_line_threshold']}`", + f"- early watchlist: `{summary['early_watchlist_count']}`", + f"- watch threshold lines: `{summary['watch_line_threshold']}`", + f"- watchlist: `{summary['watchlist_count']}`", + f"- hotspots: `{summary['hotspot_count']}`", + f"- blockers: `{summary['blocker_count']}`", + "", + "This report keeps maintainability risk visible before the Meta Skill grows more gates, renderers, and CLI commands.", + "", + "## Hotspots", + "", + ] + hotspots = report.get("hotspots", []) + if hotspots: + lines.extend(["| File | Lines | Kind | Severity | Recommended action |", "| --- | ---: | --- | --- | --- |"]) + for item in hotspots: + lines.append( + f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | `{item['severity']}` | {item['recommendation']} |" + ) + else: + lines.append("No file-size hotspots found.") + lines.extend(["", "## Watchlist", ""]) + watchlist = report.get("watchlist", []) + if watchlist: + lines.extend(["| File | Lines | Kind | Recommended next split |", "| --- | ---: | --- | --- |"]) + for item in watchlist: + lines.append(f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | {item['recommendation']} |") + else: + lines.append("No near-threshold files found.") + lines.extend(["", "## Early Watchlist", ""]) + early_watchlist = report.get("early_watchlist", []) + if early_watchlist: + lines.extend(["| File | Lines | Kind | Recommended next split |", "| --- | ---: | --- | --- |"]) + for item in early_watchlist: + lines.append(f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | {item['recommendation']} |") + else: + lines.append("No early watch files found.") + lines.extend(["", "## Largest Files", ""]) + if report.get("largest_files"): + lines.extend(["| File | Lines | Kind | Severity |", "| --- | ---: | --- | --- |"]) + for item in report["largest_files"]: + lines.append(f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | `{item['severity']}` |") + else: + lines.append("No Python files found under scripts/ or tests/.") + lines.extend( + [ + "", + "## Release Rule", + "", + "- `block` hotspots should be split before governed release.", + "- `warn` hotspots can ship only when Review Studio keeps them visible and a reviewer accepts the modularization plan.", + "- Do not split a file only for line count; split when a stable responsibility boundary is clear.", + ] + ) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render architecture maintainability evidence for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--warn-lines", type=int, default=900) + parser.add_argument("--block-lines", type=int, default=1500) + parser.add_argument("--trend-lines", type=int, default=600) + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, args.warn_lines, args.block_lines, args.trend_lines, args.generated_at) + output_json = Path(args.output_json) if args.output_json else skill_dir / "reports" / "architecture_maintainability.json" + output_md = Path(args.output_md) if args.output_md else skill_dir / "reports" / "architecture_maintainability.md" + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_benchmark_reproducibility.py b/scripts/render_benchmark_reproducibility.py new file mode 100644 index 00000000..d422c6d3 --- /dev/null +++ b/scripts/render_benchmark_reproducibility.py @@ -0,0 +1,685 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import subprocess +from datetime import date +from pathlib import Path +from typing import Any + +from benchmark_release_lock import git_status, release_lock_status + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a release-facing benchmark reproducibility manifest and Markdown report." + +METHODOLOGY_SECTIONS = [ + "## Benchmark Types", + "## Sample Sources", + "## Evaluation Dimensions", + "## Weighting Rule", + "## Failure Disclosure", + "## Reproduction", +] + +REQUIRED_ARTIFACTS = [ + ("methodology", "reports/benchmark_methodology.md"), + ("failure_disclosure", "evals/failure-cases.md"), + ("output_cases", "evals/output/cases.jsonl"), + ("output_schema", "evals/output/schema.json"), + ("output_scorecard", "reports/output_quality_scorecard.json"), + ("output_execution", "reports/output_execution_runs.json"), + ("blind_review", "reports/output_blind_review_pack.json"), + ("review_adjudication", "reports/output_review_adjudication.json"), + ("trigger_scorecard", "reports/route_scorecard.json"), + ("runtime_conformance", "reports/conformance_matrix.json"), + ("trust_report", "reports/security_trust_report.json"), + ("python_compatibility", "reports/python_compatibility.json"), + ("registry_audit", "reports/registry_audit.json"), + ("package_verification", "reports/package_verification.json"), + ("install_simulation", "reports/install_simulation.json"), + ("skill_os2_audit", "reports/skill_os2_audit.json"), + ("world_class_evidence_plan", "reports/world_class_evidence_plan.json"), + ("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"), + ("world_class_evidence_intake", "reports/world_class_evidence_intake.json"), + ("world_class_evidence_preflight", "reports/world_class_evidence_preflight.json"), + ("world_class_submission_review", "reports/world_class_submission_review.json"), + ("world_class_operator_runbook", "reports/world_class_operator_runbook.json"), + ("world_class_operator_runbook_markdown", "reports/world_class_operator_runbook.md"), + ("world_class_operator_runbook_html", "reports/world_class_operator_runbook.html"), + ("world_class_claim_guard", "reports/world_class_claim_guard.json"), +] + +REPRODUCTION_COMMANDS = [ + { + "label": "source commit", + "command": "git rev-parse HEAD", + "evidence": "git commit hash", + }, + { + "label": "trigger eval", + "command": "make eval-suite", + "evidence": "reports/eval_suite.json", + }, + { + "label": "output eval", + "command": "python3 scripts/yao.py output-eval", + "evidence": "reports/output_quality_scorecard.json", + }, + { + "label": "output execution", + "command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'", + "evidence": "reports/output_execution_runs.json", + }, + { + "label": "blind review adjudication", + "command": "python3 scripts/yao.py output-review", + "evidence": "reports/output_review_adjudication.json", + }, + { + "label": "skill ir", + "command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json", + "evidence": "skill-ir/examples/yao-meta-skill.json", + }, + { + "label": "runtime conformance", + "command": "python3 scripts/yao.py conformance .", + "evidence": "reports/conformance_matrix.json", + }, + { + "label": "trust report", + "command": "python3 scripts/yao.py trust .", + "evidence": "reports/security_trust_report.json", + }, + { + "label": "python compatibility", + "command": "python3 scripts/yao.py python-compat .", + "evidence": "reports/python_compatibility.json", + }, + { + "label": "package", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip", + "evidence": "dist/yao-meta-skill.zip", + }, + { + "label": "package verify", + "command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip", + "evidence": "reports/package_verification.json", + }, + { + "label": "install simulate", + "command": "python3 scripts/yao.py install-simulate . --package-dir dist", + "evidence": "reports/install_simulation.json", + }, + { + "label": "registry audit", + "command": "python3 scripts/yao.py registry-audit .", + "evidence": "reports/registry_audit.json", + }, + { + "label": "skill os audit", + "command": "python3 scripts/yao.py skill-os2-audit .", + "evidence": "reports/skill_os2_audit.json", + }, + { + "label": "world-class evidence plan", + "command": "python3 scripts/yao.py world-class-evidence .", + "evidence": "reports/world_class_evidence_plan.json", + }, + { + "label": "world-class evidence ledger", + "command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_ledger.json", + }, + { + "label": "world-class evidence intake", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_intake.json", + }, + { + "label": "world-class evidence preflight", + "command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_evidence_preflight.json", + }, + { + "label": "world-class submission review", + "command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_submission_review.json", + }, + { + "label": "world-class operator runbook", + "command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions", + "evidence": "reports/world_class_operator_runbook.json", + }, + { + "label": "world-class claim guard", + "command": "python3 scripts/yao.py world-class-claim-guard .", + "evidence": "reports/world_class_claim_guard.json", + }, + { + "label": "evidence consistency", + "command": "python3 scripts/yao.py evidence-consistency .", + "evidence": "reports/evidence_consistency.json", + }, + { + "label": "full ci", + "command": "make ci-test", + "evidence": "CI target output", + }, +] + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_commit(skill_dir: Path) -> str: + try: + proc = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=skill_dir, + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return "unknown" + return proc.stdout.strip() or "unknown" + + +def count_jsonl(path: Path) -> int: + if not path.exists(): + return 0 + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + + +def count_failure_cases(path: Path) -> int: + if not path.exists(): + return 0 + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.startswith("### ")) + + +def methodology_check(path: Path) -> dict[str, Any]: + text = path.read_text(encoding="utf-8") if path.exists() else "" + sections = [{"heading": heading, "exists": heading in text} for heading in METHODOLOGY_SECTIONS] + return { + "path": "reports/benchmark_methodology.md", + "exists": path.exists(), + "sections": sections, + "missing_sections": [item["heading"] for item in sections if not item["exists"]], + } + + +def artifact_record(skill_dir: Path, label: str, rel: str) -> dict[str, Any]: + path = skill_dir / rel + record: dict[str, Any] = { + "label": label, + "path": rel, + "exists": path.exists(), + } + if path.exists() and path.is_file(): + record["bytes"] = path.stat().st_size + record["sha256"] = sha256_file(path) + return record + + +def evidence_bundle_fingerprint(artifacts: list[dict[str, Any]]) -> dict[str, Any]: + digest = hashlib.sha256() + existing_count = 0 + missing_paths = [] + for artifact in sorted(artifacts, key=lambda item: str(item.get("path", ""))): + path = str(artifact.get("path", "")) + digest.update(path.encode("utf-8")) + digest.update(b"\0") + digest.update(str(artifact.get("label", "")).encode("utf-8")) + digest.update(b"\0") + digest.update(str(bool(artifact.get("exists"))).encode("utf-8")) + digest.update(b"\0") + digest.update(str(artifact.get("sha256", "")).encode("utf-8")) + digest.update(b"\0") + if artifact.get("exists"): + existing_count += 1 + else: + missing_paths.append(path) + return { + "algorithm": "sha256(path,label,exists,artifact_sha256)", + "artifact_count": len(artifacts), + "existing_count": existing_count, + "missing_count": len(missing_paths), + "missing_paths": missing_paths, + "sha256": digest.hexdigest(), + } + + +def public_claim_blockers( + local_reproducibility_ready: bool, + release_lock_ready: bool, + provider_evidence_complete: bool, + human_review_complete: bool, + world_class_ready: bool, + world_class_open_gap_count: int, + world_class_ledger_pending_count: int, + world_class_source_check_count: int, + world_class_source_pass_count: int, + world_class_source_blocked_count: int, +) -> list[str]: + blockers = [] + if not local_reproducibility_ready: + blockers.append("local benchmark reproducibility is incomplete") + if not release_lock_ready: + blockers.append("release lock is not clean or commit is unavailable") + if not provider_evidence_complete: + blockers.append("provider-backed model holdout evidence is incomplete") + if not human_review_complete: + blockers.append("human blind-review adjudication is incomplete") + if not world_class_ready: + blockers.append( + f"world-class evidence is not accepted yet ({world_class_open_gap_count} open gaps, " + f"{world_class_ledger_pending_count} ledger pending)" + ) + if ( + world_class_source_check_count == 0 + or world_class_source_pass_count != world_class_source_check_count + or world_class_source_blocked_count > 0 + ): + blockers.append( + "world-class source checks are not all accepted " + f"({world_class_source_pass_count}/{world_class_source_check_count} pass, " + f"{world_class_source_blocked_count} blocked)" + ) + return blockers + + +def beta_test_blockers( + local_reproducibility_ready: bool, + release_lock_ready: bool, + provider_evidence_complete: bool, +) -> list[str]: + blockers = [] + if not local_reproducibility_ready: + blockers.append("local benchmark reproducibility is incomplete") + if not release_lock_ready: + blockers.append("release lock is not clean or commit is unavailable") + if not provider_evidence_complete: + blockers.append("provider-backed model holdout source evidence is incomplete") + return blockers + + +BETA_DEFERRED_EVIDENCE = { + "provider-holdout": { + "label": "Provider holdout ledger review", + "reason": "Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims.", + }, + "human-adjudication": { + "label": "Human blind-review adjudication", + "reason": "Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims.", + }, + "native-permission-enforcement": { + "label": "Native permission enforcement evidence", + "reason": "Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims.", + }, + "native-client-telemetry": { + "label": "Real client telemetry evidence", + "reason": "Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims.", + }, +} + + +def beta_deferred_evidence(world_class_ledger: dict[str, Any]) -> list[dict[str, str]]: + deferred = [] + entries = world_class_ledger.get("entries", []) + if not isinstance(entries, list): + return deferred + for entry in entries: + if not isinstance(entry, dict) or entry.get("status") != "pending": + continue + key = str(entry.get("key") or "") + if not key: + continue + fallback = BETA_DEFERRED_EVIDENCE.get(key, {}) + deferred.append( + { + "key": key, + "label": str(entry.get("label") or fallback.get("label") or key), + "reason": str( + fallback.get("reason") + or "Formal evidence is still pending; deferred for beta/public testing and still required before public claims." + ), + } + ) + return deferred + + +def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: + reports = skill_dir / "reports" + output_quality = load_json(reports / "output_quality_scorecard.json") + output_execution = load_json(reports / "output_execution_runs.json") + output_review = load_json(reports / "output_review_adjudication.json") + skill_os2 = load_json(reports / "skill_os2_audit.json") + world_class_plan = load_json(reports / "world_class_evidence_plan.json") + world_class_ledger = load_json(reports / "world_class_evidence_ledger.json") + trust = load_json(reports / "security_trust_report.json") + package_verification = load_json(reports / "package_verification.json") + methodology = methodology_check(reports / "benchmark_methodology.md") + artifacts = [artifact_record(skill_dir, label, rel) for label, rel in REQUIRED_ARTIFACTS] + evidence_bundle = evidence_bundle_fingerprint(artifacts) + missing_artifacts = [item["path"] for item in artifacts if not item["exists"]] + output_summary = output_quality.get("summary", {}) + execution_summary = output_execution.get("summary", {}) + review_summary = output_review.get("summary", {}) + failure_case_count = count_failure_cases(skill_dir / "evals" / "failure-cases.md") + output_case_count = count_jsonl(skill_dir / "evals" / "output" / "cases.jsonl") + status = git_status(skill_dir) + commit = git_commit(skill_dir) + release_lock = release_lock_status(status, commit) + local_reproducibility_ready = ( + not methodology["missing_sections"] + and not missing_artifacts + and output_case_count >= 5 + and failure_case_count > 0 + and output_summary.get("gate_pass") is True + and execution_summary.get("command_executed_count", 0) > 0 + and execution_summary.get("timing_observed_count", 0) > 0 + ) + human_review_complete = review_summary.get("pair_count", 0) > 0 and review_summary.get("pending_count", 0) == 0 + provider_evidence_complete = execution_summary.get("model_executed_count", 0) > 0 and execution_summary.get("token_observed_count", 0) > 0 + world_class_ready = bool(skill_os2.get("summary", {}).get("world_class_ready", False)) + world_class_open_gap_count = int(skill_os2.get("summary", {}).get("open_gap_count", 0) or 0) + world_class_summary = world_class_ledger.get("summary", {}) + world_class_ledger_pending_count = int(world_class_summary.get("pending_count", 0) or 0) + world_class_source_check_count = int(world_class_summary.get("source_check_count", 0) or 0) + world_class_source_pass_count = int(world_class_summary.get("source_pass_count", 0) or 0) + world_class_source_blocked_count = int(world_class_summary.get("source_blocked_count", 0) or 0) + claim_blockers = public_claim_blockers( + local_reproducibility_ready, + release_lock["ready"], + provider_evidence_complete, + human_review_complete, + world_class_ready, + world_class_open_gap_count, + world_class_ledger_pending_count, + world_class_source_check_count, + world_class_source_pass_count, + world_class_source_blocked_count, + ) + public_claim_ready = not claim_blockers + beta_blockers = beta_test_blockers( + local_reproducibility_ready, + release_lock["ready"], + provider_evidence_complete, + ) + beta_deferred = beta_deferred_evidence(world_class_ledger) + beta_test_ready = not beta_blockers + limitations = [ + "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims.", + ] + if provider_evidence_complete: + limitations.insert( + 1, + "Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.", + ) + else: + limitations.insert( + 1, + "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + ) + return { + "schema_version": "1.0", + "ok": local_reproducibility_ready, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "commit": commit, + "git_status": status, + "summary": { + "reproducibility_ready": local_reproducibility_ready, + "release_lock_ready": release_lock["ready"], + "methodology_complete": not methodology["missing_sections"], + "required_artifact_count": len(artifacts), + "missing_artifact_count": len(missing_artifacts), + "evidence_bundle_sha256": evidence_bundle["sha256"], + "source_contract_sha256": trust.get("summary", {}).get("package_sha256", ""), + "archive_sha256": package_verification.get("summary", {}).get("archive_sha256", ""), + "output_case_count": output_case_count, + "failure_disclosure_count": failure_case_count, + "command_count": len(REPRODUCTION_COMMANDS), + "command_executed_count": execution_summary.get("command_executed_count", 0), + "timing_observed_count": execution_summary.get("timing_observed_count", 0), + "model_executed_count": execution_summary.get("model_executed_count", 0), + "token_observed_count": execution_summary.get("token_observed_count", 0), + "human_review_complete": human_review_complete, + "provider_evidence_complete": provider_evidence_complete, + "world_class_ready": world_class_ready, + "world_class_open_gap_count": world_class_open_gap_count, + "world_class_task_count": world_class_plan.get("summary", {}).get("task_count", 0), + "world_class_ledger_pending_count": world_class_ledger_pending_count, + "world_class_source_check_count": world_class_source_check_count, + "world_class_source_pass_count": world_class_source_pass_count, + "world_class_source_blocked_count": world_class_source_blocked_count, + "beta_test_ready": beta_test_ready, + "beta_test_blocker_count": len(beta_blockers), + "beta_test_deferred_evidence_count": len(beta_deferred), + "public_claim_ready": public_claim_ready, + "public_claim_blocker_count": len(claim_blockers), + "working_tree_dirty": status.get("dirty"), + "changed_file_count": status.get("changed_file_count"), + "source_tree_dirty": status.get("source_dirty"), + "source_changed_file_count": status.get("source_changed_file_count"), + "generated_tree_dirty": status.get("generated_dirty"), + "generated_changed_file_count": status.get("generated_changed_file_count"), + }, + "beta_test_release": { + "ready": beta_test_ready, + "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "blockers": beta_blockers, + "allowed_deferred_evidence": beta_deferred, + "policy": "Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted.", + "required_wording": "Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline.", + }, + "public_claim": { + "ready": public_claim_ready, + "scope": "public benchmark or world-class readiness claim", + "blockers": claim_blockers, + "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks.", + }, + "release_lock": release_lock, + "evidence_bundle": evidence_bundle, + "methodology": methodology, + "artifacts_checked": artifacts, + "missing_artifacts": missing_artifacts, + "reproduction_commands": REPRODUCTION_COMMANDS, + "failure_disclosure": { + "path": "evals/failure-cases.md", + "case_count": failure_case_count, + "policy": "Keep representative failures visible and tied to regression checks.", + }, + "limitations": limitations, + "artifacts": { + "json": "reports/benchmark_reproducibility.json", + "markdown": "reports/benchmark_reproducibility.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Benchmark Reproducibility", + "", + f"Generated at: `{report['generated_at']}`", + f"Commit: `{report['commit']}`", + f"Working tree dirty at generation: `{str(summary.get('working_tree_dirty')).lower()}`", + f"Source tree dirty at generation: `{str(summary.get('source_tree_dirty')).lower()}`", + f"Generated evidence dirty at generation: `{str(summary.get('generated_tree_dirty')).lower()}`", + f"Evidence bundle SHA256: `{summary.get('evidence_bundle_sha256', '')}`", + "", + "## Summary", + "", + f"- reproducibility ready: `{str(summary['reproducibility_ready']).lower()}`", + f"- release lock ready: `{str(summary.get('release_lock_ready')).lower()}`", + f"- methodology complete: `{str(summary['methodology_complete']).lower()}`", + f"- required artifacts: `{summary['required_artifact_count']}`", + f"- missing artifacts: `{summary['missing_artifact_count']}`", + f"- source contract sha256: `{summary.get('source_contract_sha256', '')[:12]}`", + f"- archive sha256: `{summary.get('archive_sha256', '')[:12]}`", + f"- output cases: `{summary['output_case_count']}`", + f"- disclosed failure cases: `{summary['failure_disclosure_count']}`", + f"- reproduction commands: `{summary['command_count']}`", + f"- provider evidence complete: `{str(summary['provider_evidence_complete']).lower()}`", + f"- human review complete: `{str(summary['human_review_complete']).lower()}`", + f"- world-class ready: `{str(summary['world_class_ready']).lower()}`", + f"- world-class source checks: `{summary.get('world_class_source_pass_count', 0)}` pass / `{summary.get('world_class_source_check_count', 0)}` total; `{summary.get('world_class_source_blocked_count', 0)}` blocked", + f"- beta test ready: `{str(summary['beta_test_ready']).lower()}`", + f"- beta test blockers: `{summary['beta_test_blocker_count']}`", + f"- beta deferred evidence: `{summary['beta_test_deferred_evidence_count']}`", + f"- public claim ready: `{str(summary['public_claim_ready']).lower()}`", + f"- public claim blockers: `{summary['public_claim_blocker_count']}`", + f"- changed files at generation: `{summary.get('changed_file_count')}`", + f"- source changed files at generation: `{summary.get('source_changed_file_count')}`", + f"- generated changed files at generation: `{summary.get('generated_changed_file_count')}`", + "", + "This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.", + "", + ] + beta_release = report.get("beta_test_release", {}) + lines.extend( + [ + "## Beta Test Boundary", + "", + f"- ready: `{str(beta_release.get('ready')).lower()}`", + f"- scope: {beta_release.get('scope', 'beta/public test release')}", + f"- policy: {beta_release.get('policy', '')}", + f"- required wording: {beta_release.get('required_wording', '')}", + "", + "| Blocker |", + "| --- |", + ] + ) + beta_blockers = beta_release.get("blockers", []) + if beta_blockers: + lines.extend(f"| {item} |" for item in beta_blockers) + else: + lines.append("| none |") + lines.extend(["", "| Deferred evidence | Reason |", "| --- | --- |"]) + deferred = beta_release.get("allowed_deferred_evidence", []) + if deferred: + lines.extend(f"| `{item.get('key', '')}` | {item.get('reason', '')} |" for item in deferred) + else: + lines.append("| none | none |") + lines.extend( + [ + "", + "## Public Claim Boundary", + "", + ] + ) + lines.extend( + [ + f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`", + f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}", + f"- policy: {report.get('public_claim', {}).get('policy', '')}", + "", + "| Blocker |", + "| --- |", + ] + ) + blockers = report.get("public_claim", {}).get("blockers", []) + if blockers: + lines.extend(f"| {item} |" for item in blockers) + else: + lines.append("| none |") + lines.extend( + [ + "", + "## Release Lock", + "", + f"- ready: `{str(report.get('release_lock', {}).get('ready')).lower()}`", + f"- reason: {report.get('release_lock', {}).get('reason', 'unknown')}", + f"- status scope: {report.get('release_lock', {}).get('status_scope', 'generation-time status')}", + "", + "## Evidence Bundle", + ] + ) + lines.extend( + [ + "", + f"- algorithm: `{report.get('evidence_bundle', {}).get('algorithm', '')}`", + f"- artifacts: `{report.get('evidence_bundle', {}).get('existing_count', 0)}` / `{report.get('evidence_bundle', {}).get('artifact_count', 0)}`", + f"- sha256: `{report.get('evidence_bundle', {}).get('sha256', '')}`", + "", + "## Methodology Sections", + "", + "| Section | Status |", + "| --- | --- |", + ] + ) + for section in report["methodology"]["sections"]: + lines.append(f"| `{section['heading']}` | {'present' if section['exists'] else 'missing'} |") + lines.extend(["", "## Required Artifacts", "", "| Label | Path | Status | SHA256 |", "| --- | --- | --- | --- |"]) + for artifact in report["artifacts_checked"]: + digest = artifact.get("sha256", "") + lines.append( + f"| {artifact['label']} | `{artifact['path']}` | {'present' if artifact['exists'] else 'missing'} | `{digest[:12]}` |" + ) + lines.extend(["", "## Reproduction Commands", ""]) + for command in report["reproduction_commands"]: + lines.append(f"- `{command['command']}`") + lines.append(f" - evidence: `{command['evidence']}`") + lines.extend(["", "## Failure Disclosure", ""]) + disclosure = report["failure_disclosure"] + lines.append(f"- path: `{disclosure['path']}`") + lines.append(f"- disclosed cases: `{disclosure['case_count']}`") + lines.append(f"- policy: {disclosure['policy']}") + lines.extend(["", "## Limits", ""]) + lines.extend(f"- {item}" for item in report["limitations"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render benchmark reproducibility evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/benchmark_reproducibility.json") + parser.add_argument("--output-md", default="reports/benchmark_reproducibility.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_context_reports.py b/scripts/render_context_reports.py index ea57191d..a7f6e43d 100644 --- a/scripts/render_context_reports.py +++ b/scripts/render_context_reports.py @@ -52,6 +52,9 @@ def main() -> None: "budget_limit": stats.get("context_budget_limit"), "initial_tokens": stats.get("estimated_initial_load_tokens"), "skill_body_tokens": stats.get("skill_body_tokens"), + "deferred_resource_tokens": stats.get("deferred_resource_tokens"), + "large_deferred_resource_dirs": stats.get("large_deferred_resource_dirs", []), + "deferred_resource_governance": stats.get("deferred_resource_governance", {}), "quality_density": stats.get("quality_density"), "unused_resource_dirs": stats.get("unused_resource_dirs", []), "ok": report.get("ok"), @@ -72,14 +75,20 @@ def main() -> None: lines = [ "# Context Budget Summary", "", - "| Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status |", - "| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |", + "| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |", ] for row in rows: unused = ", ".join(row["unused_resource_dirs"]) if row["unused_resource_dirs"] else "-" + governance = row.get("deferred_resource_governance", {}) or {} + governance_status = governance.get("status", "unknown") + large_dirs = ", ".join( + f"{item['path']}:{item['estimated_tokens']}" + for item in row.get("large_deferred_resource_dirs", []) + ) or "-" status = "ok" if row["ok"] else "fail" lines.append( - f"| {row['label']} | `{row['path']}` | `{row['budget_tier']}` | {row['budget_limit']} | {row['initial_tokens']} | {row['skill_body_tokens']} | {row['quality_density']} | {unused} | {status} |" + f"| {row['label']} | `{row['path']}` | `{row['budget_tier']}` | {row['budget_limit']} | {row['initial_tokens']} | {row['skill_body_tokens']} | {row['deferred_resource_tokens']} | `{governance_status}` | {large_dirs} | {row['quality_density']} | {unused} | {status} |" ) lines.extend(["", "Per-target JSON reports are written beside each target and under `reports/`."]) (ROOT / "reports" / "context_budget.md").write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scripts/render_daily_skillops_report.py b/scripts/render_daily_skillops_report.py new file mode 100644 index 00000000..77b913c6 --- /dev/null +++ b/scripts/render_daily_skillops_report.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date, datetime, timezone +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any + +from propose_adaptation import build_report as build_proposal_report +from propose_adaptation import render_markdown as render_proposals_markdown +from skillops_opportunity import build_opportunities, decision_policy_contract, summarize_opportunities +from summarize_user_signals import build_report as build_pattern_report +from summarize_user_signals import render_markdown as render_patterns_markdown + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a Daily SkillOps report that summarizes explicit-source patterns, proposals, approval state, and release evidence without scanning private logs or applying patches." + +SUMMARY_FIELDS = [ + "decision", + "source_supplied", + "pattern_count", + "proposal_count", + "approval_count", + "pending_review_count", + "applied_count", + "rollback_count", + "local_blueprint_ready", + "public_world_class_ready", + "world_class_pending_count", + "release_lock_ready", + "evidence_consistency_ok", + "writes_source_files", + "auto_patch_enabled", + "failure_count", +] + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def report_date(generated_at: str) -> str: + if len(generated_at) >= 10: + candidate = generated_at[:10] + try: + date.fromisoformat(candidate) + return candidate + except ValueError: + pass + return date.today().isoformat() + + +def display_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return f"[external-explicit-source]/{path.name}" + + +def resolve_output(skill_dir: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else skill_dir / path + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def default_daily_path(skill_dir: Path, generated_at: str, suffix: str) -> Path: + return skill_dir / "reports" / "skillops" / "daily" / f"{report_date(generated_at)}.{suffix}" + + +def source_summary(patterns: dict[str, Any]) -> dict[str, Any]: + source = patterns.get("source", {}) if isinstance(patterns.get("source"), dict) else {} + privacy = patterns.get("privacy_contract", {}) if isinstance(patterns.get("privacy_contract"), dict) else {} + return { + "path": source.get("path", ""), + "fingerprint_sha256": source.get("fingerprint_sha256", ""), + "record_count": source.get("record_count", 0), + "explicit_source": privacy.get("explicit_source_required") is True, + "raw_content_stored": privacy.get("raw_content_stored") is True, + "redacted_excerpts_only": privacy.get("redacted_excerpts_only") is True, + } + + +def compact_proposals(proposals: dict[str, Any]) -> list[dict[str, Any]]: + rows = [] + for item in proposals.get("proposals", []): + if not isinstance(item, dict): + continue + rows.append( + { + "proposal_id": item.get("proposal_id", ""), + "pattern_id": item.get("pattern_id", ""), + "title": item.get("title", ""), + "risk_level": item.get("risk_level", ""), + "status": item.get("status", ""), + "requires_approval": item.get("requires_approval") is True, + "target_files": item.get("target_files", []), + "verification_commands": item.get("verification_commands", []), + } + ) + return rows + + +def build_actions(summary: dict[str, Any], proposal_rows: list[dict[str, Any]]) -> list[dict[str, str]]: + actions: list[dict[str, str]] = [] + if summary["source_supplied"] is False and summary["pattern_count"] == 0: + actions.append( + { + "key": "provide-explicit-source", + "priority": "medium", + "action": "Run daily-skillops with --source pointing to a curated JSONL or Markdown conversation summary.", + } + ) + if proposal_rows: + actions.append( + { + "key": "review-adaptation-proposals", + "priority": "high", + "action": "Review proposal-only adaptation items before preparing any approval ledger entry.", + } + ) + if summary["pending_review_count"]: + actions.append( + { + "key": "resolve-pending-approvals", + "priority": "high", + "action": "Approve, reject, or expire pending adaptive approval ledger entries before applying patches.", + } + ) + if summary["world_class_pending_count"]: + actions.append( + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for the pending world-class ledger entries.", + } + ) + if summary["evidence_consistency_ok"] is not True: + actions.append( + { + "key": "refresh-evidence-consistency", + "priority": "high", + "action": "Regenerate evidence-consistency before using this report for release decisions.", + } + ) + if not actions: + actions.append( + { + "key": "monitor", + "priority": "low", + "action": "No action required beyond routine daily monitoring.", + } + ) + return actions + + +def build_report( + skill_dir: Path, + generated_at: str, + source: Path | None = None, + min_support: int = 2, + allow_history_source: bool = False, + patterns_json: Path | None = None, + proposals_json: Path | None = None, + write_source_reports: bool = True, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports = skill_dir / "reports" + patterns_path = resolve_output(skill_dir, str(patterns_json or "reports/user_patterns.json")) + proposals_path = resolve_output(skill_dir, str(proposals_json or "reports/adaptation_proposals.json")) + failures: list[str] = [] + + if source is not None: + pattern_report = build_pattern_report( + skill_dir, + source, + min_support=max(2, min_support), + generated_at=generated_at, + allow_history_source=allow_history_source, + ) + if pattern_report.get("ok") is True and write_source_reports: + pattern_report["artifacts"] = { + "json": display_path(patterns_path, skill_dir), + "markdown": display_path(patterns_path.with_suffix(".md"), skill_dir), + } + write_json(patterns_path, pattern_report) + write_text(patterns_path.with_suffix(".md"), render_patterns_markdown(pattern_report)) + else: + failures.extend(pattern_report.get("failures", [])) + else: + pattern_report = load_json(patterns_path) + + if pattern_report and pattern_report.get("ok") is True: + proposal_source_path = patterns_path + with TemporaryDirectory() as temp_dir: + if source is not None and not write_source_reports: + proposal_source_path = Path(temp_dir) / "user_patterns.json" + write_json(proposal_source_path, pattern_report) + proposal_report = build_proposal_report(skill_dir, proposal_source_path, generated_at) + if proposal_report.get("ok") is True and write_source_reports: + proposal_report["artifacts"] = { + "json": display_path(proposals_path, skill_dir), + "markdown": display_path(proposals_path.with_suffix(".md"), skill_dir), + } + write_json(proposals_path, proposal_report) + write_text(proposals_path.with_suffix(".md"), render_proposals_markdown(proposal_report)) + elif proposal_report.get("ok") is not True: + failures.extend(proposal_report.get("failures", [])) + else: + proposal_report = load_json(proposals_path) + if source is not None and not pattern_report: + failures.append("Pattern report could not be built from the explicit source.") + + approval = load_json(reports / "adaptation_approval_ledger.json") + regression = load_json(reports / "adaptation_regression_report.json") + coverage = load_json(reports / "skill_os2_coverage.json") + ledger = load_json(reports / "world_class_evidence_ledger.json") + consistency = load_json(reports / "evidence_consistency.json") + benchmark = load_json(reports / "benchmark_reproducibility.json") + + pattern_summary = pattern_report.get("summary", {}) if isinstance(pattern_report.get("summary"), dict) else {} + proposal_summary = proposal_report.get("summary", {}) if isinstance(proposal_report.get("summary"), dict) else {} + approval_summary = approval.get("summary", {}) if isinstance(approval.get("summary"), dict) else {} + regression_summary = regression.get("summary", {}) if isinstance(regression.get("summary"), dict) else {} + coverage_summary = coverage.get("summary", {}) if isinstance(coverage.get("summary"), dict) else {} + ledger_summary = ledger.get("summary", {}) if isinstance(ledger.get("summary"), dict) else {} + benchmark_summary = benchmark.get("summary", {}) if isinstance(benchmark.get("summary"), dict) else {} + proposal_contract = proposal_report.get("proposal_contract", {}) if isinstance(proposal_report.get("proposal_contract"), dict) else {} + + proposal_rows = compact_proposals(proposal_report) + pattern_rows = pattern_report.get("patterns", []) if isinstance(pattern_report.get("patterns"), list) else [] + opportunities = build_opportunities(pattern_rows, proposal_rows) + opportunity_summary = summarize_opportunities(opportunities) + failure_count = len(failures) + summary = { + "decision": "blocked" if failure_count else "proposal-review" if proposal_rows else "monitor", + "source_supplied": source is not None, + "pattern_count": int(pattern_summary.get("pattern_count", 0) or 0), + "proposal_count": int(proposal_summary.get("proposal_count", 0) or len(proposal_rows)), + "approval_count": int(approval_summary.get("approval_count", 0) or 0), + "pending_review_count": int(approval_summary.get("pending_review_count", 0) or 0), + "applied_count": int(approval_summary.get("applied_count", 0) or regression_summary.get("applied_count", 0) or 0), + "rollback_count": int(approval_summary.get("rollback_count", 0) or regression_summary.get("rollback_count", 0) or 0), + "local_blueprint_ready": coverage_summary.get("local_blueprint_ready") is True, + "public_world_class_ready": coverage_summary.get("public_world_class_ready") is True, + "world_class_pending_count": int(ledger_summary.get("pending_count", coverage_summary.get("world_class_evidence_pending_count", 0)) or 0), + "release_lock_ready": benchmark_summary.get("release_lock_ready") is True, + "evidence_consistency_ok": consistency.get("ok") is True, + "writes_source_files": False, + "auto_patch_enabled": False, + "failure_count": failure_count, + } + actions = build_actions(summary, proposal_rows) + operations_contract = { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "explicit_source_required_for_scan": True, + "implicit_private_log_scan": False, + "raw_content_stored": False, + "redacted_excerpts_only": True, + "proposal_only": proposal_contract.get("proposal_only") is not False, + "approval_required_for_writes": True, + "writes_source_files": False, + "auto_patch_enabled": False, + "daily_report_counts_as_world_class_evidence": False, + } + report = { + "schema_version": "1.0", + "ok": failure_count == 0, + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + **{key: summary[key] for key in SUMMARY_FIELDS}, + "summary": summary, + "operations_contract": operations_contract, + "report_contract": { + "schema_version": "1.0", + "contract": "daily-skillops-report", + "top_level_mirrors_summary": True, + "summary_fields": SUMMARY_FIELDS, + "source_of_truth": ["summary", "operations_contract"], + }, + "source": source_summary(pattern_report) if pattern_report else {}, + "patterns": pattern_report.get("patterns", []) if isinstance(pattern_report.get("patterns"), list) else [], + "proposals": proposal_rows, + "opportunity_summary": opportunity_summary, + "opportunities": opportunities, + "decision_policy": decision_policy_contract(), + "approval": { + "approval_count": summary["approval_count"], + "pending_review_count": summary["pending_review_count"], + "applied_count": summary["applied_count"], + "rollback_count": summary["rollback_count"], + }, + "release_state": { + "local_blueprint_ready": summary["local_blueprint_ready"], + "public_world_class_ready": summary["public_world_class_ready"], + "world_class_pending_count": summary["world_class_pending_count"], + "release_lock_ready": summary["release_lock_ready"], + "evidence_consistency_ok": summary["evidence_consistency_ok"], + }, + "actions": actions, + "failures": failures, + "source_reports": { + "patterns": display_path(patterns_path, skill_dir), + "proposals": display_path(proposals_path, skill_dir), + "approval_ledger": "reports/adaptation_approval_ledger.json", + "regression": "reports/adaptation_regression_report.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + "evidence_consistency": "reports/evidence_consistency.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", + }, + "artifacts": { + "json": str(default_daily_path(skill_dir, generated_at, "json").relative_to(skill_dir)), + "markdown": str(default_daily_path(skill_dir, generated_at, "md").relative_to(skill_dir)), + }, + } + return report + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Daily SkillOps Report", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- source supplied: `{str(summary['source_supplied']).lower()}`", + f"- patterns: `{summary['pattern_count']}`", + f"- proposals: `{summary['proposal_count']}`", + f"- pending approvals: `{summary['pending_review_count']}`", + f"- applied patches: `{summary['applied_count']}`", + f"- rollbacks: `{summary['rollback_count']}`", + f"- local blueprint ready: `{str(summary['local_blueprint_ready']).lower()}`", + f"- public world-class ready: `{str(summary['public_world_class_ready']).lower()}`", + f"- world-class pending: `{summary['world_class_pending_count']}`", + f"- release lock ready: `{str(summary['release_lock_ready']).lower()}`", + f"- evidence consistency ok: `{str(summary['evidence_consistency_ok']).lower()}`", + "", + "This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence.", + "", + "## Privacy Boundary", + "", + ] + contract = report["operations_contract"] + for key in [ + "explicit_source_required_for_scan", + "implicit_private_log_scan", + "raw_content_stored", + "redacted_excerpts_only", + "proposal_only", + "approval_required_for_writes", + "writes_source_files", + "auto_patch_enabled", + "daily_report_counts_as_world_class_evidence", + ]: + lines.append(f"- {key}: `{str(contract[key]).lower()}`") + lines.extend(["", "## Actions", ""]) + for action in report["actions"]: + lines.append(f"- `{action['priority']}` {action['action']}") + lines.extend(["", "## Opportunities", ""]) + opportunity_summary = report.get("opportunity_summary", {}) + lines.extend( + [ + f"- count: `{opportunity_summary.get('opportunity_count', 0)}`", + f"- top score: `{opportunity_summary.get('top_score', 0)}`", + f"- ready for approval review: `{opportunity_summary.get('ready_for_approval_review_count', 0)}`", + ] + ) + for opportunity in report.get("opportunities", []): + lines.extend( + [ + f"### {opportunity.get('title', '')}", + "", + f"- ID: `{opportunity.get('opportunity_id', '')}`", + f"- Action: `{opportunity.get('action_type', '')}`", + f"- Decision: `{opportunity.get('decision', '')}`", + f"- Score: `{opportunity.get('score', 0)}`", + f"- Risk: `{opportunity.get('risk_level', '')}`", + f"- Policy: {opportunity.get('policy_reason', '')}", + ] + ) + lines.extend(["", "## Patterns", ""]) + patterns = report.get("patterns", []) + if not patterns: + lines.append("- No repeated pattern met the support threshold.") + for pattern in patterns: + lines.append( + f"- `{pattern.get('pattern_id', '')}`: {pattern.get('label', '')} " + f"(support `{pattern.get('support_count', 0)}`, confidence `{pattern.get('confidence', '')}`)" + ) + lines.extend(["", "## Proposals", ""]) + proposals = report.get("proposals", []) + if not proposals: + lines.append("- No adaptation proposals are waiting for review.") + for proposal in proposals: + lines.extend( + [ + f"### {proposal['title']}", + "", + f"- ID: `{proposal['proposal_id']}`", + f"- Pattern: `{proposal['pattern_id']}`", + f"- Status: `{proposal['status']}`", + f"- Risk: `{proposal['risk_level']}`", + f"- Requires approval: `{str(proposal['requires_approval']).lower()}`", + "- Target files:", + ] + ) + lines.extend(f" - `{path}`" for path in proposal.get("target_files", [])) + lines.append("- Verification:") + lines.extend(f" - `{command}`" for command in proposal.get("verification_commands", [])) + lines.append("") + lines.extend(["## Evidence", ""]) + for label, path in report["source_reports"].items(): + lines.append(f"- {label}: `{path}`") + if report["failures"]: + lines.extend(["", "## Failures", ""]) + lines.extend(f"- {failure}" for failure in report["failures"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a Daily SkillOps report from explicit-source adaptive evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--source", help="Explicit curated source file to scan before rendering the daily report.") + parser.add_argument("--patterns-json", default="reports/user_patterns.json") + parser.add_argument("--proposals-json", default="reports/adaptation_proposals.json") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--min-support", type=int, default=2) + parser.add_argument("--generated-at", default=utc_now()) + parser.add_argument("--allow-history-source", action="store_true") + parser.add_argument("--no-refresh-source-reports", action="store_true") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report( + skill_dir, + args.generated_at, + source=Path(args.source) if args.source else None, + min_support=args.min_support, + allow_history_source=args.allow_history_source, + patterns_json=Path(args.patterns_json), + proposals_json=Path(args.proposals_json), + write_source_reports=not args.no_refresh_source_reports, + ) + output_json = resolve_output(skill_dir, args.output_json) if args.output_json else default_daily_path(skill_dir, args.generated_at, "json") + output_md = resolve_output(skill_dir, args.output_md) if args.output_md else default_daily_path(skill_dir, args.generated_at, "md") + report["artifacts"] = { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + } + write_json(output_json, report) + write_text(output_md, render_markdown(report)) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py new file mode 100644 index 00000000..068c713d --- /dev/null +++ b/scripts/render_evidence_consistency.py @@ -0,0 +1,676 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +from datetime import date +from pathlib import Path +from typing import Any + +from evidence_consistency_artifact_roles import build_preflight_artifact_role_handoff_checks +from evidence_consistency_core import ( + ADOPTION_SUMMARY_KEYS, + BENCHMARK_SUMMARY_KEYS, + LEDGER_SUMMARY_KEYS, + LOCKSTEP_SECTIONS, + REQUIRED_REPORTS, + REQUIRED_TEXT_REPORTS, + add_check, + as_int, + beta_public_claim_split_values, + compare_summary_keys, + compare_values, + gate_by_key, + load_json, + load_text, + nested, + rel_path, + render_markdown, + report_contract, + scanned_surface_paths, +) +from evidence_consistency_phase_queue import build_phase_queue_consistency_check +from evidence_consistency_release import build_release_evidence_flow_check +from evidence_consistency_skill_os2_review import build_skill_os2_review_current_evidence_check +from evidence_consistency_world_class import build_world_class_workflow_check +from skill_ir_paths import find_skill_ir_path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a cross-report evidence consistency gate for generated Skill OS reports." + + +def git_worktree_status(skill_dir: Path) -> dict[str, Any]: + try: + proc = subprocess.run( + ["git", "-C", str(skill_dir), "status", "--porcelain=v1", "-uall"], + capture_output=True, + text=True, + check=False, + ) + except OSError: + return {"available": False, "clean": None, "changed_file_count": None} + if proc.returncode != 0: + return {"available": False, "clean": None, "changed_file_count": None} + changes = [line for line in proc.stdout.splitlines() if line.strip()] + return {"available": True, "clean": not changes, "changed_file_count": len(changes)} + + +def build_review_studio_gate_action_mirror_check(review_studio: dict[str, Any]) -> dict[str, Any]: + gates = review_studio.get("gates", []) if isinstance(review_studio.get("gates", []), list) else [] + actions = ( + review_studio.get("review_actions", []) + if isinstance(review_studio.get("review_actions", []), list) + else [] + ) + non_pass_gates = [gate for gate in gates if isinstance(gate, dict) and gate.get("status") != "pass"] + pass_gates = [gate for gate in gates if isinstance(gate, dict) and gate.get("status") == "pass"] + actions_by_gate = { + str(action.get("gate_key", "")): action + for action in actions + if isinstance(action, dict) and str(action.get("gate_key", "")) + } + expected = { + "non_pass_gate_keys": sorted(str(gate.get("key", "")) for gate in non_pass_gates), + "action_gate_keys": sorted(str(gate.get("key", "")) for gate in non_pass_gates), + "pass_gate_action_ids_empty": True, + "gate_action_mirrors": { + str(gate.get("key", "")): { + "review_action_id": f"review-action-{gate.get('key', '')}", + "review_action_status": gate.get("status", ""), + "title_present": True, + "next_step_present": True, + "reason_present": True, + "source_ref_present": True, + "verification_command_present": True, + } + for gate in non_pass_gates + }, + } + actual = { + "non_pass_gate_keys": sorted(str(gate.get("key", "")) for gate in non_pass_gates), + "action_gate_keys": sorted(actions_by_gate), + "pass_gate_action_ids_empty": all(not gate.get("review_action_id") for gate in pass_gates), + "gate_action_mirrors": {}, + } + for gate in non_pass_gates: + key = str(gate.get("key", "")) + action = actions_by_gate.get(key, {}) + actual["gate_action_mirrors"][key] = { + "review_action_id": gate.get("review_action_id", ""), + "review_action_status": gate.get("review_action_status", ""), + "title_present": bool(gate.get("review_action_title")), + "next_step_present": bool(gate.get("review_action_next_step")), + "reason_present": bool(gate.get("review_action_reason")), + "source_ref_present": bool(gate.get("review_action_source_ref_count")), + "verification_command_present": bool(gate.get("review_action_verification_command")), + "top_level_action_id": action.get("action_id", ""), + "top_level_action_status": action.get("status", ""), + "top_level_title_present": bool(action.get("title")), + "top_level_next_step_present": bool(action.get("next_step")), + "top_level_reason_present": bool(action.get("reason")), + "top_level_source_ref_present": bool(action.get("source_refs")), + "top_level_verification_command_present": bool(action.get("verification_command")), + } + expected["gate_action_mirrors"][key]["top_level_action_id"] = f"review-action-{key}" + expected["gate_action_mirrors"][key]["top_level_action_status"] = gate.get("status", "") + expected["gate_action_mirrors"][key]["top_level_title_present"] = True + expected["gate_action_mirrors"][key]["top_level_next_step_present"] = True + expected["gate_action_mirrors"][key]["top_level_reason_present"] = True + expected["gate_action_mirrors"][key]["top_level_source_ref_present"] = True + expected["gate_action_mirrors"][key]["top_level_verification_command_present"] = True + return { + "key": "review-studio-gate-action-mirror", + "label": "Review Studio gates mirror review actions", + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": [REQUIRED_REPORTS["review_studio"]], + "detail": ( + "Every non-pass Review Studio gate must have exactly one top-level review action and a gate-local " + "action summary with source refs and a verification command." + ), + } + + +def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: + reports: dict[str, dict[str, Any]] = {} + text_reports: dict[str, str] = {} + checks: list[dict[str, Any]] = [] + load_failures: dict[str, str] = {} + for name, relative in REQUIRED_REPORTS.items(): + payload, failure = load_json(skill_dir / relative) + reports[name] = payload + if failure: + load_failures[relative] = failure + for name, relative in REQUIRED_TEXT_REPORTS.items(): + text, failure = load_text(skill_dir / relative) + text_reports[name] = text + if failure: + load_failures[relative] = failure + add_check( + checks, + key="required-report-artifacts", + label="Required report artifacts are readable", + status="pass" if not load_failures else "fail", + expected="all required JSON and Markdown reports exist and parse", + actual=load_failures or "all readable", + paths=list(REQUIRED_REPORTS.values()) + list(REQUIRED_TEXT_REPORTS.values()), + detail="The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable.", + ) + checks.append(build_release_evidence_flow_check(skill_dir)) + + benchmark = reports["benchmark"] + overview = reports["overview"] + interpretation = reports["interpretation"] + adoption = reports["adoption"] + ledger = reports["world_class_ledger"] + world_class_plan = reports["world_class_plan"] + world_class_intake = reports["world_class_intake"] + world_class_preflight = reports["world_class_preflight"] + world_class_submission_review = reports["world_class_submission_review"] + world_class_operator_runbook = reports["world_class_operator_runbook"] + coverage = reports["skill_os2_coverage"] + review_studio = reports["review_studio"] + package_verification = reports["package_verification"] + install_simulation = reports["install_simulation"] + trust = reports["trust"] + context_budget = reports["context_budget"] + claim_guard = reports["world_class_claim_guard"] + + checks.append(build_review_studio_gate_action_mirror_check(review_studio)) + + benchmark_summary = nested(benchmark, ["summary"], {}) + adoption_summary = nested(adoption, ["summary"], {}) + ledger_summary = nested(ledger, ["summary"], {}) + coverage_summary = nested(coverage, ["summary"], {}) + studio_summary = nested(review_studio, ["summary"], {}) + package_summary = nested(package_verification, ["summary"], {}) + install_summary = nested(install_simulation, ["summary"], {}) + trust_summary = nested(trust, ["summary"], {}) + context_stats = nested(context_budget, ["stats"], {}) + claim_guard_summary = nested(claim_guard, ["summary"], {}) + preflight_summary = nested(world_class_preflight, ["summary"], {}) + context_governance = ( + context_stats.get("deferred_resource_governance", {}) if isinstance(context_stats, dict) else {} + ) + if not isinstance(context_governance, dict): + context_governance = {} + large_deferred_dirs = context_stats.get("large_deferred_resource_dirs", []) if isinstance(context_stats, dict) else [] + top_deferred = "none" + if isinstance(large_deferred_dirs, list) and large_deferred_dirs: + first = large_deferred_dirs[0] if isinstance(large_deferred_dirs[0], dict) else {} + top_deferred = f"{first.get('path', 'resource')} {first.get('estimated_tokens', 'n/a')}" + context_expected_status = "pass" if context_budget.get("ok") else "block" + if context_budget.get("warnings") and context_expected_status == "pass": + context_expected_status = "warn" + context_expected_detail = ( + f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/" + f"{context_stats.get('context_budget_limit', 'n/a')}; " + f"deferred {context_stats.get('deferred_resource_tokens', 'n/a')}/" + f"{context_stats.get('deferred_resource_warn_threshold', 'n/a')}; " + f"top deferred {top_deferred}; " + f"resource governance {context_governance.get('status', 'unknown')}; " + f"quality density {context_stats.get('quality_density', 'n/a')}" + ) + context_gate = gate_by_key(review_studio, "context-budget") + compare_values( + checks, + key="review-studio-context-budget-mirror", + label="Review Studio mirrors context budget governance", + expected={ + "status": context_expected_status, + "detail": context_expected_detail, + "evidence": REQUIRED_REPORTS["context_budget"], + }, + actual={ + "status": context_gate.get("status"), + "detail": context_gate.get("detail"), + "evidence": context_gate.get("evidence"), + }, + paths=[REQUIRED_REPORTS["context_budget"], REQUIRED_REPORTS["review_studio"]], + detail=( + "Review Studio must not keep stale context warnings after context reports prove large deferred resources are governed." + ), + ) + if isinstance(benchmark_summary, dict): + compare_values( + checks, + key="benchmark-release-lock-self-consistency", + label="Benchmark release lock matches source dirty state", + expected=not bool(nested(benchmark, ["git_status", "source_dirty"], True)), + actual=benchmark_summary.get("release_lock_ready"), + paths=[REQUIRED_REPORTS["benchmark"]], + detail=( + "The benchmark release lock must be blocked by source changes, while generated evidence artifacts are " + "tracked as generation context." + ), + ) + current_git_status = git_worktree_status(skill_dir) + if current_git_status.get("available") and current_git_status.get("clean") is True: + compare_values( + checks, + key="benchmark-clean-worktree-release-lock", + label="Clean worktree keeps a clean benchmark release lock", + expected=True, + actual=benchmark_summary.get("release_lock_ready"), + paths=[REQUIRED_REPORTS["benchmark"]], + detail=( + "If the current worktree is clean, the committed benchmark report must not still carry a dirty release lock from an earlier generation." + ), + ) + else: + add_check( + checks, + key="benchmark-clean-worktree-release-lock", + label="Clean worktree keeps a clean benchmark release lock", + status="pass", + expected="checked only when git is available and the current worktree is clean", + actual=current_git_status, + paths=[REQUIRED_REPORTS["benchmark"]], + detail=( + "Dirty or non-git worktrees cannot prove final release-lock freshness, so this check is advisory until the final clean-lock pass." + ), + ) + skill_name = str(overview.get("name") or nested(review_studio, ["data", "frontmatter", "name"]) or skill_dir.name) + expected_skill_ir_path = find_skill_ir_path(skill_dir, skill_name, require_schema=True) + expected_skill_ir = { + "source_path": expected_skill_ir_path, + "exists": bool(expected_skill_ir_path and (skill_dir / expected_skill_ir_path).exists()), + "schema_version": load_json(skill_dir / expected_skill_ir_path)[0].get("schema_version") + if expected_skill_ir_path + else "", + } + actual_skill_ir = { + "overview_source_path": nested(overview, ["skill_ir", "source_path"], ""), + "interpretation_source_path": nested(interpretation, ["skill_ir", "source_path"], ""), + "review_studio_evidence_path": nested(review_studio, ["evidence_paths", "skill_ir"], ""), + "overview_deliverable": expected_skill_ir_path in nested(overview, ["skill_summary", "deliverables"], []), + "interpretation_deliverable": expected_skill_ir_path + in nested(interpretation, ["skill_summary", "deliverables"], []), + } + compare_values( + checks, + key="skill-ir-evidence-path-contract", + label="Human-facing reports expose the canonical Skill IR artifact", + expected={ + "overview_source_path": expected_skill_ir["source_path"], + "interpretation_source_path": expected_skill_ir["source_path"], + "review_studio_evidence_path": expected_skill_ir["source_path"], + "overview_deliverable": True, + "interpretation_deliverable": True, + "exists": True, + "schema_version": "2.0.0", + }, + actual={**actual_skill_ir, **{key: expected_skill_ir[key] for key in ("exists", "schema_version")}}, + paths=[ + REQUIRED_REPORTS["overview"], + REQUIRED_REPORTS["interpretation"], + REQUIRED_REPORTS["review_studio"], + expected_skill_ir_path or "skill-ir/examples", + ], + detail="Skill IR is the 2.0 platform-neutral semantic source, so user-facing reports must link to the artifact that actually exists.", + ) + for report_key, payload in [("overview", overview), ("interpretation", interpretation)]: + embedded_benchmark = nested(payload, ["benchmark_reproducibility"], {}) + compare_values( + checks, + key=f"{report_key}-benchmark-commit", + label=f"{report_key} embeds the benchmark commit", + expected=benchmark.get("commit"), + actual=nested(embedded_benchmark, ["commit"]), + paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]], + detail="Human-facing reports must point to the same benchmark release-lock commit.", + ) + compare_summary_keys( + checks, + key_prefix=f"{report_key}-benchmark-summary", + label=f"{report_key} embeds benchmark summary fields", + source_summary=benchmark_summary if isinstance(benchmark_summary, dict) else {}, + embedded_summary=nested(embedded_benchmark, ["summary"], {}), + keys=BENCHMARK_SUMMARY_KEYS, + paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]], + ) + compare_summary_keys( + checks, + key_prefix=f"{report_key}-adoption-summary", + label=f"{report_key} embeds adoption drift summary fields", + source_summary=adoption_summary if isinstance(adoption_summary, dict) else {}, + embedded_summary=nested(payload, ["adoption_drift", "summary"], {}), + keys=ADOPTION_SUMMARY_KEYS, + paths=[REQUIRED_REPORTS["adoption"], REQUIRED_REPORTS[report_key]], + ) + compare_summary_keys( + checks, + key_prefix=f"{report_key}-world-class-ledger-summary", + label=f"{report_key} embeds world-class ledger summary fields", + source_summary=ledger_summary if isinstance(ledger_summary, dict) else {}, + embedded_summary=nested(payload, ["world_class_evidence_ledger", "summary"], {}), + keys=LEDGER_SUMMARY_KEYS, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]], + ) + readiness_expected = { + "ready": ledger_summary.get("ready_to_claim_world_class") if isinstance(ledger_summary, dict) else None, + "decision": ledger_summary.get("decision") if isinstance(ledger_summary, dict) else None, + "pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None, + "accepted_count": ledger_summary.get("accepted_count") if isinstance(ledger_summary, dict) else None, + "source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None, + "source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None, + } + readiness = nested(payload, ["world_class_readiness"], {}) + readiness_actual = {key: readiness.get(key) if isinstance(readiness, dict) else None for key in readiness_expected} + compare_values( + checks, + key=f"{report_key}-world-class-readiness", + label=f"{report_key} derives readiness from the ledger", + expected=readiness_expected, + actual=readiness_actual, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]], + detail="Readiness summaries must be derived from the evidence ledger, not hand-maintained copy.", + ) + + for section in LOCKSTEP_SECTIONS: + compare_values( + checks, + key=f"overview-interpretation-lockstep-{section.replace('_', '-')}", + label=f"Overview and interpretation share {section}", + expected=overview.get(section), + actual=interpretation.get(section), + paths=[REQUIRED_REPORTS["overview"], REQUIRED_REPORTS["interpretation"]], + detail="The first-class interpretation report must stay in lockstep with the canonical overview model.", + ) + + for report_key, expected_html in [ + ("overview", "reports/skill-overview.html"), + ("interpretation", "reports/skill-interpretation.html"), + ]: + contract = report_contract(reports[report_key]) + expected = { + "schema_version": "2.0", + "default_language": "zh-CN", + "layout": "kami-white-audit-v2", + "html_report": expected_html, + "html_exists": True, + } + actual = { + "schema_version": contract.get("schema_version"), + "default_language": contract.get("default_language"), + "layout": contract.get("layout"), + "html_report": contract.get("html_report"), + "html_exists": (skill_dir / expected_html).exists(), + } + compare_values( + checks, + key=f"{report_key}-html-contract", + label=f"{report_key} has a stable HTML contract", + expected=expected, + actual=actual, + paths=[REQUIRED_REPORTS[report_key], expected_html], + detail="Report output paths and language defaults are part of the user-facing contract.", + ) + + if isinstance(ledger_summary, dict): + expected_boundary = { + "world_class_evidence_pending_count": ledger_summary.get("pending_count"), + "public_world_class_ready": ledger_summary.get("ready_to_claim_world_class"), + } + actual_boundary = { + "world_class_evidence_pending_count": coverage_summary.get("world_class_evidence_pending_count") + if isinstance(coverage_summary, dict) + else None, + "public_world_class_ready": coverage_summary.get("public_world_class_ready") + if isinstance(coverage_summary, dict) + else None, + } + compare_values( + checks, + key="coverage-world-class-boundary", + label="Coverage report mirrors world-class evidence boundary", + expected=expected_boundary, + actual=actual_boundary, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["skill_os2_coverage"]], + detail="Blueprint coverage can be locally complete while public world-class evidence remains pending.", + ) + benchmark_boundary = { + "world_class_ledger_pending_count": ledger_summary.get("pending_count"), + "world_class_source_check_count": ledger_summary.get("source_check_count"), + "world_class_source_pass_count": ledger_summary.get("source_pass_count"), + "world_class_source_blocked_count": ledger_summary.get("source_blocked_count"), + "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), + } + actual_benchmark_boundary = { + key: benchmark_summary.get(key) if isinstance(benchmark_summary, dict) else None + for key in benchmark_boundary + } + compare_values( + checks, + key="benchmark-world-class-boundary", + label="Benchmark report mirrors world-class evidence boundary", + expected=benchmark_boundary, + actual=actual_benchmark_boundary, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]], + detail="Benchmark reproducibility must not overstate public claim readiness.", + ) + beta_boundary, actual_beta_boundary = beta_public_claim_split_values( + benchmark, + ledger, + review_studio, + ) + compare_values( + checks, + key="benchmark-beta-public-claim-split", + label="Benchmark separates beta testing from public claims", + expected=beta_boundary, + actual=actual_beta_boundary, + paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS["world_class_ledger"]], + detail=( + "Beta/public testing may proceed with human blind review deferred, but public world-class or superiority claims must remain blocked." + ), + ) + preflight_boundary = { + "pending_count": ledger_summary.get("pending_count"), + "source_check_count": ledger_summary.get("source_check_count"), + "source_pass_count": ledger_summary.get("source_pass_count"), + "source_blocked_count": ledger_summary.get("source_blocked_count"), + "ready_to_claim_world_class": ledger_summary.get("ready_to_claim_world_class"), + "preflight_counts_as_evidence": False, + "credential_value_exposed": False, + } + actual_preflight_boundary = { + key: preflight_summary.get(key) if isinstance(preflight_summary, dict) else None + for key in preflight_boundary + } + compare_values( + checks, + key="preflight-world-class-boundary", + label="Preflight mirrors ledger without accepting evidence", + expected=preflight_boundary, + actual=actual_preflight_boundary, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["world_class_preflight"]], + detail="Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness.", + ) + checks.extend( + build_preflight_artifact_role_handoff_checks( + skill_dir=skill_dir, + world_class_preflight=world_class_preflight, + review_studio=review_studio, + report_paths=REQUIRED_REPORTS, + ) + ) + checks.append( + build_phase_queue_consistency_check( + world_class_preflight=world_class_preflight, + world_class_operator_runbook=world_class_operator_runbook, + review_studio=review_studio, + report_paths=REQUIRED_REPORTS, + ) + ) + + public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False + compare_values( + checks, + key="review-studio-no-overclaim", + label="Review Studio does not overclaim pending world-class evidence", + expected=False if not public_ready else True, + actual=studio_summary.get("decision") in {"pass", "release", "ready", "world-class-ready"} + if isinstance(studio_summary, dict) + else None, + paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["review_studio"]], + detail="When world-class evidence is pending, Review Studio must stay in a review or warning posture.", + ) + claim_surface_paths = scanned_surface_paths(claim_guard) + required_claim_surfaces = [ + "README.md", + "SKILL.md", + "manifest.json", + "agents/interface.yaml", + "evidence/world_class/README.md", + "security/permission_policy.json", + "reports/world_class_evidence_ledger.json", + ] + optional_claim_surfaces = [ + "dist/manifest.json", + "dist/targets/openai/adapter.json", + ] + required_claim_surfaces.extend( + path for path in optional_claim_surfaces if (skill_dir / path).exists() + ) + prohibited_claim_surface_prefixes = [ + "dist/install-simulation/", + "evidence/world_class/submissions/", + ] + json_claim_surface_count = as_int(claim_guard_summary.get("json_claim_surface_count")) + metadata_claim_surface_count = as_int(claim_guard_summary.get("metadata_claim_surface_count")) + package_claim_surface_count = as_int(claim_guard_summary.get("package_claim_surface_count")) + claim_surface_count = as_int(claim_guard_summary.get("claim_surface_count")) + expected_claim_guard_surface = { + "overclaim_guard_active": True, + "violation_count": 0, + "ledger_ready_to_claim_world_class": ledger_summary.get("ready_to_claim_world_class") + if isinstance(ledger_summary, dict) + else None, + "ledger_pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None, + "metadata_covers_json": True, + "package_surface_minimum": True, + "claim_surface_covers_package": True, + "required_surfaces": {path: True for path in required_claim_surfaces}, + "prohibited_surfaces": [], + } + actual_claim_guard_surface = { + "overclaim_guard_active": claim_guard_summary.get("overclaim_guard_active"), + "violation_count": claim_guard_summary.get("violation_count"), + "ledger_ready_to_claim_world_class": claim_guard_summary.get("ledger_ready_to_claim_world_class"), + "ledger_pending_count": claim_guard_summary.get("ledger_pending_count"), + "metadata_covers_json": ( + metadata_claim_surface_count is not None + and json_claim_surface_count is not None + and metadata_claim_surface_count >= json_claim_surface_count + ), + "package_surface_minimum": package_claim_surface_count is not None and package_claim_surface_count >= 5, + "claim_surface_covers_package": ( + claim_surface_count is not None + and package_claim_surface_count is not None + and claim_surface_count >= package_claim_surface_count + ), + "required_surfaces": {path: path in claim_surface_paths for path in required_claim_surfaces}, + "prohibited_surfaces": sorted( + path + for path in claim_surface_paths + if any(path.startswith(prefix) for prefix in prohibited_claim_surface_prefixes) + ), + } + compare_values( + checks, + key="claim-guard-package-runtime-surface", + label="Claim guard covers package and runtime claim surfaces", + expected=expected_claim_guard_surface, + actual=actual_claim_guard_surface, + paths=[ + REQUIRED_REPORTS["world_class_claim_guard"], + "manifest.json", + "agents/interface.yaml", + "dist/manifest.json", + "dist/targets/openai/adapter.json", + "evidence/world_class/README.md", + "security/permission_policy.json", + REQUIRED_REPORTS["world_class_ledger"], + ], + detail="The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted.", + ) + checks.append( + build_world_class_workflow_check( + ledger=ledger, + world_class_plan=world_class_plan, + world_class_intake=world_class_intake, + world_class_submission_review=world_class_submission_review, + world_class_operator_runbook=world_class_operator_runbook, + review_studio=review_studio, + report_paths=REQUIRED_REPORTS, + ) + ) + skill_os2_review = text_reports.get("skill_os2_review", "") + checks.append( + build_skill_os2_review_current_evidence_check( + skill_dir=skill_dir, + skill_os2_review=skill_os2_review, + studio_summary=studio_summary if isinstance(studio_summary, dict) else {}, + trust_summary=trust_summary if isinstance(trust_summary, dict) else {}, + package_summary=package_summary if isinstance(package_summary, dict) else {}, + install_summary=install_summary if isinstance(install_summary, dict) else {}, + benchmark_summary=benchmark_summary if isinstance(benchmark_summary, dict) else {}, + context_stats=context_stats if isinstance(context_stats, dict) else {}, + required_text_reports=REQUIRED_TEXT_REPORTS, + required_reports=REQUIRED_REPORTS, + ) + ) + status_counts: dict[str, int] = {"pass": 0, "warn": 0, "fail": 0} + for check in checks: + status_counts[check["status"]] = status_counts.get(check["status"], 0) + 1 + summary = { + "check_count": len(checks), + "pass_count": status_counts.get("pass", 0), + "warn_count": status_counts.get("warn", 0), + "fail_count": status_counts.get("fail", 0), + "decision": "consistent" if status_counts.get("fail", 0) == 0 else "evidence-drift-detected", + } + return { + "schema_version": "1.0", + "ok": summary["fail_count"] == 0, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": summary, + "status_counts": status_counts, + "checks": checks, + "artifacts": { + "json": "reports/evidence_consistency.json", + "markdown": "reports/evidence_consistency.md", + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render cross-report evidence consistency checks.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/evidence_consistency.json") + parser.add_argument("--output-md", default="reports/evidence_consistency.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_portability_report.py b/scripts/render_portability_report.py index dbd8d23d..54b3db39 100644 --- a/scripts/render_portability_report.py +++ b/scripts/render_portability_report.py @@ -5,6 +5,8 @@ from pathlib import Path import yaml +from skill_ir_paths import find_skill_ir + ROOT = Path(__file__).resolve().parent.parent @@ -18,14 +20,7 @@ def load_yaml(path: Path) -> dict: def find_ir(root: Path) -> tuple[dict, str]: - candidates = [ - root / "reports" / "skill-ir.json", - root / "skill-ir" / "examples" / f"{root.name}.json", - ] - for path in candidates: - if path.exists(): - return load_json(path), str(path.relative_to(root)) - return {}, "missing" + return find_skill_ir(root, root.name, fallback_source="missing") def band_for(score: int) -> str: @@ -129,13 +124,21 @@ def render_markdown(report: dict) -> str: def main() -> None: parser = argparse.ArgumentParser(description="Render a portability score from neutral metadata, contracts, and snapshots.") + parser.add_argument("skill_dir", nargs="?", default=str(ROOT), help="Skill directory to inspect. Defaults to this repo root.") parser.add_argument("--output-json", default="reports/portability_score.json") parser.add_argument("--output-md", default="reports/portability_score.md") args = parser.parse_args() - report = build_report(ROOT) - output_json = ROOT / args.output_json - output_md = ROOT / args.output_md + skill_root = Path(args.skill_dir).resolve() + report = build_report(skill_root) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_root / output_json + if not output_md.is_absolute(): + output_md = skill_root / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") output_md.write_text(render_markdown(report), encoding="utf-8") print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/scripts/render_reference_synthesis.py b/scripts/render_reference_synthesis.py index 67cb11a0..67cdb18f 100644 --- a/scripts/render_reference_synthesis.py +++ b/scripts/render_reference_synthesis.py @@ -5,6 +5,8 @@ import re from pathlib import Path from typing import Any +from reference_synthesis_markdown import render_markdown + try: import yaml except ImportError: # pragma: no cover @@ -507,100 +509,6 @@ def build_summary(skill_dir: Path) -> dict[str, Any]: } -def render_markdown(summary: dict[str, Any]) -> str: - lines = [ - "# Reference Synthesis", - "", - f"Skill: `{summary['skill_name']}`", - f"- Description: {summary['description']}", - f"- Intent confidence: `{summary['intent_confidence']['score']}/100` (`{summary['intent_confidence']['band']}`)", - "", - "## Live GitHub Benchmarks", - "", - ] - if summary["github_benchmarks"]: - for repo in summary["github_benchmarks"]: - lines.extend( - [ - f"### {repo['name']}", - f"- URL: {repo['url']}", - f"- Stars: `{repo['stars']}`", - ] - ) - for item in repo.get("borrow", []): - lines.append(f"- Borrow: {item}") - lines.append("") - else: - lines.append("- No live GitHub benchmarks are attached yet.") - lines.append("") - - lines.extend(["## Curated World-Class Pattern Tracks", ""]) - for track in summary["source_tracks"]: - lines.extend( - [ - f"### {track['name']}", - f"- Type: `{track['source_type']}`", - f"- Evidence mode: `{track['evidence_mode']}`", - f"- Why relevant: {track['why_relevant']}", - f"- Borrow: {track['borrow']}", - f"- Avoid: {track['avoid']}", - "", - ] - ) - - lines.extend(["## Borrow Now", ""]) - for item in summary["synthesis"]["borrow_now"]: - lines.append(f"- {item}") - - lines.extend(["", "## Avoid Now", ""]) - for item in summary["synthesis"]["avoid_now"]: - lines.append(f"- {item}") - - lines.extend(["", "## Pattern Gate", ""]) - pattern_gate = summary["synthesis"]["pattern_gate"] - lines.append(f"- Summary: {pattern_gate['summary']}") - lines.append(f"- Acceptance threshold: `{pattern_gate['threshold']}/4`") - if pattern_gate["accepted"]: - lines.append("- Accepted patterns:") - for item in pattern_gate["accepted"][:5]: - lines.append( - f" - **{item['name']}**: {item['score']}/4 " - f"({', '.join(item['passed'])})" - ) - if pattern_gate["deferred"]: - lines.append("- Deferred patterns:") - for item in pattern_gate["deferred"][:5]: - lines.append( - f" - **{item['name']}**: missing {', '.join(item['missing']) or 'none'}" - ) - - lines.extend(["", "## Default Recommendation", ""]) - lines.append(f"- Summary: {summary['synthesis']['recommendation']['summary']}") - lines.append(f"- Why: {summary['synthesis']['recommendation']['why']}") - lines.append(f"- User decision required: `{summary['synthesis']['recommendation']['user_decision_required']}`") - - lines.extend(["", "## Visibility Mode", ""]) - lines.append(f"- Mode: `{summary['synthesis']['visibility']['mode']}`") - if summary["synthesis"]["visibility"]["reasons"]: - lines.append(f"- Reasons: {', '.join(summary['synthesis']['visibility']['reasons'])}") - lines.append(f"- User note: {summary['synthesis']['visibility']['user_note']}") - lines.append(f"- Reviewer note: {summary['synthesis']['visibility']['reviewer_note']}") - - lines.extend(["", "## Conflict Check", ""]) - if summary["synthesis"]["conflicts"]: - for conflict in summary["synthesis"]["conflicts"]: - lines.append(f"- **{conflict['key']}**: {conflict['summary']}") - else: - lines.append("- No material design conflict detected. Keep the synthesis silent for the user.") - - lines.extend(["", "## Quality Lift Thesis", ""]) - for item in summary["synthesis"]["quality_risers"]: - lines.append(f"- {item}") - - lines.extend(["", "## Decision Prompt", "", summary["synthesis"]["decision_prompt"], ""]) - return "\n".join(lines).strip() + "\n" - - def render_reference_synthesis( skill_dir: Path, output_md: Path | None = None, diff --git a/scripts/render_review_annotations.py b/scripts/render_review_annotations.py index 58dbe681..19bc6cf5 100644 --- a/scripts/render_review_annotations.py +++ b/scripts/render_review_annotations.py @@ -18,11 +18,14 @@ VALID_GATES = { "context-budget", "runtime-matrix", "trust-report", + "python-compat", + "architecture-maintainability", "permission-gates", "permission-runtime", "skill-atlas", "operations-loop", "review-waivers", + "world-class-evidence", "registry-audit", "release-notes", } diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 74879be5..f33ce212 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -2,68 +2,24 @@ import argparse import html import json -import os from pathlib import Path from typing import Any -try: - import yaml -except ImportError: # pragma: no cover - yaml = None - -try: - from trust_check import permission_governance_status as compute_permission_governance_status - from trust_check import script_inventory as trust_script_inventory -except ImportError: # pragma: no cover - compute_permission_governance_status = None - trust_script_inventory = None - -from review_studio_formatting import registry_package_summary, render_kv_grid +from review_studio_actions import build_review_actions, render_review_actions +from review_studio_data import evidence_paths, insight_cards, load_review_data +from review_studio_formatting import registry_package_summary, render_gate_contract_panel, render_kv_grid +from review_studio_gates import add_blockers_from_gate, build_gates, gate_contract, weighted_score from review_studio_layout import render_review_nav, review_studio_css +from review_studio_output_review import render_output_review_section +from review_studio_panels import render_gate_list, render_insights, render_issue_list, render_review_annotations_panel +from review_studio_skillops import render_skillops_section +from review_studio_waivers import render_waiver_candidates_panel +from review_studio_world_class import render_world_class_evidence_entries, render_world_class_intake_checklist ROOT = Path(__file__).resolve().parent.parent -def load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - return payload if isinstance(payload, dict) else {} - - -def load_yaml(path: Path) -> dict[str, Any]: - if not path.exists() or yaml is None: - return {} - payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - return payload if isinstance(payload, dict) else {} - - -def parse_frontmatter(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - if not lines or lines[0].strip() != "---": - return {} - try: - end_index = lines[1:].index("---") + 1 - except ValueError: - return {} - text = "\n".join(lines[1:end_index]) - if yaml is not None: - payload = yaml.safe_load(text) or {} - return payload if isinstance(payload, dict) else {} - data = {} - for line in text.splitlines(): - if ":" in line: - key, value = line.split(":", 1) - data[key.strip()] = value.strip().strip('"') - return data - - def display_path(skill_dir: Path, path: Path) -> str: try: return str(path.resolve().relative_to(skill_dir.resolve())) @@ -74,999 +30,6 @@ def display_path(skill_dir: Path, path: Path) -> str: return str(path.resolve()) -def link_from(output_html: Path, target: Path) -> str: - return os.path.relpath(target.resolve(), output_html.parent.resolve()) - - -def report_link(output_html: Path, skill_dir: Path, rel_path: str) -> str: - return link_from(output_html, skill_dir / rel_path) - - -def find_line(path: Path, patterns: list[str] | None = None) -> int | None: - if not path.exists(): - return None - if not patterns: - return 1 - try: - lines = path.read_text(encoding="utf-8", errors="replace").splitlines() - except OSError: - return 1 - for pattern in patterns: - for index, line in enumerate(lines, start=1): - if pattern in line: - return index - return 1 - - -def source_refs( - skill_dir: Path, - output_html: Path, - specs: list[dict[str, Any]], -) -> list[dict[str, Any]]: - refs: list[dict[str, Any]] = [] - for spec in specs: - rel_path = str(spec.get("path", "")).strip() - if not rel_path: - continue - path = skill_dir / rel_path - exists = path.exists() - line = find_line(path, spec.get("patterns", [])) - refs.append( - { - "path": rel_path, - "label": str(spec.get("label", rel_path)), - "kind": str(spec.get("kind", "source")), - "line": line, - "exists": exists, - "link": link_from(output_html, path) if exists else "", - } - ) - return refs - - -def gate(key: str, label: str, status: str, detail: str, evidence: str, link: str = "") -> dict[str, str]: - return { - "key": key, - "label": label, - "status": status, - "detail": detail, - "evidence": evidence, - "link": link, - } - - -def status_label(status: str) -> str: - return {"pass": "通过", "warn": "关注", "block": "阻断"}.get(status, status) - - -def add_blockers_from_gate(gates: list[dict[str, str]]) -> tuple[list[dict[str, str]], list[dict[str, str]]]: - blockers = [item for item in gates if item["status"] == "block"] - warnings = [item for item in gates if item["status"] == "warn"] - return blockers, warnings - - -def target_maturity(skill_dir: Path, overview: dict[str, Any]) -> str: - manifest = load_json(skill_dir / "manifest.json") - if manifest.get("maturity_tier"): - return str(manifest["maturity_tier"]) - metadata = overview.get("metadata", {}) if isinstance(overview, dict) else {} - if metadata.get("maturity_tier"): - return str(metadata["maturity_tier"]) - return "scaffold" - - -def min_output_cases(maturity: str) -> int: - if maturity in {"library", "governed"}: - return 5 - if maturity == "production": - return 3 - return 1 - - -def fallback_permission_governance(skill_dir: Path) -> dict[str, Any]: - if compute_permission_governance_status is None or trust_script_inventory is None: - return {} - try: - scripts = trust_script_inventory(skill_dir) - return compute_permission_governance_status(skill_dir, scripts) - except Exception: - return {} - - -def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: - overview = data["overview"] - maturity = target_maturity(skill_dir, overview) - gates: list[dict[str, str]] = [] - - intent = data["intent_confidence"] - intent_score = int(intent.get("score", 0) or 0) - intent_status = "pass" if intent.get("gate_passed") or intent_score >= 75 else "warn" - gates.append( - gate( - "intent-canvas", - "意图画布", - intent_status, - f"intent confidence {intent_score}/100; {intent.get('recommended_action', 'review current intent frame')}", - "reports/intent-confidence.json", - report_link(output_html, skill_dir, "reports/intent-confidence.md"), - ) - ) - - route = data["route_scorecard"] - route_summary = route.get("summary", {}) - misroutes = int(route_summary.get("misroute_count", len(route.get("misroutes", []))) or 0) - ambiguous = int(route_summary.get("ambiguous_case_count", len(route.get("ambiguous_cases", []))) or 0) - if not route: - route_status = "warn" - route_detail = "route scorecard is missing; run route-scorecard before release review" - else: - route_status = "block" if misroutes else ("warn" if ambiguous else "pass") - route_detail = f"{route_summary.get('total_cases', 0)} trigger cases; {misroutes} misroutes; {ambiguous} ambiguous" - gates.append( - gate( - "trigger-lab", - "触发实验", - route_status, - route_detail, - "reports/route_scorecard.json", - report_link(output_html, skill_dir, "reports/route_scorecard.md"), - ) - ) - - output = data["output_quality"] - output_execution = data["output_execution"] - output_blind = data["output_blind_review"] - output_review = data["output_review_adjudication"] - output_summary = output.get("summary", {}) - output_execution_summary = output_execution.get("summary", {}) - output_blind_summary = output_blind.get("summary", {}) - output_review_summary = output_review.get("summary", {}) - required_cases = min_output_cases(maturity) - case_count = int(output_summary.get("case_count", 0) or 0) - file_backed = int(output_summary.get("file_backed_case_count", 0) or 0) - near_neighbor = int(output_summary.get("near_neighbor_case_count", 0) or 0) - boundary = int(output_summary.get("boundary_case_count", 0) or 0) - blind_pair_count = int(output_blind_summary.get("pair_count", 0) or 0) - execution_variant_count = int(output_execution_summary.get("variant_run_count", 0) or 0) - execution_command_count = int(output_execution_summary.get("command_executed_count", 0) or 0) - execution_model_count = int(output_execution_summary.get("model_executed_count", 0) or 0) - execution_recorded_count = int(output_execution_summary.get("recorded_fixture_count", 0) or 0) - review_pair_count = int(output_review_summary.get("pair_count", 0) or 0) - review_judgment_count = int(output_review_summary.get("judgment_count", 0) or 0) - review_invalid_count = int(output_review_summary.get("invalid_decision_count", 0) or 0) - blind_missing = maturity in {"production", "library", "governed"} and (not output_blind or blind_pair_count < case_count) - execution_failed = bool(output_execution) and (not output_execution.get("ok", True) or int(output_execution_summary.get("failure_count", 0) or 0) > 0) - review_invalid = bool(output_review) and (not output_review.get("ok", True) or review_invalid_count > 0) - output_blocked = ( - not output.get("ok", False) - or not output_summary.get("gate_pass", False) - or case_count < required_cases - or execution_failed - or review_invalid - ) - output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing - if not output: - output_status = "warn" - output_detail = "output eval scorecard is missing; generate it before production review" - else: - output_status = "block" if output_blocked else ("warn" if output_warn else "pass") - output_detail = ( - f"{case_count}/{required_cases} cases; with-skill {output_summary.get('with_skill_pass_rate', 0)}; " - f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}; " - f"blind A/B {blind_pair_count}" - + ( - f"; exec {execution_variant_count}; command {execution_command_count}; " - f"model {execution_model_count}; recorded {execution_recorded_count}" - if output_execution - else "" - ) - + (f"; reviewed {review_judgment_count}/{review_pair_count}" if output_review else "") - ) - gates.append( - gate( - "output-lab", - "输出实验", - output_status, - output_detail, - "reports/output_quality_scorecard.json", - report_link(output_html, skill_dir, "reports/output_quality_scorecard.md"), - ) - ) - - context = data["context_budget"] - context_stats = context.get("stats", {}) - context_status = "pass" if context.get("ok") else "block" - if context.get("warnings"): - context_status = "warn" if context_status == "pass" else context_status - if not context: - context_status = "warn" - context_detail = ( - f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/" - f"{context_stats.get('context_budget_limit', 'n/a')}; quality density {context_stats.get('quality_density', 'n/a')}" - ) - gates.append( - gate( - "context-budget", - "上下文", - context_status, - context_detail, - "reports/context_budget.json", - report_link(output_html, skill_dir, "reports/context_budget.md"), - ) - ) - - conformance = data["conformance"] - conformance_summary = conformance.get("summary", {}) - fail_count = int(conformance_summary.get("fail_count", 0) or 0) - if not conformance: - conformance_status = "warn" - conformance_detail = "runtime conformance matrix is missing" - else: - conformance_status = "block" if fail_count else "pass" - conformance_detail = f"{conformance_summary.get('pass_count', 0)} / {conformance_summary.get('target_count', 0)} targets pass" - gates.append( - gate( - "runtime-matrix", - "运行矩阵", - conformance_status, - conformance_detail, - "reports/conformance_matrix.json", - report_link(output_html, skill_dir, "reports/conformance_matrix.md"), - ) - ) - - trust = data["trust"] - trust_summary = trust.get("summary", {}) - if not trust: - trust_status = "warn" - trust_detail = "security trust report is missing" - else: - trust_status = "block" if trust.get("failures") else ("warn" if trust.get("warnings") else "pass") - trust_detail = ( - f"{trust_summary.get('secret_findings', 0)} secrets; " - f"{trust_summary.get('script_count', 0)} scripts; " - f"{trust_summary.get('network_script_count', 0)} network-capable scripts; " - f"{trust_summary.get('help_smoke_failed_count', 0)} help smoke failures" - ) - gates.append( - gate( - "trust-report", - "信任报告", - trust_status, - trust_detail, - "reports/security_trust_report.json", - report_link(output_html, skill_dir, "reports/security_trust_report.md"), - ) - ) - - permission_governance = trust.get("permission_governance", {}) if isinstance(trust.get("permission_governance", {}), dict) else {} - if trust and not permission_governance: - permission_governance = fallback_permission_governance(skill_dir) - if not trust: - permission_status = "warn" - permission_detail = "permission governance evidence is missing because trust report is missing" - elif not permission_governance: - permission_status = "warn" - permission_detail = "permission governance evidence is missing from trust report" - else: - required = int(permission_governance.get("required_count", 0) or 0) - approved = int(permission_governance.get("approval_count", 0) or 0) - gaps = ( - int(permission_governance.get("missing_count", 0) or 0) - + int(permission_governance.get("invalid_count", 0) or 0) - + int(permission_governance.get("expired_count", 0) or 0) - ) - if gaps: - permission_status = "block" if maturity == "governed" else "warn" - else: - permission_status = "pass" - required_names = ", ".join(permission_governance.get("required_capabilities", []) or []) or "none" - permission_detail = f"{approved}/{required} permissions approved; gaps {gaps}; required {required_names}" - gates.append( - gate( - "permission-gates", - "权限批准", - permission_status, - permission_detail, - "reports/security_trust_report.json + security/permission_policy.json", - report_link(output_html, skill_dir, "security/permission_policy.md"), - ) - ) - - runtime_permissions = data["runtime_permissions"] - runtime_permissions_summary = runtime_permissions.get("summary", {}) - if not runtime_permissions: - runtime_permission_status = "block" if maturity == "governed" else "warn" - runtime_permission_detail = "runtime permission probe report is missing" - elif runtime_permissions.get("failures"): - runtime_permission_status = "block" - runtime_permission_detail = f"{runtime_permissions_summary.get('failure_count', len(runtime_permissions.get('failures', [])))} runtime permission probe failures" - else: - runtime_permission_status = "pass" - runtime_permission_detail = ( - f"{runtime_permissions_summary.get('pass_count', 0)}/{runtime_permissions_summary.get('target_count', 0)} targets probed; " - f"native {runtime_permissions_summary.get('native_enforcement_count', 0)}; " - f"metadata fallback {runtime_permissions_summary.get('metadata_fallback_count', 0)}; " - f"residual risks {runtime_permissions_summary.get('residual_risk_count', 0)}" - ) - gates.append( - gate( - "permission-runtime", - "权限探针", - runtime_permission_status, - runtime_permission_detail, - "reports/runtime_permission_probes.json", - report_link(output_html, skill_dir, "reports/runtime_permission_probes.md"), - ) - ) - - atlas = data["atlas"] - atlas_summary = atlas.get("summary", {}) - actionable_route_collisions = int( - atlas_summary.get("actionable_route_collision_count", atlas_summary.get("route_collision_count", 0)) or 0 - ) - actionable_owner_gaps = int(atlas_summary.get("actionable_owner_gap_count", atlas_summary.get("owner_gap_count", 0)) or 0) - actionable_stale = int(atlas_summary.get("actionable_stale_count", atlas_summary.get("stale_count", 0)) or 0) - atlas_issues = actionable_route_collisions + actionable_owner_gaps + actionable_stale - if not atlas: - atlas_status = "warn" - atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown" - else: - atlas_status = "warn" if atlas_issues else "pass" - atlas_detail = ( - f"{atlas_summary.get('skill_count', 0)} skills, " - f"{atlas_summary.get('actionable_skill_count', atlas_summary.get('skill_count', 0))} actionable; " - f"{actionable_route_collisions} actionable route collisions; " - f"{actionable_owner_gaps} actionable owner gaps; " - f"{actionable_stale} actionable stale; " - f"{atlas_summary.get('non_actionable_issue_count', 0)} scoped non-actionable issues" - ) - gates.append( - gate( - "skill-atlas", - "组合治理", - atlas_status, - atlas_detail, - "reports/skill_atlas.json", - report_link(output_html, skill_dir, "reports/skill_atlas.html"), - ) - ) - - adoption = data["adoption_drift"] - adoption_summary = adoption.get("summary", {}) - if not adoption: - adoption_status = "warn" - adoption_detail = "adoption drift report is missing; real usage impact is unknown" - elif adoption.get("failures"): - adoption_status = "block" - adoption_detail = f"telemetry privacy or schema failures: {len(adoption.get('failures', []))}" - else: - risk_band = adoption_summary.get("risk_band", "no-data") - adoption_status = "warn" if risk_band in {"no-data", "medium", "high"} else "pass" - adoption_detail = ( - f"{adoption_summary.get('event_count', 0)} metadata events; " - f"adoption {adoption_summary.get('adoption_rate', 0)}; " - f"missed {adoption_summary.get('missed_trigger_count', 0)}; " - f"bad-output {adoption_summary.get('bad_output_count', 0)}; " - f"risk {risk_band}" - ) - gates.append( - gate( - "operations-loop", - "运营回路", - adoption_status, - adoption_detail, - "reports/adoption_drift_report.json", - report_link(output_html, skill_dir, "reports/adoption_drift_report.md"), - ) - ) - - waiver = data["review_waivers"] - waiver_summary = waiver.get("summary", {}) - active_covered = set(waiver_summary.get("covered_gate_keys", []) or []) - prior_blockers = [item for item in gates if item["status"] == "block"] - prior_warnings = [item for item in gates if item["status"] == "warn"] - unwaived_warnings = [item for item in prior_warnings if item["key"] not in active_covered] - if not waiver: - waiver_status = "warn" - waiver_detail = "review waiver ledger is missing; warning acceptance is not auditable" - elif waiver.get("failures"): - waiver_status = "block" - waiver_detail = f"{len(waiver.get('failures', []))} invalid waiver records" - elif prior_blockers: - waiver_status = "block" - waiver_detail = f"{len(prior_blockers)} blocker gates cannot be waived in v0" - elif unwaived_warnings: - waiver_status = "warn" - waiver_detail = ( - f"{waiver_summary.get('active_count', 0)} active waivers; " - f"{len(unwaived_warnings)} warning gates still need reviewer decision" - ) - else: - waiver_status = "pass" - waiver_detail = f"{waiver_summary.get('active_count', 0)} active waivers cover current warnings" - gates.append( - gate( - "review-waivers", - "人工批准", - waiver_status, - waiver_detail, - "reports/review_waivers.json", - report_link(output_html, skill_dir, "reports/review_waivers.md"), - ) - ) - - registry = data["registry"] - install = data["install_simulation"] - if not registry: - if maturity in {"library", "governed"}: - registry_status = "warn" - registry_detail = "registry audit is missing; package metadata is not reviewable" - else: - registry_status = "pass" - registry_detail = "registry audit is optional until team distribution is required" - else: - compatibility = registry.get("package", {}).get("compatibility", {}) - pass_count = sum(1 for status in compatibility.values() if status == "pass") - registry_status = "block" if registry.get("failures") else ("warn" if registry.get("warnings") else "pass") - registry_detail = ( - f"{registry.get('package', {}).get('name', 'package')} " - f"{registry.get('package', {}).get('version', 'n/a')}; " - f"{pass_count}/{len(compatibility)} compatibility entries pass" - ) - if install: - if install.get("failures"): - registry_status = "block" - install_summary = install.get("summary", {}) - registry_detail += ( - f"; install {'pass' if install.get('ok') else 'fail'}" - f" with {install_summary.get('adapter_count', 0)} adapters" - ) - gates.append( - gate( - "registry-audit", - "注册审计", - registry_status, - registry_detail, - "reports/registry_audit.json + reports/install_simulation.json", - report_link(output_html, skill_dir, "reports/registry_audit.md"), - ) - ) - - promotion = data["promotion"] - migration_path = ROOT / "docs" / "migration-v2.md" - if promotion: - promotion_summary = promotion.get("summary", {}) - blocked = int(promotion_summary.get("blocked", 0) or 0) - release_status = "block" if blocked else "pass" - release_detail = f"{promotion_summary.get('promote', 0)} promote; {promotion_summary.get('keep_current', 0)} keep current; {blocked} blocked" - else: - release_status = "warn" - release_detail = "promotion decisions are missing; release notes need reviewer confirmation" - upgrade = data["upgrade_check"] - if upgrade: - upgrade_summary = upgrade.get("summary", {}) - if upgrade.get("failures"): - release_status = "block" - elif upgrade.get("warnings") and release_status == "pass": - release_status = "warn" - release_detail += ( - f"; upgrade {upgrade_summary.get('declared_bump', 'n/a')}" - f" declared / {upgrade_summary.get('recommended_bump', 'n/a')} recommended" - ) - gates.append( - gate( - "release-notes", - "发布路线", - release_status, - release_detail, - "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", - report_link(output_html, skill_dir, "reports/promotion_decisions.md") if promotion else str(migration_path), - ) - ) - - return gates - - -def weighted_score(gates: list[dict[str, str]]) -> int: - weights = { - "trigger-lab": 15, - "output-lab": 20, - "context-budget": 10, - "runtime-matrix": 10, - "trust-report": 10, - "permission-gates": 10, - "permission-runtime": 10, - "skill-atlas": 10, - "operations-loop": 10, - "review-waivers": 10, - "registry-audit": 10, - "release-notes": 10, - "intent-canvas": 10, - } - earned = 0.0 - total = 0.0 - for item in gates: - weight = weights.get(item["key"], 5) - total += weight - if item["status"] == "pass": - earned += weight - elif item["status"] == "warn": - earned += weight * 0.6 - return int(round(earned / total * 100)) if total else 0 - - -def evidence_paths(skill_dir: Path) -> dict[str, str]: - rels = { - "skill_overview": "reports/skill-overview.html", - "review_viewer": "reports/review-viewer.html", - "output_eval": "reports/output_quality_scorecard.md", - "output_execution": "reports/output_execution_runs.md", - "output_blind_review": "reports/output_blind_review_pack.md", - "output_review_adjudication": "reports/output_review_adjudication.md", - "runtime_conformance": "reports/conformance_matrix.md", - "trust_report": "reports/security_trust_report.md", - "permission_policy": "security/permission_policy.md", - "runtime_permissions": "reports/runtime_permission_probes.md", - "skill_atlas": "reports/skill_atlas.html", - "compiled_targets": "reports/compiled_targets.md", - "adoption_drift": "reports/adoption_drift_report.md", - "review_waivers": "reports/review_waivers.md", - "review_annotations": "reports/review_annotations.md", - "registry_audit": "reports/registry_audit.md", - "package_verification": "reports/package_verification.md", - "install_simulation": "reports/install_simulation.md", - "upgrade_check": "reports/upgrade_check.md", - "migration": "docs/migration-v2.md", - "skill_ir": "reports/skill-ir.json", - } - return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()} - - -def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: - reports = skill_dir / "reports" - return { - "overview": load_json(reports / "skill-overview.json"), - "intent_confidence": load_json(reports / "intent-confidence.json"), - "intent_dialogue": load_json(reports / "intent-dialogue.json"), - "route_scorecard": load_json(reports / "route_scorecard.json"), - "output_quality": load_json(reports / "output_quality_scorecard.json"), - "output_execution": load_json(reports / "output_execution_runs.json"), - "output_blind_review": load_json(reports / "output_blind_review_pack.json"), - "output_review_adjudication": load_json(reports / "output_review_adjudication.json"), - "compiled_targets": load_json(reports / "compiled_targets.json"), - "conformance": load_json(reports / "conformance_matrix.json"), - "runtime_permissions": load_json(reports / "runtime_permission_probes.json"), - "trust": load_json(reports / "security_trust_report.json"), - "context_budget": load_json(reports / "context_budget.json"), - "promotion": load_json(reports / "promotion_decisions.json"), - "atlas": load_json(reports / "skill_atlas.json"), - "adoption_drift": load_json(reports / "adoption_drift_report.json"), - "review_waivers": load_json(reports / "review_waivers.json"), - "review_annotations": load_json(reports / "review_annotations.json"), - "registry": load_json(reports / "registry_audit.json"), - "package_verification": load_json(reports / "package_verification.json"), - "install_simulation": load_json(reports / "install_simulation.json"), - "upgrade_check": load_json(reports / "upgrade_check.json"), - "manifest": load_json(skill_dir / "manifest.json"), - "frontmatter": parse_frontmatter(skill_dir / "SKILL.md"), - "interface": load_yaml(skill_dir / "agents" / "interface.yaml"), - } - - -def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: - overview = data["overview"] - output = data["output_quality"].get("summary", {}) - output_execution = data["output_execution"].get("summary", {}) - output_blind = data["output_blind_review"].get("summary", {}) - output_review = data["output_review_adjudication"].get("summary", {}) - compiled = data["compiled_targets"].get("summary", {}) - conformance = data["conformance"].get("summary", {}) - runtime_permissions = data["runtime_permissions"].get("summary", {}) - trust = data["trust"].get("summary", {}) - atlas = data["atlas"].get("summary", {}) - adoption = data["adoption_drift"].get("summary", {}) - waivers = data["review_waivers"].get("summary", {}) - annotations = data["review_annotations"].get("summary", {}) - registry = data["registry"].get("package", {}) - package_verification = data["package_verification"].get("summary", {}) - install_simulation = data["install_simulation"].get("summary", {}) - upgrade = data["upgrade_check"].get("summary", {}) - cards = [ - { - "label": "Skill IR", - "value": str(overview.get("skill_ir", {}).get("schema_version", "missing")), - "detail": f"{overview.get('skill_ir', {}).get('target_count', 0)} targets in platform-neutral contract", - }, - { - "label": "Compiler", - "value": f"{compiled.get('pass_count', 0)}/{compiled.get('target_count', 0)}", - "detail": "target contracts compiled from Skill IR", - }, - { - "label": "Output Delta", - "value": str(output.get("delta", "n/a")), - "detail": f"{output.get('case_count', 0)} cases; {output.get('file_backed_case_count', 0)} file-backed", - }, - { - "label": "Exec Runs", - "value": str(output_execution.get("variant_run_count", 0)), - "detail": ( - f"command {output_execution.get('command_executed_count', 0)}; " - f"model {output_execution.get('model_executed_count', 0)}; " - f"recorded {output_execution.get('recorded_fixture_count', 0)}" - ), - }, - { - "label": "Blind A/B", - "value": str(output_blind.get("pair_count", 0)), - "detail": "review pairs hide baseline vs with-skill labels", - }, - { - "label": "Review A/B", - "value": f"{output_review.get('judgment_count', 0)}/{output_review.get('pair_count', 0)}", - "detail": f"adjudication decisions; pending {output_review.get('pending_count', 0)}", - }, - { - "label": "Runtime", - "value": f"{conformance.get('pass_count', 0)}/{conformance.get('target_count', 0)}", - "detail": "target conformance pass rate", - }, - { - "label": "Perm Probe", - "value": f"{runtime_permissions.get('metadata_fallback_count', 0)}/{runtime_permissions.get('target_count', 0)}", - "detail": f"{runtime_permissions.get('native_enforcement_count', 0)} native enforcement targets", - }, - { - "label": "Trust", - "value": str(trust.get("secret_findings", 0)), - "detail": f"{trust.get('script_count', 0)} scripts scanned; secrets found", - }, - { - "label": "Atlas", - "value": str(atlas.get("route_collision_count", 0)), - "detail": f"{atlas.get('skill_count', 0)} scanned skills; route collisions", - }, - { - "label": "Drift", - "value": str(adoption.get("risk_band", "n/a")), - "detail": f"{adoption.get('event_count', 0)} metadata events; {adoption.get('missed_trigger_count', 0)} missed triggers", - }, - { - "label": "Waivers", - "value": str(waivers.get("active_count", 0)), - "detail": f"{waivers.get('covered_gate_count', 0)} gates covered; human risk decisions", - }, - { - "label": "Notes", - "value": f"{annotations.get('open_count', 0)}/{annotations.get('annotation_count', 0)}", - "detail": f"{annotations.get('open_blocker_count', 0)} open blocker annotations", - }, - { - "label": "Registry", - "value": str(registry.get("version", "n/a")), - "detail": f"{len(registry.get('targets', []))} targets; {registry.get('license', 'no license')} license", - }, - { - "label": "Archive", - "value": "pass" if data["package_verification"].get("ok") else "n/a", - "detail": f"{package_verification.get('archive_entry_count', 0)} zip entries; package verification", - }, - { - "label": "Install", - "value": "pass" if data["install_simulation"].get("ok") else "n/a", - "detail": f"{install_simulation.get('adapter_count', 0)} adapters readable; local install simulation", - }, - { - "label": "Upgrade", - "value": str(upgrade.get("recommended_bump", "n/a")), - "detail": f"declared {upgrade.get('declared_bump', 'n/a')}; {upgrade.get('breaking_change_count', 0)} breaking changes", - }, - ] - return cards - - -def render_gate_list(gates: list[dict[str, str]]) -> str: - items = [] - for item in gates: - link_html = f"证据" if item.get("link") else "" - items.append( - "
    " - f"
    {html.escape(status_label(item['status']))}

    {html.escape(item['label'])}

    " - f"

    {html.escape(item['detail'])}

    " - f"
    {html.escape(item['evidence'])} {link_html}
    " - "
    " - ) - return "".join(items) - - -def render_insights(cards: list[dict[str, str]]) -> str: - return "".join( - ( - "
    " - f"{html.escape(item['label'])}" - f"{html.escape(item['value'])}" - f"

    {html.escape(item['detail'])}

    " - "
    " - ) - for item in cards - ) - - -def render_issue_list(title: str, items: list[dict[str, str]]) -> str: - if not items: - return f"

    {html.escape(title)}

    无。

    " - body = "".join( - ( - "
  • " - f"{html.escape(item['label'])}" - f"{html.escape(item['detail'])}" - "
    • " - ) - for item in items - ) - return f"

    {html.escape(title)}

      {body}
    " - - -ACTION_GUIDANCE: dict[str, dict[str, str]] = { - "intent-canvas": { - "summary": "收紧真实任务、输入、输出、排除项和成功标准。", - "why": "低 intent confidence 会让后续 Skill IR、输出评测和 Review Studio 结论建立在模糊意图上。", - "source_fix": "reports/intent-dialogue.md + reports/intent-confidence.md", - "source_paths": [ - {"path": "reports/intent-dialogue.md", "label": "intent dialogue", "kind": "report", "patterns": ["# Intent"]}, - {"path": "reports/intent-confidence.md", "label": "intent confidence", "kind": "report", "patterns": ["# Intent"]}, - ], - "verification": "python3 scripts/yao.py intent-confidence .", - }, - "trigger-lab": { - "summary": "修正 route scorecard 中的误触发、漏触发或 ambiguous case。", - "why": "触发错误会让正确 Skill 失活，或让相邻 Skill 被错误调用。", - "source_fix": "SKILL.md frontmatter description + evals/*/trigger_cases.json", - "source_paths": [ - {"path": "SKILL.md", "label": "frontmatter description", "kind": "source", "patterns": ["description:"]}, - {"path": "evals/trigger_cases.json", "label": "trigger eval cases", "kind": "eval", "patterns": ["should_trigger"]}, - {"path": "reports/route_scorecard.md", "label": "route scorecard", "kind": "report", "patterns": ["# Route"]}, - ], - "verification": "python3 scripts/build_confusion_matrix.py", - }, - "output-lab": { - "summary": "补足 output eval 的 case 数、file-backed case、near-neighbor case 和 boundary case。", - "why": "没有输出质量证据时，Skill 只能证明会触发，不能证明输出真的更好。", - "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md", - "source_paths": [ - {"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ["case_id"]}, - {"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]}, - {"path": "reports/output_execution_runs.md", "label": "output execution runs", "kind": "report", "patterns": ["# Output Execution"]}, - {"path": "reports/output_blind_review_pack.md", "label": "blind A/B review pack", "kind": "report", "patterns": ["# Output Blind"]}, - {"path": "reports/output_review_adjudication.md", "label": "review adjudication", "kind": "report", "patterns": ["# Output Review"]}, - ], - "verification": "python3 scripts/run_output_execution.py", - }, - "context-budget": { - "summary": "压缩入口与高成本 references，保留最小可路由上下文。", - "why": "上下文成本过高会降低加载稳定性，并挤压用户任务材料的预算。", - "source_fix": "SKILL.md + references/", - "source_paths": [ - {"path": "SKILL.md", "label": "entrypoint", "kind": "source", "patterns": ["# Yao Meta Skill"]}, - {"path": "reports/context_budget.md", "label": "context budget", "kind": "report", "patterns": ["# Context"]}, - ], - "verification": "python3 scripts/render_context_reports.py", - }, - "runtime-matrix": { - "summary": "修复目标端结构、metadata、相对路径、fallback 或 adapter target 声明。", - "why": "runtime conformance 失败意味着包可能被目标客户端错误加载或静默降级。", - "source_fix": "agents/interface.yaml + reports/conformance_matrix.md", - "source_paths": [ - {"path": "agents/interface.yaml", "label": "portable interface", "kind": "source", "patterns": ["adapter_targets"]}, - {"path": "reports/conformance_matrix.md", "label": "conformance matrix", "kind": "report", "patterns": ["# Runtime"]}, - ], - "verification": "python3 scripts/run_conformance_suite.py .", - }, - "trust-report": { - "summary": "处理脚本 help surface、依赖 pin、network policy、secret 和权限声明。", - "why": "团队分发时，脚本和依赖是主要供应链风险面，warning 必须有明确处置。", - "source_fix": "reports/security_trust_report.md + security/*.md + scripts/", - "source_paths": [ - {"path": "reports/security_trust_report.md", "label": "trust report", "kind": "report", "patterns": ["# Security"]}, - {"path": "security/script_policy.md", "label": "script policy", "kind": "policy", "patterns": ["# Script"]}, - {"path": "security/network_policy.md", "label": "network policy", "kind": "policy", "patterns": ["# Network"]}, - ], - "verification": "python3 scripts/trust_check.py .", - }, - "permission-gates": { - "summary": "补齐高权限能力的 reviewer、scope、reason、expires_at 和目标端 enforcement 说明。", - "why": "权限契约只有在批准人、有效期和目标端处置方式明确时，才能支撑 governed release。", - "source_fix": "security/permission_policy.json + security/permission_policy.md", - "source_paths": [ - {"path": "security/permission_policy.json", "label": "permission approvals", "kind": "policy", "patterns": ["approved"]}, - {"path": "security/permission_policy.md", "label": "permission method", "kind": "policy", "patterns": ["# Permission"]}, - ], - "verification": "python3 scripts/trust_check.py .", - }, - "permission-runtime": { - "summary": "生成并修复目标包的 runtime permission probe 报告。", - "why": "目标端即使只能提供 metadata fallback，也必须明确 native enforcement 缺口、表示位置和 operator note。", - "source_fix": "dist/targets/*/adapter.json + reports/runtime_permission_probes.md", - "source_paths": [ - {"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]}, - {"path": "dist/targets/openai/adapter.json", "label": "OpenAI adapter", "kind": "package", "patterns": ["target_permission_contract"]}, - {"path": "dist/targets/claude/adapter.json", "label": "Claude adapter", "kind": "package", "patterns": ["target_permission_contract"]}, - {"path": "dist/targets/generic/adapter.json", "label": "generic adapter", "kind": "package", "patterns": ["target_permission_contract"]}, - ], - "verification": "python3 scripts/probe_runtime_permissions.py . --package-dir dist", - }, - "skill-atlas": { - "summary": "处理 portfolio 里的路由冲突、owner 缺口、stale skill 和重复能力。", - "why": "单个 Skill 质量很高仍可能在团队 skill library 中被相邻 Skill 冲突削弱。", - "source_fix": "reports/skill_atlas.html + skill_atlas/catalog.json", - "source_paths": [ - {"path": "skill_atlas/catalog.json", "label": "skill atlas catalog", "kind": "atlas", "patterns": ["summary"]}, - {"path": "skill_atlas/policy.json", "label": "atlas scope policy", "kind": "policy", "patterns": ["scope"]}, - {"path": "reports/skill_atlas.html", "label": "skill atlas report", "kind": "report", "patterns": ["Skill Atlas"]}, - ], - "verification": "python3 scripts/build_skill_atlas.py --workspace-root .", - }, - "operations-loop": { - "summary": "记录 metadata-only 使用事件，或明确当前 release 缺少真实使用信号。", - "why": "没有运营回路时，reviewer 无法判断采用率、误触发、坏输出和 review overdue 的真实影响。", - "source_fix": "reports/adoption_drift_report.md", - "source_paths": [ - {"path": "reports/adoption_drift_report.md", "label": "adoption drift report", "kind": "report", "patterns": ["# Adoption"]}, - {"path": "references/telemetry-drift-method.md", "label": "telemetry method", "kind": "method", "patterns": ["# Telemetry"]}, - ], - "verification": "python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted", - }, - "review-waivers": { - "summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", - "why": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", - "source_fix": "reports/review_waivers.md", - "source_paths": [ - {"path": "reports/review_waivers.md", "label": "waiver ledger", "kind": "report", "patterns": ["# Review"]}, - {"path": "references/review-waiver-method.md", "label": "waiver method", "kind": "method", "patterns": ["# Review"]}, - ], - "verification": "python3 scripts/render_review_waivers.py .", - }, - "registry-audit": { - "summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。", - "why": "分发元数据不完整时，团队无法安全安装、升级或追溯包体来源。", - "source_fix": "registry/ + reports/registry_audit.md", - "source_paths": [ - {"path": "registry/packages/yao-meta-skill.json", "label": "registry package", "kind": "registry", "patterns": ["version"]}, - {"path": "reports/registry_audit.md", "label": "registry audit", "kind": "report", "patterns": ["# Registry"]}, - {"path": "reports/install_simulation.md", "label": "install simulation", "kind": "report", "patterns": ["# Install"]}, - ], - "verification": "python3 scripts/registry_audit.py .", - }, - "release-notes": { - "summary": "确认 promotion、upgrade diff、breaking changes、migration guide 和 known limitations。", - "why": "发布说明不完整会让使用者无法判断升级风险和迁移动作。", - "source_fix": "reports/upgrade_check.md + docs/migration-v2.md", - "source_paths": [ - {"path": "reports/upgrade_check.md", "label": "upgrade check", "kind": "report", "patterns": ["# Upgrade"]}, - {"path": "docs/migration-v2.md", "label": "migration guide", "kind": "docs", "patterns": ["# Migration"]}, - {"path": "reports/promotion_decisions.md", "label": "promotion decisions", "kind": "report", "patterns": ["# Promotion"]}, - ], - "verification": "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json", - }, -} - - -def build_review_actions(gates: list[dict[str, str]], skill_dir: Path, output_html: Path) -> list[dict[str, Any]]: - actions: list[dict[str, Any]] = [] - for gate_item in gates: - if gate_item["status"] == "pass": - continue - guidance = ACTION_GUIDANCE.get( - gate_item["key"], - { - "summary": "打开证据报告，修复当前 gate 暴露的问题。", - "why": "该 gate 未通过，release reviewer 需要明确处置动作。", - "source_fix": gate_item.get("evidence", ""), - "source_paths": [], - "verification": "python3 scripts/render_review_studio.py .", - }, - ) - refs = source_refs(skill_dir, output_html, guidance.get("source_paths", [])) - actions.append( - { - "gate_key": gate_item["key"], - "label": gate_item["label"], - "status": gate_item["status"], - "priority": "blocker" if gate_item["status"] == "block" else "warning", - "summary": guidance["summary"], - "why": guidance["why"], - "source_fix": guidance["source_fix"], - "source_refs": refs, - "evidence": gate_item.get("evidence", ""), - "evidence_link": gate_item.get("link", ""), - "verification_command": guidance["verification"], - } - ) - return actions - - -def render_action_source_refs(refs: list[dict[str, Any]]) -> str: - if not refs: - return "

    暂无结构化 source refs；请先打开证据报告。

    " - items = [] - for ref in refs: - line_suffix = f":{ref['line']}" if ref.get("line") else "" - label = f"{ref['path']}{line_suffix}" - if ref.get("exists") and ref.get("link"): - path_html = f"{html.escape(label)}" - else: - path_html = f"{html.escape(label)} · missing" - items.append( - "
  • " - f"{path_html}" - f"{html.escape(ref.get('label', 'source'))} · {html.escape(ref.get('kind', 'source'))}" - "
    • " - ) - return "
      " + "".join(items) + "
    " - - -def render_review_actions(actions: list[dict[str, Any]]) -> str: - if not actions: - return "

    当前没有 blocker 或 warning。保持现有证据链即可。

    " - cards = [] - for item in actions: - link_html = f"打开证据" if item.get("evidence_link") else "" - source_refs_html = render_action_source_refs(item.get("source_refs", [])) - cards.append( - "
    " - f"
    {html.escape(status_label(item['status']))}

    {html.escape(item['label'])}

    " - f"

    {html.escape(item['summary'])}

    " - f"{html.escape(item['why'])}" - f"
    修复位置
    {html.escape(item['source_fix'])}
    " - f"
    验证命令
    {html.escape(item['verification_command'])}
    " - f"{source_refs_html}" - f"
    {html.escape(item['evidence'])} {link_html}
    " - "
    " - ) - return "".join(cards) - - -def render_review_annotations_panel(annotations_report: dict[str, Any]) -> str: - annotations = annotations_report.get("annotations", []) if isinstance(annotations_report, dict) else [] - if not annotations: - return "

    当前没有 reviewer 批注。

    " - cards = [] - for item in annotations: - line_suffix = f":{item['line']}" if item.get("line") else "" - target_label = f"{item.get('target_path', '')}{line_suffix}" - meta = " · ".join( - part - for part in [ - str(item.get("gate_key", "")), - str(item.get("reviewer", "")), - str(item.get("created_at", "")), - ] - if part - ) - cards.append( - "
    " - f"
    {html.escape(str(item.get('severity', 'note')))} · {html.escape(str(item.get('status', 'open')))}" - f"

    {html.escape(str(item.get('id', 'annotation')))}

    " - f"

    {html.escape(str(item.get('body', '')))}

    " - f"
    位置
    {html.escape(target_label)}
    " - f"
    Gate
    {html.escape(str(item.get('gate_key', '')))}
    " - f"
    建议
    {html.escape(str(item.get('suggested_action', '') or '无'))}
    " - f"{html.escape(meta)}" - f"
    {html.escape(str(item.get('source_excerpt', '')))}
    " - "
    " - ) - return "".join(cards) - - def render_html(report: dict[str, Any]) -> str: summary = report["summary"] gates = report["gates"] @@ -1081,6 +44,7 @@ def render_html(report: dict[str, Any]) -> str: description = overview.get("description") or frontmatter.get("description", "") nav_html = render_review_nav() gates_html = render_gate_list(gates) + gate_contract_panel = render_gate_contract_panel(report.get("gate_contract", {})) metrics_html = render_insights(insights) blockers_html = render_issue_list("阻断事项", blockers) warnings_html = render_issue_list("关注事项", warnings) @@ -1090,13 +54,25 @@ def render_html(report: dict[str, Any]) -> str: output_execution_summary = report["data"]["output_execution"].get("summary", {}) output_blind_summary = report["data"]["output_blind_review"].get("summary", {}) output_review_summary = report["data"]["output_review_adjudication"].get("summary", {}) + benchmark_report = report["data"].get("benchmark_reproducibility", {}) + benchmark_summary = benchmark_report.get("summary", {}) + public_claim = benchmark_report.get("public_claim", {}) if isinstance(benchmark_report.get("public_claim", {}), dict) else {} + blueprint_summary = report["data"]["skill_os2_coverage"].get("summary", {}) conformance_summary = report["data"]["conformance"].get("summary", {}) compiled_summary = report["data"]["compiled_targets"].get("summary", {}) trust_summary = report["data"]["trust"].get("summary", {}) + python_compat_summary = report["data"]["python_compatibility"].get("summary", {}) runtime_permissions_summary = report["data"]["runtime_permissions"].get("summary", {}) atlas_summary = report["data"]["atlas"].get("summary", {}) adoption_summary = report["data"]["adoption_drift"].get("summary", {}) waiver_summary = report["data"]["review_waivers"].get("summary", {}) + world_class_ledger = report["data"].get("world_class_evidence_ledger", {}) + world_class_summary = world_class_ledger.get("summary", {}) + world_class_intake = report["data"].get("world_class_evidence_intake", {}) + world_class_intake_summary = world_class_intake.get("summary", {}) + world_class_claim_summary = report["data"].get("world_class_claim_guard", {}).get("summary", {}) + world_class_entries_html = render_world_class_evidence_entries(world_class_ledger) + world_class_intake_checklist_html = render_world_class_intake_checklist(world_class_intake) annotation_summary = report["data"]["review_annotations"].get("summary", {}) annotation_caption = ( f"{annotation_summary.get('annotation_count', 0)} 条批注；" @@ -1105,6 +81,7 @@ def render_html(report: dict[str, Any]) -> str: ) registry_package = report["data"]["registry"].get("package", {}) package_summary = report["data"]["package_verification"].get("summary", {}) + install_summary = report["data"]["install_simulation"].get("summary", {}) atlas_panel = render_kv_grid( atlas_summary, [ @@ -1113,6 +90,9 @@ def render_html(report: dict[str, Any]) -> str: "actionable_route_collision_count", "actionable_owner_gap_count", "actionable_stale_count", + "telemetry_report_count", + "actionable_drift_signal_count", + "drift_signal_count", "non_actionable_issue_count", ], "skill atlas summary missing", @@ -1141,9 +121,66 @@ def render_html(report: dict[str, Any]) -> str: ) review_panel = render_kv_grid( output_review_summary, - ["pair_count", "judgment_count", "pending_count", "agreement_count", "disagreement_count", "invalid_decision_count"], + [ + "pair_count", + "judgment_count", + "pending_count", + "agreement_count", + "disagreement_count", + "invalid_decision_count", + "answer_revealed_count", + "pending_answer_hidden_count", + "reviewer_checklist_count", + "reviewer_checklist_pending_count", + ], "review adjudication report missing", ) + public_claim_panel = render_kv_grid( + benchmark_summary, + [ + "reproducibility_ready", + "release_lock_ready", + "public_claim_ready", + "public_claim_blocker_count", + "provider_evidence_complete", + "human_review_complete", + "world_class_ready", + ], + "benchmark reproducibility report missing", + ) + public_claim_blocker_rows = public_claim.get("blockers", []) + if public_claim_blocker_rows: + public_claim_blockers_html = ( + "
      " + + "".join( + f"
    • 阻断{html.escape(str(item))}
      • " + for item in public_claim_blocker_rows + ) + + "
    " + ) + else: + public_claim_blockers_html = "

    无公开声明阻断。

    " + output_review_section_html = render_output_review_section(report["data"].get("output_review_adjudication", {})) + blueprint_panel = render_kv_grid( + blueprint_summary, + [ + "item_count", + "module_count", + "recommended_pr_count", + "pass_count", + "warn_count", + "missing_count", + "extension_track_count", + "extension_covered_count", + "extension_partial_count", + "extension_planned_count", + "adaptive_extension_ready", + "local_blueprint_ready", + "public_world_class_ready", + "world_class_evidence_pending_count", + ], + "Skill OS 2.0 blueprint coverage missing", + ) conformance_panel = render_kv_grid( conformance_summary, ["target_count", "pass_count", "fail_count", "warning_count", "failure_count"], @@ -1159,9 +196,17 @@ def render_html(report: dict[str, Any]) -> str: ["secret_findings", "script_count", "network_script_count", "help_smoke_failed_count", "package_sha256"], "security trust report missing", ) + python_compat_panel = render_kv_grid( + python_compat_summary, + ["target_python", "file_count", "issue_count", "syntax_error_count", "fstring_311_violation_count"], + "python compatibility report missing", + ) runtime_boundary_panel = render_kv_grid( runtime_permissions_summary, - ["target_count", "pass_count", "native_enforcement_count", "metadata_fallback_count", "residual_risk_count", "failure_count"], + [ + "target_count", "pass_count", "native_enforcement_count", "metadata_fallback_count", + "installer_enforcement_pass_count", "installer_permission_failure_count", "residual_risk_count", "failure_count", + ], "runtime permission probe summary missing", ) adoption_panel = render_kv_grid( @@ -1169,11 +214,52 @@ def render_html(report: dict[str, Any]) -> str: ["event_count", "adoption_rate", "missed_trigger_count", "bad_output_count", "risk_band"], "no adoption drift summary", ) + skillops_section_html = render_skillops_section(report["data"]) waiver_panel = render_kv_grid( waiver_summary, ["waiver_count", "active_count", "expired_count", "invalid_count", "covered_gate_count"], "no review waiver summary", ) + waiver_candidates_panel = render_waiver_candidates_panel(report["data"].get("review_waivers", {})) + world_class_panel = render_kv_grid( + world_class_summary, + [ + "ledger_entry_count", + "accepted_count", + "pending_count", + "human_pending_count", + "external_pending_count", + "overclaim_guard_active", + "ready_to_claim_world_class", + ], + "world-class evidence ledger missing", + ) + world_class_intake_panel = render_kv_grid( + world_class_intake_summary, + [ + "schema_present", + "template_count", + "template_pass_count", + "submission_count", + "valid_submission_count", + "invalid_submission_count", + "ready_for_external_collection", + "ready_for_ledger_review", + "ready_to_claim_world_class", + ], + "world-class evidence intake missing", + ) + world_class_claim_panel = render_kv_grid( + world_class_claim_summary, + [ + "ledger_ready_to_claim_world_class", + "ledger_pending_count", + "claim_surface_count", + "violation_count", + "overclaim_guard_active", + ], + "world-class claim guard missing", + ) registry_panel = render_kv_grid( registry_package_summary(registry_package), [ @@ -1194,6 +280,23 @@ def render_html(report: dict[str, Any]) -> str: ["target_count", "adapter_count", "archive_present", "archive_entry_count", "failure_count", "warning_count", "archive_sha256"], "package verification missing", ) + install_panel = render_kv_grid( + install_summary, + [ + "archive_extracted", + "entrypoint_loaded", + "manifest_loaded", + "interface_loaded", + "adapter_count", + "installer_permission_enforced_count", + "installer_permission_failure_count", + "permission_target_count", + "permission_capability_count", + "failure_count", + "warning_count", + ], + "install simulation missing", + ) evidence_html = "".join( f"
  • {html.escape(key)}{html.escape(value)}
    • " for key, value in report["evidence_paths"].items() @@ -1232,6 +335,11 @@ def render_html(report: dict[str, Any]) -> str:
    {gates_html}
    +
    +

    闸门契约

    +
    {gate_contract_panel}
    +
    +
    {blockers_html} {warnings_html} @@ -1274,13 +382,14 @@ def render_html(report: dict[str, Any]) -> str:

    审定报告

    {review_panel}
    + {output_review_section_html}

    发布标准

    Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。

    人工结论

    没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。

    -

    评审方式

    先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

    +

    评审方式

    先打开 reports/output_review_kit.html 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

    运行方式

    reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

    @@ -1299,6 +408,11 @@ def render_html(report: dict[str, Any]) -> str:

    安全边界

    高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

    +
    +

    Python 兼容

    {python_compat_panel}
    +

    解释器边界

    CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。

    +
    +

    权限批准

    {html.escape(gate_details.get('permission-gates', 'permission governance missing'))}

    批准策略

    高权限能力需要 reviewer、scope、reason、expires_at 和 openai/claude/generic 目标端 enforcement 说明。

    @@ -1319,9 +433,54 @@ def render_html(report: dict[str, Any]) -> str:

    漂移信号

    {adoption_panel}
    -
    -

    人工批准

    {html.escape(gate_details.get('review-waivers', 'review waiver ledger missing'))}

    -

    批准台账

    {waiver_panel}
    + {skillops_section_html} + +
    +

    人工批准

    +

    warning 可以被有边界地接受，但必须写入 reviewer、理由、范围和到期时间；blocker 与 world-class 完成证据不能通过 waiver 变成通过。

    +
    +

    批准概况

    {html.escape(gate_details.get('review-waivers', 'review waiver ledger missing'))}

    +

    批准台账

    {waiver_panel}
    +
    +

    批准候选

    + {waiver_candidates_panel} +
    + +
    +

    世界证据

    +

    这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

    + {world_class_entries_html} +
    + +
    +

    提交清单

    +

    每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界；只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后，才进入 ledger review。

    + {world_class_intake_checklist_html} +
    + +
    +

    蓝图覆盖

    {blueprint_panel}
    +

    覆盖边界

    蓝图覆盖只证明 2.0 模块、建议 PR、脚本、报告和测试在本地闭环；public world-class 仍以 world-class evidence ledger 的真人和外部证据为准。

    +
    + +
    +

    公开声明

    {public_claim_panel}
    +

    声明阻断

    {public_claim_blockers_html}
    +
    + +
    +

    世界证据

    {html.escape(gate_details.get('world-class-evidence', 'world-class evidence ledger missing'))}

    +

    证据台账

    {world_class_panel}
    +
    + +
    +

    证据入口

    {world_class_intake_panel}
    +

    入口边界

    intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。

    +
    + +
    +

    声明守卫

    {world_class_claim_panel}
    +

    声明边界

    claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

    @@ -1333,6 +492,11 @@ def render_html(report: dict[str, Any]) -> str:

    发布路线

    {html.escape(gate_details.get('release-notes', 'release notes missing'))}

    包体验证

    {package_panel}
    + +
    +

    安装模拟

    {install_panel}
    +

    权限覆盖

    安装模拟会读取解压后的 target adapter，并按 declared_capabilities 核对 security/permission_policy.json 中的有效批准、证据、过期时间和目标端 enforcement 说明。

    +
    @@ -1347,8 +511,19 @@ def render_review_studio(skill_dir: Path, output_html: Path | None = None, outpu output_json = output_json or reports_dir / "review-studio.json" data = load_review_data(skill_dir) gates = build_gates(skill_dir, output_html, data) + contract = gate_contract(gates) blockers, warnings = add_blockers_from_gate(gates) - review_actions = build_review_actions(gates, skill_dir, output_html) + review_actions = build_review_actions(gates, skill_dir, output_html, data) + action_by_gate = {str(item.get("gate_key", "")): item for item in review_actions} + for gate in gates: + action = action_by_gate.get(str(gate.get("key", ""))) + gate["review_action_id"] = str(action.get("action_id", "")) if action else "" + gate["review_action_status"] = str(action.get("status", "")) if action else "" + gate["review_action_title"] = str(action.get("title", "")) if action else "" + gate["review_action_next_step"] = str(action.get("next_step", "")) if action else "" + gate["review_action_reason"] = str(action.get("reason", "")) if action else "" + gate["review_action_source_ref_count"] = len(action.get("source_refs", [])) if action else 0 + gate["review_action_verification_command"] = str(action.get("verification_command", "")) if action else "" score = weighted_score(gates) annotation_summary = data["review_annotations"].get("summary", {}) open_annotation_blockers = int(annotation_summary.get("open_blocker_count", 0) or 0) @@ -1356,12 +531,13 @@ def render_review_studio(skill_dir: Path, output_html: Path | None = None, outpu decision = "blocked" if blockers or open_annotation_blockers else ("review" if warnings or open_annotation_warnings else "ready") report = { "schema_version": "2.0", - "ok": True, + "ok": contract["ok"], "skill_dir": display_path(skill_dir, skill_dir), "summary": { "decision": decision, "world_class_score": score, "gate_count": len(gates), + "gate_contract_ok": contract["ok"], "blocker_count": len(blockers), "warning_count": len(warnings), "action_count": len(review_actions), @@ -1371,6 +547,7 @@ def render_review_studio(skill_dir: Path, output_html: Path | None = None, outpu "open_annotation_warning_count": open_annotation_warnings, }, "gates": gates, + "gate_contract": contract, "blockers": blockers, "warnings": warnings, "review_actions": review_actions, @@ -1392,13 +569,10 @@ def main() -> None: parser.add_argument("--output-html") parser.add_argument("--output-json") args = parser.parse_args() - payload = render_review_studio( - Path(args.skill_dir), - output_html=Path(args.output_html).resolve() if args.output_html else None, - output_json=Path(args.output_json).resolve() if args.output_json else None, - ) + output_html = Path(args.output_html).resolve() if args.output_html else None + output_json = Path(args.output_json).resolve() if args.output_json else None + payload = render_review_studio(Path(args.skill_dir), output_html=output_html, output_json=output_json) print(json.dumps(payload, ensure_ascii=False, indent=2)) - if __name__ == "__main__": main() diff --git a/scripts/render_review_viewer.py b/scripts/render_review_viewer.py index d2ee2d5f..83832027 100644 --- a/scripts/render_review_viewer.py +++ b/scripts/render_review_viewer.py @@ -2,315 +2,24 @@ import argparse import html import json -import re from pathlib import Path -from render_intent_confidence import render_intent_confidence -from render_intent_dialogue import render_intent_dialogue -from render_iteration_directions import render_iteration_directions -from render_artifact_design_profile import render_artifact_design_profile -from render_output_risk_profile import render_output_risk_profile -from render_prompt_quality_profile import render_prompt_quality_profile -from render_reference_scan import render_reference_scan -from render_reference_synthesis import render_reference_synthesis -from render_skill_overview import render_skill_overview +from review_viewer_data import ( + architecture_steps, + benchmark_cards, + compare_rows, + ensure_report_inputs, + evidence_readiness, + synthesis_cards, + variant_diff_cards, +) -def load_json(path: Path) -> dict: - if not path.exists(): - return {} - return json.loads(path.read_text(encoding="utf-8")) +ASSETS_DIR = Path(__file__).resolve().parent.parent / "assets" -def load_feedback_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "feedback-log.json") - return payload if isinstance(payload, dict) else {} - - -def load_baseline_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "baseline-compare.json") - return payload if isinstance(payload, dict) else {} - - -def load_specific_compare(skill_dir: Path) -> dict: - candidates = [ - skill_dir / "reports" / "description_optimization.json", - skill_dir.parent / "optimization" / "reports" / "description_optimization.json", - ] - for path in candidates: - payload = load_json(path) - if isinstance(payload, dict) and payload: - return payload - return {} - - -def load_specific_promotion(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "promotion_decisions.json") - return payload if isinstance(payload, dict) else {} - - -def load_benchmark_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "github-benchmark-scan.json") - return payload if isinstance(payload, dict) else {} - - -def load_reference_synthesis_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "reference-synthesis.json") - return payload if isinstance(payload, dict) else {} - - -def load_output_risk_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "output-risk-profile.json") - return payload if isinstance(payload, dict) else {} - - -def load_artifact_design_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "artifact-design-profile.json") - return payload if isinstance(payload, dict) else {} - - -def load_prompt_quality_summary(skill_dir: Path) -> dict: - payload = load_json(skill_dir / "reports" / "prompt-quality-profile.json") - return payload if isinstance(payload, dict) else {} - - -def ensure_report_inputs(skill_dir: Path) -> dict: - overview_json = skill_dir / "reports" / "skill-overview.json" - intent_confidence_json = skill_dir / "reports" / "intent-confidence.json" - intent_json = skill_dir / "reports" / "intent-dialogue.json" - reference_json = skill_dir / "reports" / "reference-scan.json" - reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json" - output_risk_json = skill_dir / "reports" / "output-risk-profile.json" - artifact_design_json = skill_dir / "reports" / "artifact-design-profile.json" - prompt_quality_json = skill_dir / "reports" / "prompt-quality-profile.json" - directions_json = skill_dir / "reports" / "iteration-directions.json" - - overview_payload = load_json(overview_json) if overview_json.exists() else {} - intent_confidence_payload = load_json(intent_confidence_json) if intent_confidence_json.exists() else {} - intent_payload = load_json(intent_json) if intent_json.exists() else {} - reference_payload = load_json(reference_json) if reference_json.exists() else {} - reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {} - output_risk_payload = load_json(output_risk_json) if output_risk_json.exists() else {} - artifact_design_payload = load_json(artifact_design_json) if artifact_design_json.exists() else {} - prompt_quality_payload = load_json(prompt_quality_json) if prompt_quality_json.exists() else {} - directions_payload = load_json(directions_json) if directions_json.exists() else {} - - intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"] - intent = intent_payload or render_intent_dialogue(skill_dir)["summary"] - reference = reference_payload or render_reference_scan(skill_dir, [])["summary"] - reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"] - output_risk = output_risk_payload or render_output_risk_profile(skill_dir)["summary"] - artifact_design = artifact_design_payload or render_artifact_design_profile(skill_dir)["summary"] - prompt_quality = prompt_quality_payload or render_prompt_quality_profile(skill_dir)["summary"] - iteration_payload = directions_payload or render_iteration_directions(skill_dir) - iteration = iteration_payload.get("summary", {}) - overview = overview_payload or render_skill_overview(skill_dir)["summary"] - feedback = load_feedback_summary(skill_dir) - baseline = load_baseline_summary(skill_dir) - compare = load_specific_compare(skill_dir) - promotion = load_specific_promotion(skill_dir) - benchmark = load_benchmark_summary(skill_dir) - reference_synthesis = load_reference_synthesis_summary(skill_dir) - output_risk = load_output_risk_summary(skill_dir) or output_risk - artifact_design = load_artifact_design_summary(skill_dir) or artifact_design - prompt_quality = load_prompt_quality_summary(skill_dir) or prompt_quality - return { - "overview": overview, - "intent_confidence": intent_confidence, - "intent": intent, - "reference": reference, - "iteration": iteration_payload, - "feedback": feedback, - "baseline": baseline, - "compare": compare, - "promotion": promotion, - "benchmark": benchmark, - "reference_synthesis": reference_synthesis, - "output_risk": output_risk, - "artifact_design": artifact_design, - "prompt_quality": prompt_quality, - } - - -def architecture_steps(overview: dict) -> list[dict]: - logic = overview.get("logic_steps", [])[:3] - usage = overview.get("usage_steps", [])[:2] - return [ - {"label": "Inputs", "detail": "workflow, prompt, transcript, docs, or notes"}, - {"label": "Boundary", "detail": overview.get("description", "Define the recurring job and exclusions.")}, - {"label": "Logic", "detail": "; ".join(logic) if logic else "Understand, execute, and validate."}, - {"label": "Usage", "detail": "; ".join(usage) if usage else "Load the skill and follow the workflow."}, - {"label": "Next", "detail": "Review the top iteration directions before growing the package."}, - ] - - -def compare_rows(compare: dict) -> list[dict]: - if not compare: - return [] - rows = [] - items = [ - ("Baseline", compare.get("baseline", {})), - ("Current", compare.get("current_candidate", {})), - (compare.get("winner", {}).get("label", "Winner"), compare.get("winner", {})), - ] - for label, payload in items: - if not payload: - continue - dev = payload.get("dev", {}) - holdout = payload.get("holdout", {}) - rows.append( - { - "label": label, - "tokens": payload.get("estimated_tokens", 0), - "dev_errors": dev.get("total_errors", 0), - "holdout_errors": holdout.get("total_errors", 0), - "strategy": payload.get("strategy", "existing"), - } - ) - return rows - - -def benchmark_cards(benchmark: dict) -> list[dict]: - cards = [] - for repo in benchmark.get("repositories", [])[:3]: - cards.append( - { - "name": repo.get("full_name", "Unknown repo"), - "borrow": repo.get("borrow", [])[:2], - "avoid": repo.get("avoid", [])[:1], - } - ) - return cards - - -def synthesis_cards(reference_synthesis: dict) -> list[dict]: - cards = [] - for track in reference_synthesis.get("source_tracks", [])[:3]: - cards.append( - { - "name": track.get("name", "Unknown track"), - "borrow": [track.get("borrow", "")] if track.get("borrow") else [], - "avoid": [track.get("avoid", "")] if track.get("avoid") else [], - } - ) - return cards - - -def split_sentences(text: str) -> list[str]: - if not text: - return [] - parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", " ".join(text.split())) if item.strip()] - return parts - - -def metric_delta(current: int | float, baseline: int | float) -> str: - delta = current - baseline - if delta == 0: - return "0" - return f"{delta:+}" - - -def variant_diff_cards(compare: dict) -> list[dict]: - baseline = compare.get("baseline", {}) - current = compare.get("current_candidate", {}) - winner = compare.get("winner", {}) - variants = [ - ("Baseline", baseline), - ("Current", current), - (f"Winner — {winner.get('label', 'Winner')}", winner), - ] - baseline_sentences = split_sentences(baseline.get("description", "")) - baseline_set = set(baseline_sentences) - baseline_dev = baseline.get("dev", {}).get("total_errors", 0) - baseline_holdout = baseline.get("holdout", {}).get("total_errors", 0) - cards = [] - seen = set() - for label, payload in variants: - if not payload: - continue - unique_key = (payload.get("description"), payload.get("strategy"), label) - if unique_key in seen: - continue - seen.add(unique_key) - description = payload.get("description", "") - sentences = split_sentences(description) - sentence_set = set(sentences) - added = [item for item in sentences if item not in baseline_set][:3] - removed = [item for item in baseline_sentences if item not in sentence_set][:2] - dev_errors = payload.get("dev", {}).get("total_errors", 0) - holdout_errors = payload.get("holdout", {}).get("total_errors", 0) - cards.append( - { - "label": label, - "strategy": payload.get("strategy", "existing"), - "description": description, - "tokens": payload.get("estimated_tokens", 0), - "dev_errors": dev_errors, - "holdout_errors": holdout_errors, - "token_delta": metric_delta(payload.get("estimated_tokens", 0), baseline.get("estimated_tokens", 0)), - "dev_delta": metric_delta(dev_errors, baseline_dev), - "holdout_delta": metric_delta(holdout_errors, baseline_holdout), - "added": added if label != "Baseline" else baseline_sentences[:3], - "removed": removed, - } - ) - return cards - - -def evidence_readiness(report: dict) -> dict: - intent_confidence = report.get("intent_confidence", {}) - reference_synthesis = report.get("reference_synthesis", {}) - output_risk = report.get("output_risk", {}) - artifact_design = report.get("artifact_design", {}) - prompt_quality = report.get("prompt_quality", {}) - benchmark = report.get("benchmark", {}) - synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {} - pattern_gate = synthesis.get("pattern_gate", {}) if isinstance(synthesis, dict) else {} - accepted_patterns = pattern_gate.get("accepted", []) if isinstance(pattern_gate, dict) else [] - conflicts = synthesis.get("conflicts", []) if isinstance(synthesis, dict) else [] - checks = [ - { - "label": "Intent clarity", - "status": "ready" if intent_confidence.get("gate_passed") else "needs review", - "detail": f"{intent_confidence.get('score', 0)}/100 intent confidence.", - }, - { - "label": "Benchmark coverage", - "status": "ready" if len(benchmark.get("repositories", [])) >= 2 else "needs evidence", - "detail": f"{len(benchmark.get('repositories', []))} GitHub benchmark repositories attached.", - }, - { - "label": "Pattern gate", - "status": "ready" if accepted_patterns else "needs review", - "detail": pattern_gate.get("summary", "No pattern gate summary attached."), - }, - { - "label": "Conflict handling", - "status": "ready" if not conflicts else "decision needed", - "detail": "No material conflicts detected." if not conflicts else conflicts[0].get("summary", "Conflict detected."), - }, - { - "label": "Output risk profile", - "status": "ready" if output_risk.get("risk_families") else "needs review", - "detail": f"{len(output_risk.get('risk_families', []))} output risk families attached.", - }, - { - "label": "Artifact design profile", - "status": "ready" if artifact_design.get("primary_artifact") else "needs review", - "detail": artifact_design.get("primary_artifact", {}).get("direction", "No artifact design profile attached."), - }, - { - "label": "Prompt quality profile", - "status": "ready" if prompt_quality.get("quality_matrix") else "needs review", - "detail": f"{prompt_quality.get('overall_quality_score', 0)}/100 prompt-facing quality score.", - }, - ] - ready_count = sum(1 for item in checks if item["status"] == "ready") - return { - "score": int(ready_count / len(checks) * 100), - "checks": checks, - "reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery.", - } +def review_viewer_css() -> str: + return (ASSETS_DIR / "review-viewer.css").read_text(encoding="utf-8").strip() def render_html(report: dict) -> str: @@ -566,182 +275,7 @@ def render_html(report: dict) -> str: {html.escape(overview.get('display_name', overview.get('title', 'Skill Review')))} Review Viewer diff --git a/scripts/render_review_waivers.py b/scripts/render_review_waivers.py index 70eb23d8..eb13cff2 100644 --- a/scripts/render_review_waivers.py +++ b/scripts/render_review_waivers.py @@ -2,26 +2,17 @@ import argparse import hashlib import json -from datetime import date +from datetime import date, timedelta from pathlib import Path from typing import Any +from review_studio_gates import REVIEW_STUDIO_GATE_KEYS + ROOT = Path(__file__).resolve().parent.parent -KNOWN_GATE_KEYS = { - "intent-canvas", - "trigger-lab", - "output-lab", - "context-budget", - "runtime-matrix", - "trust-report", - "permission-gates", - "permission-runtime", - "skill-atlas", - "operations-loop", - "registry-audit", - "release-notes", -} +NON_WAIVABLE_GATE_KEYS = {"review-waivers", "world-class-evidence"} +WAIVERABLE_GATE_KEYS = REVIEW_STUDIO_GATE_KEYS - NON_WAIVABLE_GATE_KEYS +KNOWN_GATE_KEYS = WAIVERABLE_GATE_KEYS VALID_DECISIONS = {"accepted-risk", "false-positive", "temporary-exception"} MIN_REASON_CHARS = 20 @@ -139,6 +130,88 @@ def validate_waivers(waivers: list[dict[str, Any]], today: date) -> tuple[list[d return normalized, failures, warnings +def output_lab_candidate(skill_dir: Path, covered_gate_keys: set[str], today: date) -> dict[str, Any] | None: + output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json").get("summary", {}) + output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {}) + output_review = load_json(skill_dir / "reports" / "output_review_adjudication.json").get("summary", {}) + if not output_quality and not output_execution and not output_review: + return None + pending = int(output_review.get("pending_count", 0) or 0) + model_executed = int(output_execution.get("model_executed_count", 0) or 0) + failure_count = int(output_quality.get("failure_count", 0) or 0) + if pending == 0 and model_executed > 0 and failure_count == 0: + return None + status = "covered" if "output-lab" in covered_gate_keys else "needs-reviewer-decision" + return { + "gate_key": "output-lab", + "label": "Output Lab", + "status": status, + "waiver_allowed": True, + "decision_options": sorted(VALID_DECISIONS), + "risk_summary": ( + f"review pending {pending}; model-executed {model_executed}; " + f"output failures {failure_count}" + ), + "required_review": [ + "Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.", + "Reviewer names the release scope and expiry date.", + "Reviewer links output_review_adjudication or output_execution evidence.", + ], + "suggested_evidence": "reports/output_review_adjudication.md", + "suggested_command": ( + "python3 scripts/yao.py review-waivers . --add-waiver " + "--gate-key output-lab --reviewer \"\" " + "--reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" " + f"--expires-at {(today + timedelta(days=365)).isoformat()} " + "--evidence reports/output_review_adjudication.md" + ), + "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence.", + } + + +def world_class_boundary(skill_dir: Path) -> dict[str, Any] | None: + ledger_summary = load_json(skill_dir / "reports" / "world_class_evidence_ledger.json").get("summary", {}) + if not ledger_summary: + return None + pending = int(ledger_summary.get("pending_count", 0) or 0) + if pending == 0 and ledger_summary.get("ready_to_claim_world_class") is True: + return None + return { + "gate_key": "world-class-evidence", + "label": "World-Class Evidence", + "status": "cannot-waive", + "waiver_allowed": False, + "risk_summary": ( + f"{pending} pending evidence entries; " + f"{ledger_summary.get('human_pending_count', 0)} human pending; " + f"{ledger_summary.get('external_pending_count', 0)} external pending" + ), + "required_review": [ + "Do not use a waiver to claim public world-class readiness.", + "Either submit accepted ledger evidence or state that this release does not claim world-class completion.", + "Keep claim guard active until ledger summary.ready_to_claim_world_class is true.", + ], + "suggested_evidence": "reports/world_class_evidence_ledger.md", + "suggested_command": ( + "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions " + "&& python3 scripts/yao.py world-class-claim-guard ." + ), + "world_class_boundary": "Non-waivable completion boundary.", + } + + +def build_waiver_candidates(skill_dir: Path, covered_gate_keys: list[str], today: date) -> list[dict[str, Any]]: + covered = set(covered_gate_keys) + candidates = [] + output_candidate = output_lab_candidate(skill_dir, covered, today) + if output_candidate: + candidates.append(output_candidate) + world_boundary = world_class_boundary(skill_dir) + if world_boundary: + candidates.append(world_boundary) + return candidates + + def render_report( skill_dir: Path, waivers_json: Path | None = None, @@ -163,6 +236,11 @@ def render_report( expired = [item for item in waivers if item["status"] == "expired"] invalid = [item for item in waivers if item["status"] == "invalid"] covered_gate_keys = sorted({item["gate_key"] for item in active}) + waiver_candidates = build_waiver_candidates(skill_dir, covered_gate_keys, today) + waiverable_open = [ + item for item in waiver_candidates if item["waiver_allowed"] and item["status"] != "covered" + ] + non_waivable = [item for item in waiver_candidates if not item["waiver_allowed"]] report = { "schema_version": "1.0", "ok": not failures, @@ -175,14 +253,21 @@ def render_report( "invalid_count": len(invalid), "covered_gate_count": len(covered_gate_keys), "covered_gate_keys": covered_gate_keys, + "waiver_candidate_count": len(waiver_candidates), + "waiverable_open_count": len(waiverable_open), + "non_waivable_count": len(non_waivable), }, "policy": { "blocker_waivers_allowed": False, "minimum_reason_chars": MIN_REASON_CHARS, "expires_required": True, + "review_studio_gate_keys": sorted(REVIEW_STUDIO_GATE_KEYS), "known_gate_keys": sorted(KNOWN_GATE_KEYS), + "waiverable_gate_keys": sorted(WAIVERABLE_GATE_KEYS), + "non_waivable_gate_keys": sorted(NON_WAIVABLE_GATE_KEYS), }, "waivers": waivers, + "waiver_candidates": waiver_candidates, "failures": failures, "warnings": warnings, "artifacts": { @@ -207,12 +292,19 @@ def render_markdown(report: dict[str, Any]) -> str: f"- Expired: `{summary['expired_count']}`", f"- Invalid: `{summary['invalid_count']}`", f"- Covered gates: `{', '.join(summary['covered_gate_keys']) or 'none'}`", + f"- Waiver candidates: `{summary['waiver_candidate_count']}`", + f"- Open waiverable candidates: `{summary['waiverable_open_count']}`", + f"- Non-waivable boundaries: `{summary['non_waivable_count']}`", "", "## Policy", "", "- Blocker waivers allowed: `False`", f"- Minimum reason chars: `{report['policy']['minimum_reason_chars']}`", "- Expiry is required for every waiver.", + "- World-class evidence completion cannot be waived; it can only be proven by accepted ledger evidence.", + f"- Review Studio gates: `{', '.join(report['policy']['review_studio_gate_keys'])}`", + f"- Waiverable gates: `{', '.join(report['policy']['waiverable_gate_keys'])}`", + f"- Non-waivable gates: `{', '.join(report['policy']['non_waivable_gate_keys'])}`", "", "## Waivers", "", @@ -226,6 +318,28 @@ def render_markdown(report: dict[str, Any]) -> str: lines.append( f"| `{item['id']}` | `{item['gate_key']}` | `{item['decision']}` | {item['reviewer']} | `{item['status']}` | `{item['expires_at']}` | {reason} |" ) + lines.extend(["", "## Candidate Actions", ""]) + candidates = report.get("waiver_candidates", []) + if not candidates: + lines.append("- None") + else: + lines.extend(["| Gate | Status | Waiver | Risk | Evidence |", "| --- | --- | --- | --- | --- |"]) + for item in candidates: + risk = str(item.get("risk_summary", "")).replace("|", "\\|") + lines.append( + f"| `{item['gate_key']}` | `{item['status']}` | `{str(item['waiver_allowed']).lower()}` | {risk} | `{item.get('suggested_evidence', '')}` |" + ) + for item in candidates: + lines.extend(["", f"### {item['label']}", ""]) + lines.append(f"- gate: `{item['gate_key']}`") + lines.append(f"- status: `{item['status']}`") + lines.append(f"- waiver allowed: `{str(item['waiver_allowed']).lower()}`") + lines.append(f"- risk: {item['risk_summary']}") + lines.append(f"- evidence: `{item.get('suggested_evidence', '')}`") + lines.append(f"- verification: `{item.get('suggested_command', '')}`") + lines.append(f"- world-class boundary: {item.get('world_class_boundary', '')}") + lines.extend(["", "#### Required Review", ""]) + lines.extend(f"- {review}" for review in item.get("required_review", [])) lines.extend(["", "## Failures", ""]) lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) lines.extend(["", "## Warnings", ""]) diff --git a/scripts/render_skill_interpretation.py b/scripts/render_skill_interpretation.py new file mode 100644 index 00000000..5c8cf359 --- /dev/null +++ b/scripts/render_skill_interpretation.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + +from render_skill_overview import render_html +from skill_report_model import build_report_model + + +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders the first-class skill interpretation report while reusing the Skill Overview v2 model and layout." + + +def build_interpretation_model(skill_dir: Path) -> dict: + summary = build_report_model(skill_dir) + contract = dict(summary.get("report_contract", {})) + contract.update( + { + "schema_version": "2.0", + "report_kind": "skill-interpretation", + "canonical_overview_report": "reports/skill-overview.html", + "html_report": "reports/skill-interpretation.html", + "json_report": "reports/skill-interpretation.json", + "layout": "kami-white-audit-v2", + "default_language": "zh-CN", + "languages": ["zh-CN", "en"], + "purpose": "Explain the generated skill's role, principles, usage scenarios, trigger contract, inputs, outputs, quality evidence, risks, assets, highlights, and next upgrade directions.", + } + ) + summary["report_contract"] = contract + summary["interpretation_contract"] = { + "schema_version": "2.0", + "source_model": "skill-overview-v2", + "source_model_reused": True, + "overview_report": "reports/skill-overview.html", + "html_report": "reports/skill-interpretation.html", + "json_report": "reports/skill-interpretation.json", + "default_language": "zh-CN", + "languages": ["zh-CN", "en"], + "includes": [ + "skill role", + "principles", + "usage scenarios", + "trigger contract", + "inputs and outputs", + "quality evidence", + "risk governance", + "package assets", + "highlights", + "upgrade roadmap", + ], + } + deliverables = summary.get("skill_summary", {}).get("deliverables", []) + for artifact in ["reports/skill-interpretation.html", "reports/skill-interpretation.json"]: + if artifact not in deliverables: + deliverables.append(artifact) + if "skill_summary" in summary: + summary["skill_summary"]["deliverables"] = deliverables + return summary + + +def render_skill_interpretation(skill_dir: Path, output_html: Path | None = None, output_json: Path | None = None) -> dict: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + + output_html = output_html or reports_dir / "skill-interpretation.html" + output_json = output_json or reports_dir / "skill-interpretation.json" + + summary = build_interpretation_model(skill_dir) + output_html.write_text(render_html(summary), encoding="utf-8") + output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + + return { + "ok": True, + "skill_dir": str(skill_dir), + "artifacts": { + "html": str(output_html), + "json": str(output_json), + "overview_html": str(skill_dir / "reports" / "skill-overview.html"), + }, + "summary": { + "name": summary.get("name"), + "report_kind": summary.get("report_contract", {}).get("report_kind"), + "default_language": summary.get("report_contract", {}).get("default_language"), + "section_count": len(summary.get("report_contract", {}).get("nav_labels", [])), + "source_model_reused": summary.get("interpretation_contract", {}).get("source_model_reused"), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render the first-class HTML skill interpretation report for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-html") + parser.add_argument("--output-json") + args = parser.parse_args() + + result = render_skill_interpretation( + Path(args.skill_dir), + Path(args.output_html).resolve() if args.output_html else None, + Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_skill_os2_audit.py b/scripts/render_skill_os2_audit.py new file mode 100644 index 00000000..eba70353 --- /dev/null +++ b/scripts/render_skill_os2_audit.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +STATUS_LABELS = { + "pass": "pass", + "warn": "warn", + "human_required": "human-required", + "external_required": "external-required", + "missing": "missing", +} + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def evidence(skill_dir: Path, *paths: str) -> list[dict[str, Any]]: + items = [] + for rel in paths: + path = skill_dir / rel + items.append({"path": rel, "exists": path.exists()}) + return items + + +def status_from(condition: bool, fallback: str = "missing") -> str: + return "pass" if condition else fallback + + +def audit_item( + key: str, + label: str, + status: str, + current: str, + target: str, + evidence_items: list[dict[str, Any]], + next_action: str, +) -> dict[str, Any]: + return { + "key": key, + "label": label, + "status": status, + "current": current, + "target": target, + "evidence": evidence_items, + "next_action": next_action, + } + + +def count_sources(events_summary: dict[str, Any]) -> dict[str, int]: + values = events_summary.get("source_types", {}) + return values if isinstance(values, dict) else {} + + +def build_audit(skill_dir: Path, generated_at: str) -> dict[str, Any]: + reports = skill_dir / "reports" + skill_ir = load_json(skill_dir / "skill-ir" / "examples" / "yao-meta-skill.json") + compiled = load_json(reports / "compiled_targets.json") + output_quality = load_json(reports / "output_quality_scorecard.json") + output_execution = load_json(reports / "output_execution_runs.json") + output_review = load_json(reports / "output_review_adjudication.json") + benchmark_reproducibility = load_json(reports / "benchmark_reproducibility.json") + conformance = load_json(reports / "conformance_matrix.json") + trust = load_json(reports / "security_trust_report.json") + runtime_permissions = load_json(reports / "runtime_permission_probes.json") + atlas = load_json(reports / "skill_atlas.json") + registry = load_json(reports / "registry_audit.json") + package_verification = load_json(reports / "package_verification.json") + install = load_json(reports / "install_simulation.json") + review_studio = load_json(reports / "review-studio.json") + adoption = load_json(reports / "adoption_drift_report.json") + telemetry_hooks = load_json(reports / "telemetry_hook_recipes.json") + + compiled_summary = compiled.get("summary", {}) + output_summary = output_quality.get("summary", {}) + execution_summary = output_execution.get("summary", {}) + review_summary = output_review.get("summary", {}) + benchmark_summary = benchmark_reproducibility.get("summary", {}) + conformance_summary = conformance.get("summary", {}) + trust_summary = trust.get("summary", {}) + permission_summary = runtime_permissions.get("summary", {}) + atlas_summary = atlas.get("summary", {}) + package_summary = package_verification.get("summary", {}) + install_summary = install.get("summary", {}) + studio_summary = review_studio.get("summary", {}) + adoption_summary = adoption.get("summary", {}) + telemetry_summary = telemetry_hooks.get("summary", {}) + sources = count_sources(adoption_summary) + + items = [ + audit_item( + "skill-ir", + "Skill IR", + status_from( + skill_ir.get("schema_version") == "2.0.0" + and bool((skill_dir / "skill-ir" / "schema.json").exists()) + and bool((skill_dir / "skill-ir" / "examples" / "yao-meta-skill.json").exists()) + ), + f"schema {skill_ir.get('schema_version', 'missing')}; targets {len(skill_ir.get('targets', []))}", + "2.0 schema, root export, and target-neutral contract evidence", + evidence(skill_dir, "skill-ir/schema.json", "skill-ir/examples/yao-meta-skill.json", "references/skill-ir-method.md"), + "Keep IR as the source before target packaging.", + ), + audit_item( + "target-compiler", + "Target Compiler", + status_from(compiled_summary.get("target_count", 0) >= 5 and compiled_summary.get("pass_count") == compiled_summary.get("target_count")), + f"{compiled_summary.get('pass_count', 0)}/{compiled_summary.get('target_count', 0)} targets pass", + "OpenAI, Claude, generic, Agent Skills compatible, and VS Code contracts generated from IR", + evidence(skill_dir, "scripts/compile_skill.py", "reports/compiled_targets.json", "tests/verify_compile_skill.py"), + "Deepen target-native transforms when provider clients expose stronger runtime APIs.", + ), + audit_item( + "output-eval-lab", + "Output Eval Lab", + status_from( + output_summary.get("case_count", 0) >= 5 + and output_summary.get("gate_pass") is True + and execution_summary.get("variant_run_count", 0) >= 10 + and output_summary.get("blind_pair_count", 0) >= 5 + ), + ( + f"{output_summary.get('case_count', 0)} cases; " + f"delta {output_summary.get('delta', 'n/a')}; " + f"exec {execution_summary.get('variant_run_count', 0)}; " + f"blind {output_summary.get('blind_pair_count', 0)}" + ), + "with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy", + evidence( + skill_dir, + "evals/output/cases.jsonl", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "reports/output_quality_scorecard.json", + "reports/output_execution_runs.json", + "reports/output_blind_review_pack.json", + ), + "Add more real-file and adversarial holdout cases as usage grows.", + ), + audit_item( + "provider-holdout", + "Provider Holdout", + "pass" if execution_summary.get("model_executed_count", 0) > 0 else "external_required", + f"model-executed {execution_summary.get('model_executed_count', 0)}; token-observed {execution_summary.get('token_observed_count', 0)}", + "At least one real provider-backed holdout run with observed model/timing/token metadata", + evidence(skill_dir, "scripts/provider_output_eval_runner.py", "reports/output_execution_runs.json"), + "Run provider-backed holdout cases with real credentials and commit only aggregate evidence.", + ), + audit_item( + "human-adjudication", + "Human Adjudication", + "pass" if review_summary.get("ready_for_human_evidence") is True else "human_required", + f"{review_summary.get('judgment_count', 0)}/{review_summary.get('pair_count', 0)} decisions; pending {review_summary.get('pending_count', 0)}", + "Real reviewer decisions, blind-review attestation, and integrity fingerprints recorded before claiming output review completion", + evidence(skill_dir, "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "scripts/adjudicate_output_review.py"), + "Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.", + ), + audit_item( + "benchmark-reproducibility", + "Benchmark Reproducibility", + status_from( + benchmark_reproducibility.get("ok") is True + and benchmark_summary.get("reproducibility_ready") is True + and benchmark_summary.get("failure_disclosure_count", 0) > 0 + and benchmark_summary.get("missing_artifact_count", 1) == 0 + ), + ( + f"artifacts {benchmark_summary.get('required_artifact_count', 0)}; " + f"missing {benchmark_summary.get('missing_artifact_count', 'n/a')}; " + f"failures {benchmark_summary.get('failure_disclosure_count', 0)}" + ), + "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable", + evidence( + skill_dir, + "reports/benchmark_methodology.md", + "reports/benchmark_reproducibility.json", + "reports/benchmark_reproducibility.md", + "evals/failure-cases.md", + "tests/verify_benchmark_reproducibility.py", + ), + "Keep the manifest current with every benchmark, package, and release evidence change.", + ), + audit_item( + "runtime-conformance", + "Runtime Conformance", + status_from(conformance_summary.get("target_count", 0) >= 5 and conformance_summary.get("pass_count") == conformance_summary.get("target_count")), + f"{conformance_summary.get('pass_count', 0)}/{conformance_summary.get('target_count', 0)} targets pass", + "Target package structure, metadata, relative paths, and degradation notes pass", + evidence(skill_dir, "runtime/conformance/schema.json", "scripts/run_conformance_suite.py", "reports/conformance_matrix.json"), + "Keep target conformance fixtures updated as platform contracts change.", + ), + audit_item( + "trust-security", + "Trust Security", + status_from( + trust.get("ok") is True + and trust_summary.get("secret_findings", 1) == 0 + and trust_summary.get("help_smoke_failed_count", 1) == 0 + and trust_summary.get("permission_missing_count", 1) == 0 + ), + ( + f"secrets {trust_summary.get('secret_findings', 'n/a')}; " + f"scripts {trust_summary.get('script_count', 'n/a')}; " + f"help failures {trust_summary.get('help_smoke_failed_count', 'n/a')}" + ), + "Secrets, scripts, dependencies, permissions, and package hash are reviewable", + evidence(skill_dir, "scripts/trust_check.py", "reports/security_trust_report.json", "security/permission_policy.json"), + "Keep high-permission approvals scoped, expiring, and target-mapped.", + ), + audit_item( + "runtime-permission-metadata", + "Permission Metadata", + status_from(permission_summary.get("target_count", 0) >= 4 and permission_summary.get("fail_count", 1) == 0), + ( + f"{permission_summary.get('pass_count', 0)}/{permission_summary.get('target_count', 0)} target probes pass; " + f"metadata fallback {permission_summary.get('metadata_fallback_count', 0)}; " + f"installer enforcement {permission_summary.get('installer_enforcement_pass_count', 0)}" + ), + "Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available", + evidence(skill_dir, "scripts/probe_runtime_permissions.py", "reports/runtime_permission_probes.json"), + "Preserve residual-risk notes until real native enforcement exists.", + ), + audit_item( + "native-permission-enforcement", + "Native Permission Enforcement", + "pass" if permission_summary.get("native_enforcement_count", 0) > 0 else "external_required", + ( + f"native-enforced targets {permission_summary.get('native_enforcement_count', 0)}; " + f"installer-enforced targets {permission_summary.get('installer_enforcement_pass_count', 0)}" + ), + "At least one target/client enforces approved permissions at runtime", + evidence(skill_dir, "reports/runtime_permission_probes.json", "reports/install_simulation.json", "security/permission_policy.json"), + "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.", + ), + audit_item( + "skill-atlas", + "Skill Atlas", + status_from( + atlas_summary.get("actionable_skill_count", 0) >= 1 + and atlas_summary.get("actionable_route_collision_count", 1) == 0 + and atlas_summary.get("actionable_owner_gap_count", 1) == 0 + ), + f"{atlas_summary.get('skill_count', 0)} skills; actionable collisions {atlas_summary.get('actionable_route_collision_count', 0)}", + "Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities", + evidence(skill_dir, "scripts/build_skill_atlas.py", "skill_atlas/catalog.json", "reports/skill_atlas.json", "skill_atlas/policy.json"), + "Feed real drift data into Atlas once client telemetry is installed.", + ), + audit_item( + "registry-distribution", + "Registry Distribution", + status_from( + registry.get("ok") is True + and package_verification.get("ok") is True + and install.get("ok") is True + and package_summary.get("archive_entry_count", 0) > 0 + and install_summary.get("installer_permission_failure_count", 1) == 0 + ), + ( + f"zip entries {package_summary.get('archive_entry_count', 0)}; " + f"install failures {install_summary.get('failure_count', 0)}; " + f"permission failures {install_summary.get('installer_permission_failure_count', 0)}" + ), + "Package metadata, archive checksum, package verification, and install simulation pass", + evidence(skill_dir, "registry/packages/yao-meta-skill.json", "reports/package_verification.json", "reports/install_simulation.json"), + "Regenerate registry after package verification so checksums stay aligned.", + ), + audit_item( + "review-studio", + "Review Studio", + status_from(studio_summary.get("gate_count", 0) >= 13 and studio_summary.get("blocker_count", 1) == 0), + f"decision {studio_summary.get('decision', 'missing')}; warnings {studio_summary.get('warning_count', 0)}; score {studio_summary.get('world_class_score', 'n/a')}", + "One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations", + evidence(skill_dir, "scripts/render_review_studio.py", "reports/review-studio.json", "reports/review-studio.html"), + "Resolve human/external warning gates before claiming full release readiness.", + ), + audit_item( + "telemetry-drift", + "Telemetry Drift", + status_from( + adoption.get("ok") is True + and adoption.get("privacy_contract", {}).get("raw_content_allowed") is False + and telemetry_summary.get("recipe_count", 0) >= 5 + ), + f"events {adoption_summary.get('event_count', 0)}; risk {adoption_summary.get('risk_band', 'missing')}; recipes {telemetry_summary.get('recipe_count', 0)}", + "Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path", + evidence(skill_dir, "reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json", "scripts/import_telemetry_events.py"), + "Keep raw JSONL out of distributed packages and use aggregate reports for Atlas.", + ), + audit_item( + "native-client-telemetry", + "Native Client Telemetry", + "pass" if sources.get("external", 0) > 0 and adoption_summary.get("adoption_sample_count", 0) > 0 else "external_required", + f"external source events {sources.get('external', 0)}; adoption samples {adoption_summary.get('adoption_sample_count', 0)}", + "A real Browser/Chrome/provider client sends production metadata events", + evidence(skill_dir, "scripts/telemetry_native_host.py", "reports/adoption_drift_report.json"), + "Install a real client against the native host and import production metadata-only events.", + ), + ] + + counts: dict[str, int] = {key: 0 for key in STATUS_LABELS} + for item in items: + counts[item["status"]] = counts.get(item["status"], 0) + 1 + open_gap_count = sum(counts.get(key, 0) for key in ("warn", "human_required", "external_required", "missing")) + local_foundation_ready = counts.get("missing", 0) == 0 and counts.get("warn", 0) == 0 + world_class_ready = open_gap_count == 0 + summary = { + "item_count": len(items), + "pass_count": counts.get("pass", 0), + "warn_count": counts.get("warn", 0), + "human_required_count": counts.get("human_required", 0), + "external_required_count": counts.get("external_required", 0), + "missing_count": counts.get("missing", 0), + "open_gap_count": open_gap_count, + "local_foundation_ready": local_foundation_ready, + "world_class_ready": world_class_ready, + "decision": "world-class-ready" if world_class_ready else "continue-iteration", + } + return { + "schema_version": "1.0", + "ok": counts.get("missing", 0) == 0, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": summary, + "status_counts": counts, + "items": items, + "next_highest_leverage": [ + item for item in items if item["status"] in {"human_required", "external_required", "warn", "missing"} + ][:5], + "artifacts": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "world_class_evidence_plan": "reports/world_class_evidence_plan.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Skill OS 2.0 Audit", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- pass: `{summary['pass_count']}` / `{summary['item_count']}`", + f"- human required: `{summary['human_required_count']}`", + f"- external required: `{summary['external_required_count']}`", + f"- missing: `{summary['missing_count']}`", + f"- world-class ready: `{str(summary['world_class_ready']).lower()}`", + "- evidence plan: `reports/world_class_evidence_plan.md`", + "", + "## Audit Items", + "", + "| Area | Status | Current | Target | Next action |", + "| --- | --- | --- | --- | --- |", + ] + for item in report["items"]: + lines.append( + "| " + + " | ".join( + [ + item["label"], + STATUS_LABELS.get(item["status"], item["status"]), + item["current"].replace("|", "\\|"), + item["target"].replace("|", "\\|"), + item["next_action"].replace("|", "\\|"), + ] + ) + + " |" + ) + lines.extend(["", "## Open Highest-Leverage Gaps", ""]) + for item in report["next_highest_leverage"]: + lines.append(f"- `{item['key']}` ({STATUS_LABELS.get(item['status'], item['status'])}): {item['next_action']}") + if not report["next_highest_leverage"]: + lines.append("- None.") + lines.extend(["", "## Evidence", ""]) + for item in report["items"]: + paths = [entry["path"] for entry in item["evidence"] if entry.get("exists")] + missing = [entry["path"] for entry in item["evidence"] if not entry.get("exists")] + lines.append(f"### {item['label']}") + lines.append("") + lines.append(f"- existing evidence: {', '.join(f'`{path}`' for path in paths) if paths else '`none`'}") + if missing: + lines.append(f"- missing evidence: {', '.join(f'`{path}`' for path in missing)}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a requirement-by-requirement Skill OS 2.0 audit.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/skill_os2_audit.json") + parser.add_argument("--output-md", default="reports/skill_os2_audit.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_audit(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_skill_os2_coverage.py b/scripts/render_skill_os2_coverage.py new file mode 100644 index 00000000..aee29bda --- /dev/null +++ b/scripts/render_skill_os2_coverage.py @@ -0,0 +1,555 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from skill_os2_coverage_markdown import render_markdown + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a Skill OS 2.0 blueprint-to-evidence coverage audit." +RECOMMENDED_PR_LABELS = { + "benchmark-methodology": "Benchmark Methodology", + "output-eval-schema": "Output Eval Schema", + "output-eval-runner": "Output Eval Runner", + "output-quality-scorecard": "Output Quality Scorecard", + "skill-ir-v0": "Skill IR V0", + "compiler-refactor": "Compiler Refactor", + "agent-skills-conformance": "Agent Skills Conformance", + "trust-check": "Trust Check", + "skill-atlas-generator": "Skill Atlas Generator", + "registry-package-format": "Registry Package Format", + "review-studio-2": "Review Studio 2.0", + "migration-v2-docs": "Migration V2 Docs", + "evidence-consistency": "Evidence Consistency", +} + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def evidence(skill_dir: Path, paths: list[str]) -> list[dict[str, Any]]: + return [{"path": item, "exists": (skill_dir / item).exists()} for item in paths] + + +def all_exist(items: list[dict[str, Any]]) -> bool: + return all(item["exists"] for item in items) + + +def paths_exist(skill_dir: Path, paths: list[str]) -> bool: + return all((skill_dir / item).exists() for item in paths) + + +def latest_daily_skillops_paths(skill_dir: Path) -> list[str]: + base_paths = [ + "scripts/render_daily_skillops_report.py", + "tests/verify_daily_skillops.py", + ] + daily_dir = skill_dir / "reports" / "skillops" / "daily" + daily_json_reports = sorted(daily_dir.glob("*.json")) if daily_dir.exists() else [] + if not daily_json_reports: + return [ + *base_paths, + "reports/skillops/daily/YYYY-MM-DD.json", + "reports/skillops/daily/YYYY-MM-DD.md", + ] + latest_json = daily_json_reports[-1] + latest_md = latest_json.with_suffix(".md") + return [*base_paths, rel_path(latest_json, skill_dir), rel_path(latest_md, skill_dir)] + + +def latest_weekly_curator_paths(skill_dir: Path) -> list[str]: + base_paths = [ + "scripts/render_weekly_curator_report.py", + "tests/verify_weekly_curator.py", + ] + weekly_dir = skill_dir / "reports" / "skillops" / "weekly" + weekly_json_reports = sorted(weekly_dir.glob("*.json")) if weekly_dir.exists() else [] + if not weekly_json_reports: + return [ + *base_paths, + "reports/skillops/weekly/YYYY-WNN.json", + "reports/skillops/weekly/YYYY-WNN.md", + ] + latest_json = weekly_json_reports[-1] + latest_md = latest_json.with_suffix(".md") + return [*base_paths, rel_path(latest_json, skill_dir), rel_path(latest_md, skill_dir)] + + +def condition_status(condition: bool, evidence_items: list[dict[str, Any]]) -> str: + if not all_exist(evidence_items): + return "missing" + return "pass" if condition else "warn" + + +def build_item( + *, + key: str, + category: str, + label: str, + objective: str, + artifact_paths: list[str], + command: str, + test: str, + current: str, + condition: bool, + next_action: str, + skill_dir: Path, +) -> dict[str, Any]: + evidence_items = evidence(skill_dir, artifact_paths) + status = condition_status(condition, evidence_items) + return { + "key": key, + "category": category, + "label": label, + "status": status, + "objective": objective, + "current": current, + "command": command, + "test": test, + "evidence": evidence_items, + "next_action": next_action, + } + + +def build_extension_track( + *, + key: str, + label: str, + status: str, + objective: str, + current: str, + target: str, + artifact_paths: list[str], + next_action: str, + skill_dir: Path, +) -> dict[str, Any]: + return { + "key": key, + "label": label, + "status": status, + "objective": objective, + "current": current, + "target": target, + "evidence": evidence(skill_dir, artifact_paths), + "next_action": next_action, + } + + +def summary_value(payload: dict[str, Any], key: str, default: Any = 0) -> Any: + summary = payload.get("summary", {}) + return summary.get(key, default) if isinstance(summary, dict) else default + + +def target_names(payload: dict[str, Any]) -> set[str]: + names = set() + for item in payload.get("targets", []) or payload.get("results", []): + if isinstance(item, dict): + value = item.get("target") or item.get("platform") or item.get("name") + if value: + names.add(str(value)) + return names + + +def build_coverage(skill_dir: Path, generated_at: str) -> dict[str, Any]: + reports = skill_dir / "reports" + skill_ir = load_json(skill_dir / "skill-ir" / "examples" / "yao-meta-skill.json") + output_quality = load_json(reports / "output_quality_scorecard.json") + output_execution = load_json(reports / "output_execution_runs.json") + compiled = load_json(reports / "compiled_targets.json") + conformance = load_json(reports / "conformance_matrix.json") + trust = load_json(reports / "security_trust_report.json") + atlas = load_json(reports / "skill_atlas.json") + registry = load_json(reports / "registry_audit.json") + package_verification = load_json(reports / "package_verification.json") + install = load_json(reports / "install_simulation.json") + review_studio = load_json(reports / "review-studio.json") + adoption = load_json(reports / "adoption_drift_report.json") + telemetry_hooks = load_json(reports / "telemetry_hook_recipes.json") + benchmark = load_json(reports / "benchmark_reproducibility.json") + world_class_ledger = load_json(reports / "world_class_evidence_ledger.json") + evidence_consistency = load_json(reports / "evidence_consistency.json") + + output_case_count = summary_value(output_quality, "case_count") + output_delta = summary_value(output_quality, "delta", "n/a") + execution_count = summary_value(output_execution, "variant_run_count") + compiled_count = summary_value(compiled, "target_count") + compiled_pass = summary_value(compiled, "pass_count") + conformance_count = summary_value(conformance, "target_count") + conformance_pass = summary_value(conformance, "pass_count") + script_count = summary_value(trust, "script_count") + atlas_skill_count = summary_value(atlas, "skill_count") + package_entries = summary_value(package_verification, "archive_entry_count") + install_failures = summary_value(install, "failure_count") + studio_gates = summary_value(review_studio, "gate_count") + telemetry_recipe_count = summary_value(telemetry_hooks, "recipe_count") + benchmark_artifacts = summary_value(benchmark, "required_artifact_count") + pending_world_class = summary_value(world_class_ledger, "pending_count") + + modules = [ + build_item( + key="skill-ir", + category="core-module", + label="Skill IR", + objective="Platform-neutral capability contract exists before platform-specific packaging.", + artifact_paths=["skill-ir/schema.json", "skill-ir/examples/yao-meta-skill.json", "scripts/export_skill_ir.py", "tests/verify_skill_ir.py"], + command="python3 scripts/yao.py skill-ir .", + test="python3 tests/verify_skill_ir.py", + current=f"schema {skill_ir.get('schema_version', 'missing')}; targets {len(skill_ir.get('targets', []))}", + condition=skill_ir.get("schema_version") == "2.0.0" and len(skill_ir.get("targets", [])) >= 5, + next_action="Keep all target packages compiled from IR rather than hand-maintained per target.", + skill_dir=skill_dir, + ), + build_item( + key="output-eval-lab", + category="core-module", + label="Output Eval Lab", + objective="with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs.", + artifact_paths=["evals/output/schema.json", "evals/output/cases.jsonl", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "reports/output_quality_scorecard.json", "tests/verify_output_eval_lab.py"], + command="python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .", + test="python3 tests/verify_output_eval_lab.py", + current=f"{output_case_count} cases; delta {output_delta}; execution {execution_count}", + condition=output_case_count >= 5 and execution_count >= 10 and summary_value(output_quality, "gate_pass") is True, + next_action="Add more real-file and adversarial holdout cases as adoption data grows.", + skill_dir=skill_dir, + ), + build_item( + key="runtime-conformance", + category="core-module", + label="Runtime Conformance", + objective="Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets.", + artifact_paths=["runtime/conformance/schema.json", "scripts/run_conformance_suite.py", "reports/conformance_matrix.json", "tests/verify_conformance_suite.py"], + command="python3 scripts/yao.py conformance .", + test="python3 tests/verify_conformance_suite.py", + current=f"{conformance_pass}/{conformance_count} targets pass", + condition=conformance_count >= 5 and conformance_pass == conformance_count and "agent-skills" in target_names(conformance), + next_action="Keep conformance cases aligned with current platform metadata rules.", + skill_dir=skill_dir, + ), + build_item( + key="trust-security", + category="core-module", + label="Trust Security", + objective="Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", + artifact_paths=["scripts/trust_check.py", "security/trust_policy.md", "security/script_policy.md", "security/permission_policy.json", "reports/security_trust_report.json", "tests/verify_trust_check.py"], + command="python3 scripts/yao.py trust .", + test="python3 tests/verify_trust_check.py", + current=f"{script_count} scripts; secrets {summary_value(trust, 'secret_findings', 'n/a')}; help failures {summary_value(trust, 'help_smoke_failed_count', 'n/a')}", + condition=trust.get("ok") is True and summary_value(trust, "secret_findings", 1) == 0 and summary_value(trust, "help_smoke_failed_count", 1) == 0, + next_action="Keep high-permission approvals scoped, expiring, and mapped to target enforcement.", + skill_dir=skill_dir, + ), + build_item( + key="skill-atlas", + category="core-module", + label="Skill Atlas", + objective="Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities.", + artifact_paths=["scripts/build_skill_atlas.py", "skill_atlas/catalog.json", "skill_atlas/route_overlap_matrix.csv", "skill_atlas/dependency_graph.json", "reports/skill_atlas.json", "tests/verify_skill_atlas.py"], + command="python3 scripts/yao.py skill-atlas --workspace-root .", + test="python3 tests/verify_skill_atlas.py", + current=f"{atlas_skill_count} scanned skills; actionable collisions {summary_value(atlas, 'actionable_route_collision_count')}", + condition=atlas_skill_count >= 1 and summary_value(atlas, "actionable_route_collision_count", 1) == 0, + next_action="Use real telemetry to rank stale or drifting skills by impact, not only by static metadata.", + skill_dir=skill_dir, + ), + build_item( + key="registry-distribution", + category="core-module", + label="Registry Distribution", + objective="Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", + artifact_paths=["registry/index.schema.json", "registry/package.schema.json", "registry/packages/yao-meta-skill.json", "scripts/registry_audit.py", "reports/package_verification.json", "reports/install_simulation.json", "tests/verify_registry_audit.py"], + command="python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", + test="python3 tests/verify_registry_audit.py", + current=f"archive entries {package_entries}; install failures {install_failures}", + condition=registry.get("ok") is True and package_verification.get("ok") is True and install.get("ok") is True and package_entries > 0 and install_failures == 0, + next_action="Regenerate registry metadata after package verification so source and archive checksums stay aligned.", + skill_dir=skill_dir, + ), + build_item( + key="review-studio", + category="core-module", + label="Review Studio", + objective="One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.", + artifact_paths=["scripts/render_review_studio.py", "reports/review-studio.html", "reports/review-studio.json", "tests/verify_review_studio.py"], + command="python3 scripts/yao.py review-studio .", + test="python3 tests/verify_review_studio.py", + current=f"{studio_gates} gates; decision {summary_value(review_studio, 'decision', 'missing')}; warnings {summary_value(review_studio, 'warning_count')}", + condition=review_studio.get("ok") is True and studio_gates >= 14 and summary_value(review_studio, "blocker_count", 1) == 0, + next_action="Close pending human and external evidence before claiming full release readiness.", + skill_dir=skill_dir, + ), + build_item( + key="telemetry-drift", + category="core-module", + label="Telemetry Drift", + objective="Real usage feedback is captured as metadata-only local-first drift signals.", + artifact_paths=["scripts/emit_telemetry_event.py", "scripts/import_telemetry_events.py", "scripts/telemetry_native_host.py", "reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json", "tests/verify_telemetry_hooks.py"], + command="python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .", + test="python3 tests/verify_telemetry_hooks.py", + current=f"events {summary_value(adoption, 'event_count')}; recipes {telemetry_recipe_count}; risk {summary_value(adoption, 'risk_band', 'missing')}", + condition=adoption.get("ok") is True and adoption.get("privacy_contract", {}).get("raw_content_allowed") is False and telemetry_recipe_count >= 5, + next_action="Install a real client and import production metadata-only events into the local drift loop.", + skill_dir=skill_dir, + ), + ] + + recommended_prs = [ + ("benchmark-methodology", "reports/benchmark_methodology.md", "reports/benchmark_reproducibility.json", "tests/verify_benchmark_reproducibility.py", benchmark.get("ok") is True and summary_value(benchmark, "methodology_complete") is True, f"{benchmark_artifacts} required artifacts checked"), + ("output-eval-schema", "evals/output/schema.json", "evals/output/cases.jsonl", "tests/verify_output_eval_lab.py", output_case_count >= 5, f"{output_case_count} output cases"), + ("output-eval-runner", "scripts/run_output_eval.py", "reports/output_quality_scorecard.json", "tests/verify_output_eval_lab.py", output_quality.get("ok") is True, f"delta {output_delta}"), + ("output-quality-scorecard", "reports/output_quality_scorecard.md", "reports/output_quality_scorecard.json", "tests/verify_output_eval_lab.py", summary_value(output_quality, "gate_pass") is True, f"gate pass {summary_value(output_quality, 'gate_pass', False)}"), + ("skill-ir-v0", "skill-ir/schema.json", "skill-ir/examples/yao-meta-skill.json", "tests/verify_skill_ir.py", skill_ir.get("schema_version") == "2.0.0", f"schema {skill_ir.get('schema_version', 'missing')}"), + ("compiler-refactor", "scripts/compile_skill.py", "reports/compiled_targets.json", "tests/verify_compile_skill.py", compiled_count >= 5 and compiled_pass == compiled_count, f"{compiled_pass}/{compiled_count} compiled targets"), + ("agent-skills-conformance", "runtime/conformance/schema.json", "reports/conformance_matrix.json", "tests/verify_conformance_suite.py", "agent-skills" in target_names(conformance), "agent-skills target present"), + ("trust-check", "scripts/trust_check.py", "reports/security_trust_report.json", "tests/verify_trust_check.py", trust.get("ok") is True, f"secret findings {summary_value(trust, 'secret_findings', 'n/a')}"), + ("skill-atlas-generator", "scripts/build_skill_atlas.py", "reports/skill_atlas.json", "tests/verify_skill_atlas.py", atlas.get("ok") is True, f"{atlas_skill_count} scanned skills"), + ("registry-package-format", "registry/package.schema.json", "reports/registry_audit.json", "tests/verify_registry_audit.py", registry.get("ok") is True, f"registry ok {registry.get('ok') is True}"), + ("review-studio-2", "scripts/render_review_studio.py", "reports/review-studio.html", "tests/verify_review_studio.py", review_studio.get("ok") is True and studio_gates >= 14, f"{studio_gates} review gates"), + ("migration-v2-docs", "docs/migration-v2.md", "reports/skill-os-2-review.md", "README.md", (skill_dir / "docs" / "migration-v2.md").exists(), "migration guide present"), + ("evidence-consistency", "scripts/render_evidence_consistency.py", "reports/evidence_consistency.json", "tests/verify_evidence_consistency.py", evidence_consistency.get("ok") is True and summary_value(evidence_consistency, "fail_count") == 0, f"{summary_value(evidence_consistency, 'check_count')} consistency checks"), + ] + pr_items = [ + build_item( + key=key, + category="recommended-pr", + label=RECOMMENDED_PR_LABELS[key], + objective="Recommended Skill OS 2.0 implementation PR from the upgrade plan.", + artifact_paths=[path_a, path_b, path_c], + command="make ci-test", + test=path_c if path_c.startswith("tests/") else "docs review", + current=current, + condition=condition, + next_action="Keep this item covered as the implementation evolves.", + skill_dir=skill_dir, + ) + for key, path_a, path_b, path_c, condition, current in recommended_prs + ] + + interpretation_paths = [ + "reports/skill-overview.html", + "reports/skill-overview.json", + "scripts/render_skill_overview.py", + "scripts/render_skill_interpretation.py", + "schemas/skill-interpretation.schema.json", + "tests/verify_skill_interpretation.py", + ] + interpretation_ready = paths_exist(skill_dir, interpretation_paths) + interpretation_status = "covered" if interpretation_ready else "partial" + interpretation_current = ( + "Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests." + if interpretation_ready + else "Skill Overview v2 already covers much of the explainer experience, but dedicated skill-interpretation JSON/HTML artifacts are not first-class yet." + ) + interpretation_next_action = ( + "Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change." + if interpretation_ready + else "Decide whether overview v2 is the canonical interpretation surface; if not, add a dedicated schema, renderer, and CJK/path-safety tests." + ) + extension_tracks = [ + build_extension_track( + key="skill-interpretation-report", + label="Skill Interpretation Report", + status=interpretation_status, + objective="User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations.", + current=interpretation_current, + target="Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests.", + artifact_paths=interpretation_paths, + next_action=interpretation_next_action, + skill_dir=skill_dir, + ), + ] + adaptive_foundation_paths = [ + "references/autonomous-adaptation.md", + "references/user-memory-policy.md", + "schemas/adaptation-proposal.schema.json", + "scripts/summarize_user_signals.py", + "scripts/propose_adaptation.py", + "tests/verify_adaptation_safety.py", + ] + adaptive_apply_paths = [ + "scripts/apply_adaptation.py", + "reports/adaptation_approval_ledger.json", + "reports/adaptation_regression_report.json", + ] + adaptive_foundation_ready = paths_exist(skill_dir, adaptive_foundation_paths) + adaptive_apply_ready = paths_exist(skill_dir, adaptive_apply_paths) + adaptive_status = "covered" if adaptive_foundation_ready and adaptive_apply_ready else ("partial" if adaptive_foundation_ready else "planned") + adaptive_current = ( + "Proposal-only adapt-scan/adapt-propose foundation exists with policy, schema, and safety tests; approval-gated patch application is not implemented yet." + if adaptive_status == "partial" + else ( + "Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata." + if adaptive_status == "covered" + else "The repo has feedback, iteration, telemetry, and review artifacts, but no adapt-scan/adapt-propose/adapt-apply approval loop and no user-memory policy." + ) + ) + adaptive_next_action = ( + "Add adapt-apply only after approval ledger, allowlisted targets, dry-run diffs, regression reports, and rollback artifacts are designed." + if adaptive_status == "partial" + else "Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path." + ) + extension_tracks.append( + build_extension_track( + key="adaptive-self-iteration", + label="Adaptive Self-Iteration", + status=adaptive_status, + objective="Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback.", + current=adaptive_current, + target="Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan.", + artifact_paths=[ + *adaptive_foundation_paths, + *adaptive_apply_paths, + "reports/user_patterns.json", + "reports/adaptation_proposals.json", + "reports/iteration-directions.md", + "reports/adoption_drift_report.md", + ], + next_action=adaptive_next_action, + skill_dir=skill_dir, + ) + ) + daily_skillops_paths = latest_daily_skillops_paths(skill_dir) + daily_skillops_ready = paths_exist(skill_dir, daily_skillops_paths) + daily_skillops_status = "covered" if daily_skillops_ready else "partial" + extension_tracks.append( + build_extension_track( + key="daily-skillops-report", + label="Daily SkillOps Report", + status=daily_skillops_status, + objective="Daily operations layer summarizes explicit-source conversation patterns, proposal-only adaptation work, approval state, release locks, and world-class evidence gaps.", + current=( + "Daily SkillOps report is a CLI-backed, explicit-source operations cockpit with tests and committed dated evidence." + if daily_skillops_ready + else "Adaptive scan/propose exists, but the daily operations report is not fully wired into scripts, tests, and evidence artifacts." + ), + target="Generate reports/skillops/daily/YYYY-MM-DD.* from explicit sources without private log scanning, source writes, auto-patching, or world-class overclaiming.", + artifact_paths=daily_skillops_paths, + next_action="Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves.", + skill_dir=skill_dir, + ) + ) + weekly_curator_paths = latest_weekly_curator_paths(skill_dir) + weekly_curator_ready = paths_exist(skill_dir, weekly_curator_paths) + weekly_curator_status = "covered" if weekly_curator_ready else "partial" + extension_tracks.append( + build_extension_track( + key="weekly-curator-report", + label="Weekly Curator Report", + status=weekly_curator_status, + objective="Weekly curator layer aggregates Daily SkillOps opportunities, Skill Atlas portfolio signals, release locks, and world-class evidence gaps into a maintenance queue.", + current=( + "Weekly curator report is CLI-backed, proposal-only, and backed by committed weekly JSON/Markdown evidence." + if weekly_curator_ready + else "Daily SkillOps and Skill Atlas exist, but weekly curator aggregation is not fully wired into scripts, tests, and evidence artifacts." + ), + target="Generate reports/skillops/weekly/YYYY-WNN.* without private log scanning, source writes, auto-patching, or world-class overclaiming.", + artifact_paths=weekly_curator_paths, + next_action="Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes.", + skill_dir=skill_dir, + ) + ) + extension_counts: dict[str, int] = {} + for item in extension_tracks: + extension_counts[item["status"]] = extension_counts.get(item["status"], 0) + 1 + + items = modules + pr_items + counts: dict[str, int] = {"pass": 0, "warn": 0, "missing": 0} + for item in items: + counts[item["status"]] = counts.get(item["status"], 0) + 1 + local_ready = counts.get("missing", 0) == 0 and counts.get("warn", 0) == 0 + public_ready = local_ready and summary_value(world_class_ledger, "ready_to_claim_world_class", False) is True + decision = "world-class-ready" if public_ready else ("local-blueprint-covered-evidence-pending" if local_ready else "continue-implementation") + summary = { + "item_count": len(items), + "module_count": len(modules), + "recommended_pr_count": len(pr_items), + "pass_count": counts.get("pass", 0), + "warn_count": counts.get("warn", 0), + "missing_count": counts.get("missing", 0), + "extension_track_count": len(extension_tracks), + "extension_partial_count": extension_counts.get("partial", 0), + "extension_planned_count": extension_counts.get("planned", 0), + "extension_covered_count": extension_counts.get("covered", 0), + "adaptive_extension_ready": adaptive_status == "covered", + "local_blueprint_ready": local_ready, + "public_world_class_ready": public_ready, + "world_class_evidence_pending_count": pending_world_class, + "decision": decision, + } + return { + "schema_version": "1.0", + "ok": counts.get("missing", 0) == 0, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": summary, + "status_counts": counts, + "extension_status_counts": extension_counts, + "modules": modules, + "recommended_prs": pr_items, + "reference_extension_tracks": extension_tracks, + "next_highest_leverage": [ + "Close the four world-class evidence ledger entries with accepted human or external evidence.", + "Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves.", + "Use Daily SkillOps with explicit curated sources to review adaptation proposals without scanning private logs or auto-patching.", + "Keep the blueprint coverage report in CI so 2.0 plan drift is visible.", + ], + "source_blueprint": { + "title": "Skill Overview / Skill OS 2.0 upgrade plan", + "core_module_count": 8, + "recommended_pr_count": 13, + "reference_extension_count": len(extension_tracks), + "reference_extensions": [ + "Skill interpretation report", + "Adaptive self-iteration with local preference memory and approval gates", + "Daily SkillOps operations report", + ], + }, + "artifacts": { + "json": "reports/skill_os2_coverage.json", + "markdown": "reports/skill_os2_coverage.md", + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render Skill OS 2.0 blueprint coverage evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/skill_os2_coverage.json") + parser.add_argument("--output-md", default="reports/skill_os2_coverage.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_coverage(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_skill_overview.py b/scripts/render_skill_overview.py index ddde13a1..e14d7d0b 100644 --- a/scripts/render_skill_overview.py +++ b/scripts/render_skill_overview.py @@ -5,174 +5,20 @@ import json from pathlib import Path from skill_report_charts import render_chart_set +from skill_report_i18n import ( + KIND_ZH, + LABEL_EN, + PACKAGE_LABEL_ZH, + bi_item, + bi_span, + en_for, + mode_zh, + readable_description_zh, +) from skill_report_layout import render_language_switch, render_report_nav, skill_overview_css, skill_overview_script from skill_report_model import REPORT_NAV_V2, build_report_model -TEXT_ZH = { - "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.": "从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。", - "Understand the request.": "理解用户请求。", - "Execute the main task.": "执行核心任务。", - "Validate the result.": "校验交付结果。", - "Understand the request": "理解用户请求。", - "Execute the main task": "执行核心任务。", - "Validate the result": "校验交付结果。", - "Decide whether the request should become a skill and choose the lightest fit.": "判断请求是否应该沉淀为 Skill，并选择最轻量可靠的模式。", - "Capture job, output, exclusions, constraints, and standards.": "捕捉任务、输出、排除项、约束和质量标准。", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.": "运行参考扫描：先看外部 benchmark，再看用户材料，最后校验本地适配；只暴露不确定性或冲突。", - "Write the `description` early and test route quality before expanding the package.": "尽早写出 `description`，先测试路由质量，再扩展包体。", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.": "只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。", - "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.": "当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。", - "Turn rough requests into a compact reusable demo skill.": "把粗糙请求整理成紧凑、可复用的演示 Skill。", - "Tighten trigger and exclusions": "收紧触发与排除边界", - "Add the first execution asset": "补上第一个执行资产", - "Promote from scaffold to production-ready": "从脚手架推进到生产可用", - "Borrow one proven pattern on purpose": "有选择地借鉴一个成熟模式", - "Harden portability semantics": "加固跨环境语义", - "Create an iteration evidence loop": "建立迭代证据回路", - "The package needs clearer near-neighbor exclusions before it grows.": "在继续扩展前，需要先把相邻但不应触发的场景说清楚。", - "The package is still mostly prose. Add one asset that removes repeated manual work.": "当前包体仍偏文本说明，应先增加一个能减少重复人工操作的资产。", - "The first version exists; the next gain usually comes from adding the smallest useful gates.": "第一版已经存在，下一步收益通常来自补上最小但有效的质量门禁。", - "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.": "已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。", - "The skill already signals reuse across environments, so contract clarity matters early.": "这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。", - "The package should show what changed and why after the first draft.": "第一版之后，包体应该能说明改了什么以及为什么改。", - "Add 3 to 5 should-trigger and should-not-trigger examples.": "增加 3 到 5 个应触发和不应触发的例子。", - "Refine the frontmatter description to name the recurring job and non-goals.": "精炼 frontmatter description，明确重复任务和非目标。", - "Run a first trigger evaluation pass before expanding the package.": "扩展包体前先跑一轮触发评估。", - "Move stable procedural guidance into references if users will need it repeatedly.": "如果用户会反复使用某段流程说明，把它沉淀到 references。", - "Create one deterministic helper script if a repeated step can be executed instead of described.": "如果某个重复步骤可以执行而不是描述，就沉淀成一个确定性 helper script。", - "Keep the main SKILL.md compact and route-oriented.": "保持主 SKILL.md 简洁，并围绕路由与入口组织。", - "Decide whether this skill is personal, team-reused, or library-grade.": "判断这个 Skill 是个人使用、团队复用，还是库级基础能力。", - "Add only the gates that match that risk level.": "只添加与风险等级匹配的质量门禁。", - "Record lifecycle metadata and review cadence once reuse becomes real.": "一旦进入真实复用，就记录生命周期元数据和评审节奏。", - "Decide whether to borrow method, structure, execution, or portability, but only one of them first.": "先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。", - "Record what you will not borrow so the package stays light.": "记录本轮不借鉴的内容，避免包体过重。", - "Confirm activation mode, execution context, and trust assumptions.": "确认激活模式、执行上下文和信任假设。", - "Add or review degradation strategy for non-native targets.": "补充或复核非原生目标端的降级策略。", - "Package the skill once to verify adapter expectations.": "至少打包一次 Skill，用来验证 adapter 预期。", - "Generate the HTML skill report and keep it aligned with the package.": "生成 HTML Skill 报告，并保持它与包体内容一致。", - "Record reference scan choices and non-goals.": "记录参考扫描的取舍和非目标。", - "Capture the next iteration choice explicitly before adding more files.": "在继续增加文件前，明确记录下一轮迭代选择。", - "Cleaner routing and fewer accidental activations.": "路由更清晰，误触发更少。", - "Stronger execution quality without bloating the entrypoint.": "在不膨胀入口文件的前提下提升执行质量。", - "A clearer path from exploratory package to maintained asset.": "更清晰地从探索性包体走向可维护资产。", - "A cleaner package shape with less accidental over-design.": "包体形态更清晰，也减少偶然过度设计。", - "Safer cross-environment reuse with less target drift.": "跨环境复用更安全，目标漂移更少。", - "A clearer path for the next author or reviewer.": "让下一位作者或评审者更容易接手。", -} - -TEXT_EN = { - "触发面保持精简，并锚定在 frontmatter description。": "The trigger surface stays lean and anchored in the frontmatter description.", - "已打包 agents/interface.yaml，便于后续做跨平台适配。": "Portable interface metadata is packaged for later adapter-based export.", - "长指导被拆到 references 中，入口文件可以保持轻量。": "Extended guidance is separated into references so the entrypoint can stay compact.", - "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。": "Deterministic helper logic lives in scripts instead of hidden prompt text.", - "包内包含可随 Skill 迁移的质量门禁或触发检查。": "The package includes portable quality gates or trigger checks.", - "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。": "Use this report to quickly understand the generated skill's role, principles, trigger boundary, and deliverables.", - "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。": "Clarify the recurring job, real input shape, and deliverable output before adding references, scripts, or evals.", - "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。": "If the request is still fuzzy, tighten the boundary through intent dialogue before expanding the package.", - "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。": "Output Review Adjudication is generated to record blind-review decisions, agreement rate, and pending cases.", - "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。": "Output Execution Runs is generated to distinguish recorded fixtures, command runs, and model-run evidence.", - "尚未生成盲评审定报告。": "The blind review adjudication report has not been generated yet.", - "尚未生成输出执行证据报告。": "The output execution evidence report has not been generated yet.", - "先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。": "Record the reviewer's A/B choice before opening the answer key and calculating agreement.", - "缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。": "When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.", - "recorded fixture 只能证明可复现样本，不等同于模型执行。": "A recorded fixture proves reproducible samples only; it is not model execution.", - "只有 provider runner 返回 model metadata 时才计入 model-executed。": "Only provider runners that return model metadata count as model-executed.", -} - -MODE_ZH = { - "scaffold": "脚手架", - "production": "生产", - "library": "库级", - "governed": "治理", - "manual": "手动", - "inline": "内联", - "agent-skills": "Agent Skills", -} - -PACKAGE_LABEL_ZH = { - "SKILL.md": "Skill 入口文件", - "README.md": "人类可读使用说明", - "agents/interface.yaml": "跨平台接口元数据", - "manifest.json": "生命周期与打包元数据", - "references": "扩展指导与复用资料", - "scripts": "确定性脚本或本地工具", - "evals": "触发与质量检查", - "reports": "生成的证据与总结报告", -} - -KIND_ZH = {"file": "文件", "folder": "目录"} - -LABEL_EN = { - "强项": "Strength", - "缺口": "Gap", - "保留并复用": "Keep", - "纳入下一轮修复": "Fix next", -} - - -def contains_cjk(text: str) -> bool: - return any("\u4e00" <= char <= "\u9fff" for char in str(text)) - - -def zh_for(text: str) -> str: - value = str(text).strip() - if not value: - return "" - if value in TEXT_ZH: - return TEXT_ZH[value] - if value in TEXT_EN or contains_cjk(value): - return value - if value.startswith("Use this skill when the request matches:"): - return "当用户请求与该 Skill 的触发描述匹配时使用。" - if value.startswith("用户说出类似需求时："): - return "当用户提出与该 Skill 触发描述相近的请求时使用。" - if value.startswith("Use $") and " when you need to " in value: - skill, need = value.removeprefix("Use ").split(" when you need to ", 1) - return f"当你需要{zh_for(need).rstrip('。')}时使用 `{skill}`。" - if value.startswith("Read the strongest pattern from "): - repo = value.removeprefix("Read the strongest pattern from ").rstrip(".") - return f"阅读 `{repo}` 中最值得借鉴的模式。" - if value.startswith("Primary prompt task family:"): - return "主要提示任务类型已记录在 prompt quality profile 中。" - if value.startswith("Complexity:"): - return "复杂度判断已记录在 prompt quality profile 中。" - if value.startswith("Stability:"): - return "系统稳定性评分已记录在 system model 中。" - if value.startswith("Owned job:"): - return "负责的核心任务已在 system model 中说明。" - if value.startswith("Leverage:"): - return "关键杠杆点已在 system model 中说明。" - return "原始说明可切换到英文查看；默认中文报告保留结论与结构说明。" - - -def en_for(text: str) -> str: - value = str(text).strip() - return TEXT_EN.get(value, value) - - -def bi_span(zh: str, en: str | None = None) -> str: - english = en if en is not None else en_for(zh) - return ( - f'{html.escape(str(zh))}' - f'{html.escape(str(english))}' - ) - - -def bi_item(text: str) -> str: - return bi_span(zh_for(text), en_for(text)) - - -def mode_zh(value: str) -> str: - return MODE_ZH.get(str(value), str(value)) - - -def readable_description_zh(description: str) -> str: - if contains_cjk(description): - return description - return "该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。" - - def render_list(items: list[str], class_name: str = "list") -> str: if not items: return f'
    • {bi_span("暂无记录。", "No records yet.")}
    ' @@ -192,7 +38,7 @@ def render_metric_cards(scorecard: dict) -> str: cards.append( "
    " "
    " - f"{html.escape(str(item.get('label', key)))}" + f"{bi_item(str(item.get('label', key)))}" f"{html.escape(str(item.get('score', 'n/a')))}" "
    " f"
    {reasons}
    " @@ -219,7 +65,7 @@ def render_metric_summary(scorecard: dict) -> str: items.append( "
  • " f"{bi_span(verdict, verdict_en)}" - f"{html.escape(label)}" + f"{bi_item(label)}" f"{score}" f"{bi_item(reason)}" "
    • " @@ -242,6 +88,118 @@ def render_audit_rows(items: list[dict]) -> str: return "".join(rows) +WORLD_CLASS_CHECK_COPY = { + "Provider model run": ("提供商实跑", "Provider model run"), + "Token usage observed": ("Token 用量", "Token usage observed"), + "No pending decisions": ("无待判定", "No pending decisions"), + "Judgments complete": ("盲评完成", "Judgments complete"), + "Native enforcement": ("原生执行", "Native enforcement"), + "External events": ("外部事件", "External events"), + "Adoption sample": ("采用样本", "Adoption sample"), +} + + +def render_world_class_check(check: object) -> str: + if isinstance(check, dict): + label_en = str(check.get("label_en") or check.get("label") or "") + label_zh = str(check.get("label_zh") or "") + else: + label_en = str(check) + label_zh = "" + label_zh, label_en = WORLD_CLASS_CHECK_COPY.get(label_en, (label_zh or label_en, label_en)) + return f"
  • {bi_span(label_zh, label_en)}
    • " + + +def render_world_class_readiness(readiness: dict) -> str: + if not readiness: + return "" + kpis = [ + ( + "待补证据", + "Pending", + readiness.get("pending_count", 0), + "仍需外部或人工证据接受。", + "External or human evidence still needs acceptance.", + ), + ( + "已接受", + "Accepted", + readiness.get("accepted_count", 0), + "已通过 source check 与提交契约。", + "Passed source checks and submission contract.", + ), + ( + "源检查", + "Source Checks", + f"{readiness.get('source_pass_count', 0)} / {readiness.get('source_check_count', 0)}", + "通过数 / 总检查数。", + "Passed checks / total checks.", + ), + ] + kpi_html = "".join( + "
    " + f"{bi_span(zh, en)}" + f"{html.escape(str(value))}" + f"{bi_span(note_zh, note_en)}" + "
    " + for zh, en, value, note_zh, note_en in kpis + ) + entries = readiness.get("entries", []) + if entries: + blocks = [] + for item in entries: + blocked_checks = [ + str(check).strip() + for check in item.get("blocked_checks", []) + if str(check).strip() + ] + if blocked_checks: + checks_html = ( + "
      " + + "".join(render_world_class_check(check) for check in blocked_checks) + + "
    " + ) + else: + checks_html = ( + "

    " + f"{bi_span('当前没有阻塞检查。', 'No blocked checks right now.')}" + "

    " + ) + blocks.append( + "
    " + "
    " + f"{bi_span(str(item.get('category_zh', '外部证据')), str(item.get('category_en', 'External evidence')))}" + f"

    {bi_span(str(item.get('label_zh', item.get('key', '证据项'))), str(item.get('label_en', item.get('key', 'Evidence item'))))}

    " + "
    " + f"

    {bi_span(str(item.get('summary_zh', '仍待补充证据。')), str(item.get('summary_en', 'Evidence is still pending.')))}

    " + f"
    {bi_span('阻塞检查', 'Blocked Checks')}
    " + f"{checks_html}" + "
    " + ) + entry_html = "".join(blocks) + else: + entry_html = ( + "
    " + f"

    {bi_span('尚未生成 world-class ledger；这里只保留反过度承诺提示。', 'No world-class ledger has been generated; this panel keeps the anti-overclaim guard visible.')}

    " + "
    " + ) + status = "可宣称" if readiness.get("ready") else "证据待补" + status_en = "Claim-ready" if readiness.get("ready") else "Evidence pending" + return ( + "
    " + "
    " + "
    " + f"

    {bi_span('世界证据', 'World Evidence')}

    " + f"

    {bi_span(str(readiness.get('conclusion_zh', '世界级证据状态未知。')), str(readiness.get('conclusion_en', 'World-class evidence status is unknown.')))}

    " + "
    " + f"{bi_span(status, status_en)}" + "
    " + f"
    {kpi_html}
    " + f"
    {entry_html}
    " + "
    " + ) + + def render_score_strip(scorecard: dict) -> str: keys = ["completeness_score", "trigger_score", "evidence_score", "context_cost"] cards = [] @@ -253,7 +211,7 @@ def render_score_strip(scorecard: dict) -> str: reason = item.get("reasons", [""])[0] cards.append( "
    " - f"{html.escape(str(item.get('label', key)))}" + f"{bi_item(str(item.get('label', key)))}" f"{score}" f"" f"{bi_item(str(reason))}" @@ -289,12 +247,16 @@ def render_html(summary: dict) -> str: contract = summary.get("contract_boundary", {}) quality = summary.get("quality_review", {}) risk = summary.get("risk_governance", {}) + world_readiness = summary.get("world_class_readiness", {}) assets = summary.get("package_assets", {}) roadmap = summary.get("iteration_roadmap", {}) output_execution = summary.get("output_execution", {}) output_execution_summary = output_execution.get("summary", {}) output_review = summary.get("output_review_adjudication", {}) output_review_summary = output_review.get("summary", {}) + report_contract = summary.get("report_contract", {}) + html_report_path = str(report_contract.get("html_report") or "reports/skill-overview.html") + open_report_message = f"创建完成后建议先打开 {html_report_path}，再继续扩展包体。" hero_meta = [ (f"技能名称：{summary['name']}", f"Skill name: {summary['name']}"), (f"成熟度：{mode_zh(metadata.get('maturity_tier', 'scaffold'))}", f"Maturity: {metadata.get('maturity_tier', 'scaffold')}"), @@ -375,13 +337,13 @@ def render_html(summary: dict) -> str:

    {bi_span("YAO Skill 生成审计报告", "YAO Skill Generation Audit")}

    {html.escape(summary['name'])}

    {bi_span("技能审计报告", f"{summary['display_name']} Audit Report")}

    -

    {bi_span("这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。", summary["description"])}

    +

    {bi_span("这份报告默认使用中文简体，把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。", en_for(str(summary["description"])))}

    {hero_meta_html}
    {target_badges}
    {score_strip}
    @@ -409,17 +371,24 @@ def render_html(summary: dict) -> str:

    {bi_span("总览指标", "Metrics")}

    {bi_span("分数来自本地文件和 reports 证据，缺失时明确标为证据不足。", "Scores are derived from package files and reports; missing inputs are shown as evidence gaps.")}

    -
    -
    -
    - {charts["radar"]} -
    -

    {bi_span("指标判读", "Reading")}

    -

    {bi_span("先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。", "Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.")}

    - {render_metric_summary(scorecard)} -
    -
    -
    {render_metric_cards(scorecard)}
    +
    +

    {bi_span("指标判读", "Reading")}

    +

    {bi_span("先看雷达图判断能力短板，再看每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。", "Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.")}

    +
    +
    +
    +
    +
    + {charts["radar"]} +
    +

    {bi_span("成熟度条", "Maturity Bar")}

    +

    {bi_span("这里把每个指标压缩为状态、分数和第一条证据，先给出整体判断，再进入下方明细。", "This compresses each metric into status, score, and the first evidence item before the detailed cards below.")}

    + {render_metric_summary(scorecard)} +
    +
    +
    +
    {bi_span("指标明细", "Metric Details")}
    +
    {render_metric_cards(scorecard)}
    @@ -472,7 +441,7 @@ def render_html(summary: dict) -> str:

    {bi_span("触发描述", "Trigger")}

    -

    {bi_span(readable_description_zh(str(trigger.get("description", ""))), str(trigger.get("description", "")))}

    +

    {bi_span(readable_description_zh(str(trigger.get("description", ""))), en_for(str(trigger.get("description", ""))))}

    {bi_span("输入材料", "Inputs")}

    {render_list(contract.get("inputs", []))}
    @@ -536,6 +505,7 @@ def render_html(summary: dict) -> str:
    + {render_world_class_readiness(world_readiness)} diff --git a/scripts/render_social_preview.py b/scripts/render_social_preview.py index 7b232179..73469a5c 100644 --- a/scripts/render_social_preview.py +++ b/scripts/render_social_preview.py @@ -4,8 +4,6 @@ from __future__ import annotations import argparse from pathlib import Path -from PIL import Image, ImageDraw, ImageFont - WIDTH = 1280 HEIGHT = 640 @@ -20,15 +18,19 @@ PNG_PATH = OUT_DIR / "social-preview.png" SVG_PATH = OUT_DIR / "social-preview.svg" -def font(path: str, size: int) -> ImageFont.FreeTypeFont: - return ImageFont.truetype(path, size) - - SERIF = "/System/Library/Fonts/Supplemental/Georgia.ttf" SANS = "/System/Library/Fonts/SFNS.ttf" def draw_png() -> None: + try: + from PIL import Image, ImageDraw, ImageFont + except ImportError as exc: + raise SystemExit("Pillow is required to render PNG social previews.") from exc + + def font(path: str, size: int): + return ImageFont.truetype(path, size) + OUT_DIR.mkdir(parents=True, exist_ok=True) image = Image.new("RGB", (WIDTH, HEIGHT), BACKGROUND) draw = ImageDraw.Draw(image) diff --git a/scripts/render_telemetry_hook_recipes.py b/scripts/render_telemetry_hook_recipes.py new file mode 100644 index 00000000..cafee93f --- /dev/null +++ b/scripts/render_telemetry_hook_recipes.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +import argparse +import json +import shlex +from pathlib import Path +from typing import Any + +from render_adoption_drift_report import SENSITIVE_FIELDS, display_path + + +SCHEMA_VERSION = "1.0" +DEFAULT_RECIPES = [ + { + "id": "browser-extension", + "client": "Browser extension", + "command": "browser-extension", + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "trigger_points": [ + "Skill is explicitly selected by the user.", + "Skill activation is accepted by the host client.", + ], + }, + { + "id": "chrome-extension", + "client": "Chrome extension", + "command": "chrome-extension", + "event": "skill_output", + "activation_type": "manual", + "outcome": "edited", + "failure_type": "none", + "trigger_points": [ + "User keeps or edits a skill-generated artifact.", + "Extension records only the final outcome class, not page content.", + ], + }, + { + "id": "vscode-extension", + "client": "VS Code extension", + "command": "vscode-extension", + "event": "skill_activation", + "activation_type": "implicit", + "outcome": "accepted", + "failure_type": "none", + "trigger_points": [ + "Workspace skill route chooses this skill.", + "Extension records route outcome without file paths or code snippets.", + ], + }, + { + "id": "cli-wrapper", + "client": "CLI wrapper", + "command": "cli-wrapper", + "event": "script_run", + "activation_type": "manual", + "outcome": "unknown", + "failure_type": "none", + "trigger_points": [ + "Wrapper starts or finishes a known skill workflow command.", + "Wrapper records command family only, never command arguments.", + ], + }, + { + "id": "provider-adapter", + "client": "Provider adapter", + "command": "provider-adapter", + "event": "skill_output", + "activation_type": "manual", + "outcome": "accepted", + "failure_type": "none", + "trigger_points": [ + "Provider adapter receives an accepted or rejected skill output signal.", + "Adapter records only normalized outcome metadata.", + ], + }, +] + + +def default_spool(skill_dir: Path) -> Path: + return skill_dir / ".yao" / "telemetry_spool" / "external_events.jsonl" + + +def command_text(argv: list[str]) -> str: + return shlex.join(argv) + + +def emit_argv(skill_dir: Path, output_jsonl: Path, recipe: dict[str, Any], dry_run: bool = False) -> list[str]: + argv = [ + "python3", + "scripts/yao.py", + "telemetry-emit", + display_path(skill_dir), + "--output-jsonl", + display_path(output_jsonl), + "--event", + recipe["event"], + "--activation-type", + recipe["activation_type"], + "--outcome", + recipe["outcome"], + "--failure-type", + recipe["failure_type"], + "--command", + recipe["command"], + ] + if dry_run: + argv.append("--dry-run") + return argv + + +def import_argv(skill_dir: Path, output_jsonl: Path) -> list[str]: + return [ + "python3", + "scripts/yao.py", + "telemetry-import", + display_path(skill_dir), + "--input-jsonl", + display_path(output_jsonl), + ] + + +def build_recipe(skill_dir: Path, output_jsonl: Path, recipe: dict[str, Any]) -> dict[str, Any]: + emit = emit_argv(skill_dir, output_jsonl, recipe) + dry_run = emit_argv(skill_dir, output_jsonl, recipe, dry_run=True) + return { + **recipe, + "source": "external", + "native_auto_capture": False, + "integration_status": "client-hook-recipe", + "metadata_only": True, + "emit_argv": emit, + "emit_command": command_text(emit), + "dry_run_argv": dry_run, + "dry_run_command": command_text(dry_run), + } + + +def render_markdown(report: dict[str, Any]) -> str: + lines = [ + "# Telemetry Hook Recipes", + "", + "These recipes show how Browser, Chrome, IDE, wrapper, or provider-side integrations can call `telemetry-emit` without collecting raw prompts, outputs, transcripts, messages, notes, arguments, or file content.", + "", + "They are client hook recipes, not proof that a host client is already natively integrated.", + "", + "## Summary", + "", + f"- Recipe count: `{report['summary']['recipe_count']}`", + f"- Native auto-capture integrations claimed: `{report['summary']['native_auto_capture_count']}`", + f"- Default spool: `{report['artifacts']['spool_jsonl']}`", + "", + "## Recipes", + "", + "| Client | Command | Event | Outcome | Dry run |", + "| --- | --- | --- | --- | --- |", + ] + for recipe in report["recipes"]: + lines.append( + f"| {recipe['client']} | `{recipe['command']}` | `{recipe['event']}` | `{recipe['outcome']}` | `{recipe['dry_run_command']}` |" + ) + lines.extend( + [ + "", + "## Import", + "", + "After a client finishes a batch, import the local spool:", + "", + "```bash", + report["artifacts"]["import_command"], + "```", + "", + "## Privacy Contract", + "", + "- Allowed fields: `event`, `skill`, `version`, `source`, `command`, `activation_type`, `outcome`, `failure_type`, `timestamp`.", + f"- Blocked raw-content fields: `{', '.join(report['privacy_contract']['blocked_fields'])}`.", + "- Client integrations must map local state to normalized outcome classes before calling the hook.", + ] + ) + return "\n".join(lines) + "\n" + + +def render_recipes( + skill_dir: Path, + output_json: Path | None = None, + output_md: Path | None = None, + output_jsonl: Path | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + output_json = (output_json or reports_dir / "telemetry_hook_recipes.json").resolve() + output_md = (output_md or reports_dir / "telemetry_hook_recipes.md").resolve() + output_jsonl = (output_jsonl or default_spool(skill_dir)).resolve() + recipes = [build_recipe(skill_dir, output_jsonl, recipe) for recipe in DEFAULT_RECIPES] + import_cmd = import_argv(skill_dir, output_jsonl) + report = { + "ok": True, + "schema_version": SCHEMA_VERSION, + "skill_dir": display_path(skill_dir), + "summary": { + "recipe_count": len(recipes), + "native_auto_capture_count": sum(1 for recipe in recipes if recipe["native_auto_capture"]), + "metadata_only_recipe_count": sum(1 for recipe in recipes if recipe["metadata_only"]), + }, + "privacy_contract": { + "raw_content_allowed": False, + "blocked_fields": sorted(SENSITIVE_FIELDS), + "allowed_fields": [ + "event", + "skill", + "version", + "source", + "command", + "activation_type", + "outcome", + "failure_type", + "timestamp", + ], + }, + "recipes": recipes, + "artifacts": { + "json": display_path(output_json), + "markdown": display_path(output_md), + "spool_jsonl": display_path(output_jsonl), + "import_command": command_text(import_cmd), + "import_argv": import_cmd, + }, + "warnings": [ + "These recipes do not prove native host-client integration; they define the local metadata-only hook contract for those integrations." + ], + "failures": [], + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render metadata-only telemetry client hook recipes.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--output-jsonl") + args = parser.parse_args() + report = render_recipes( + Path(args.skill_dir), + output_json=Path(args.output_json) if args.output_json else None, + output_md=Path(args.output_md) if args.output_md else None, + output_jsonl=Path(args.output_jsonl) if args.output_jsonl else None, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_weekly_curator_report.py b/scripts/render_weekly_curator_report.py new file mode 100644 index 00000000..06b4092c --- /dev/null +++ b/scripts/render_weekly_curator_report.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from collections import Counter +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a weekly SkillOps curator report from generated reports without scanning private logs or writing source files." + +SUMMARY_FIELDS = [ + "decision", + "week_id", + "daily_report_count", + "opportunity_count", + "unique_opportunity_count", + "ready_for_approval_review_count", + "proposal_review_count", + "observe_more_evidence_count", + "report_only_count", + "top_score", + "skill_count", + "actionable_portfolio_issue_count", + "release_lock_ready", + "evidence_consistency_ok", + "public_world_class_ready", + "world_class_pending_count", + "writes_source_files", + "auto_patch_enabled", + "failure_count", +] + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def parse_date(value: str) -> date: + candidate = value[:10] if len(value) >= 10 else value + try: + return date.fromisoformat(candidate) + except ValueError: + return date.today() + + +def week_id(generated_at: str) -> str: + iso = parse_date(generated_at).isocalendar() + return f"{iso.year}-W{iso.week:02d}" + + +def display_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return f"[external-explicit-source]/{path.name}" + + +def resolve_output(skill_dir: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else skill_dir / path + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def default_weekly_path(skill_dir: Path, generated_at: str, suffix: str) -> Path: + return skill_dir / "reports" / "skillops" / "weekly" / f"{week_id(generated_at)}.{suffix}" + + +def daily_report_paths(skill_dir: Path, explicit_paths: list[Path]) -> list[Path]: + if explicit_paths: + return [path if path.is_absolute() else skill_dir / path for path in explicit_paths] + daily_dir = skill_dir / "reports" / "skillops" / "daily" + if not daily_dir.exists(): + return [] + return sorted(daily_dir.glob("*.json")) + + +def _as_int(value: Any, default: int = 0) -> int: + if isinstance(value, bool): + return default + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _dedupe_opportunities(daily_reports: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_id: dict[str, dict[str, Any]] = {} + for report in daily_reports: + report_date = str(report.get("generated_at") or "")[:10] + for item in report.get("opportunities", []): + if not isinstance(item, dict): + continue + opportunity_id = str(item.get("opportunity_id") or "") + if not opportunity_id: + continue + current = {**item, "daily_report_date": report_date} + existing = by_id.get(opportunity_id) + if existing is None or _as_int(current.get("score")) > _as_int(existing.get("score")): + by_id[opportunity_id] = current + return sorted(by_id.values(), key=lambda item: (-_as_int(item.get("score")), str(item.get("opportunity_id")))) + + +def _counter(items: list[dict[str, Any]], key: str) -> dict[str, int]: + values = Counter(str(item.get(key) or "") for item in items) + values.pop("", None) + return dict(sorted(values.items())) + + +def _portfolio_issue_count(atlas_summary: dict[str, Any]) -> int: + keys = [ + "actionable_route_collision_count", + "actionable_owner_gap_count", + "actionable_stale_count", + "actionable_drift_signal_count", + "no_route_opportunity_count", + ] + return sum(_as_int(atlas_summary.get(key)) for key in keys) + + +def build_actions(summary: dict[str, Any], opportunities: list[dict[str, Any]], atlas_summary: dict[str, Any]) -> list[dict[str, str]]: + actions: list[dict[str, str]] = [] + if summary["ready_for_approval_review_count"]: + actions.append( + { + "key": "review-ready-opportunities", + "priority": "high", + "action": "Review the top ready-for-approval SkillOps opportunities before preparing any approval ledger entry.", + } + ) + elif opportunities: + actions.append( + { + "key": "review-proposal-queue", + "priority": "medium", + "action": "Review proposal-level SkillOps opportunities and keep low-evidence items in observation.", + } + ) + if summary["actionable_portfolio_issue_count"]: + actions.append( + { + "key": "triage-skill-library", + "priority": "high", + "action": "Triage actionable route collisions, owner gaps, stale skills, drift signals, or no-route opportunities from Skill Atlas.", + } + ) + if summary["world_class_pending_count"]: + actions.append( + { + "key": "close-world-class-evidence", + "priority": "high", + "action": "Collect accepted external or human evidence for pending world-class ledger entries before public claims.", + } + ) + if summary["evidence_consistency_ok"] is not True: + actions.append( + { + "key": "refresh-evidence-consistency", + "priority": "high", + "action": "Regenerate evidence consistency before using this curator report for release decisions.", + } + ) + if not atlas_summary: + actions.append( + { + "key": "refresh-skill-atlas", + "priority": "medium", + "action": "Generate Skill Atlas so weekly curation can see portfolio-level drift and stale-skill signals.", + } + ) + if not actions: + actions.append({"key": "monitor", "priority": "low", "action": "No curation action required beyond routine monitoring."}) + return actions + + +def build_report(skill_dir: Path, generated_at: str, daily_json: list[Path] | None = None) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + failures: list[str] = [] + + daily_paths = daily_report_paths(skill_dir, daily_json or []) + daily_reports = [] + for path in daily_paths: + payload = load_json(path) + if payload.get("ok") is True: + daily_reports.append(payload) + else: + failures.append(f"Daily SkillOps report is missing or invalid: {display_path(path, skill_dir)}") + + opportunities = _dedupe_opportunities(daily_reports) + decision_counts = _counter(opportunities, "decision") + action_counts = _counter(opportunities, "action_type") + risk_counts = _counter(opportunities, "risk_level") + + atlas = load_json(reports_dir / "skill_atlas.json") + atlas_summary = atlas.get("summary", {}) if isinstance(atlas.get("summary"), dict) else {} + benchmark = load_json(reports_dir / "benchmark_reproducibility.json") + benchmark_summary = benchmark.get("summary", {}) if isinstance(benchmark.get("summary"), dict) else {} + consistency = load_json(reports_dir / "evidence_consistency.json") + ledger = load_json(reports_dir / "world_class_evidence_ledger.json") + ledger_summary = ledger.get("summary", {}) if isinstance(ledger.get("summary"), dict) else {} + + portfolio_issue_count = _portfolio_issue_count(atlas_summary) + failure_count = len(failures) + summary = { + "decision": "blocked" + if failure_count + else "curator-review" + if opportunities or portfolio_issue_count or _as_int(ledger_summary.get("pending_count")) + else "monitor", + "week_id": week_id(generated_at), + "daily_report_count": len(daily_reports), + "opportunity_count": sum(_as_int(report.get("opportunity_summary", {}).get("opportunity_count")) for report in daily_reports), + "unique_opportunity_count": len(opportunities), + "ready_for_approval_review_count": decision_counts.get("ready_for_approval_review", 0), + "proposal_review_count": decision_counts.get("proposal_review", 0), + "observe_more_evidence_count": decision_counts.get("observe_more_evidence", 0), + "report_only_count": decision_counts.get("report_only", 0), + "top_score": _as_int(opportunities[0].get("score")) if opportunities else 0, + "skill_count": _as_int(atlas_summary.get("skill_count")), + "actionable_portfolio_issue_count": portfolio_issue_count, + "release_lock_ready": benchmark_summary.get("release_lock_ready") is True, + "evidence_consistency_ok": consistency.get("ok") is True, + "public_world_class_ready": ledger_summary.get("ready_to_claim_world_class") is True, + "world_class_pending_count": _as_int(ledger_summary.get("pending_count")), + "writes_source_files": False, + "auto_patch_enabled": False, + "failure_count": failure_count, + } + actions = build_actions(summary, opportunities, atlas_summary) + curator_contract = { + "schema_version": "1.0", + "contract": "weekly-skillops-curator-report", + "cadence": "weekly", + "source_of_truth": [ + "reports/skillops/daily/*.json", + "reports/skill_atlas.json", + "reports/benchmark_reproducibility.json", + "reports/evidence_consistency.json", + "reports/world_class_evidence_ledger.json", + ], + "raw_content_stored": False, + "redacted_or_generated_evidence_only": True, + "proposal_only": True, + "writes_source_files": False, + "auto_patch_enabled": False, + "approval_required_for_writes": True, + "counts_as_world_class_evidence": False, + } + report = { + "schema_version": "1.0", + "ok": failure_count == 0, + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + **{key: summary[key] for key in SUMMARY_FIELDS}, + "summary": summary, + "curator_contract": curator_contract, + "report_contract": { + "schema_version": "1.0", + "contract": "weekly-skillops-curator-report", + "top_level_mirrors_summary": True, + "summary_fields": SUMMARY_FIELDS, + }, + "daily_reports": [display_path(path, skill_dir) for path in daily_paths], + "opportunity_summary": { + "action_type_counts": action_counts, + "decision_counts": decision_counts, + "risk_level_counts": risk_counts, + }, + "curator_queue": opportunities[:20], + "portfolio": { + "skill_count": summary["skill_count"], + "actionable_issue_count": portfolio_issue_count, + "route_collision_count": _as_int(atlas_summary.get("route_collision_count")), + "actionable_route_collision_count": _as_int(atlas_summary.get("actionable_route_collision_count")), + "owner_gap_count": _as_int(atlas_summary.get("owner_gap_count")), + "actionable_owner_gap_count": _as_int(atlas_summary.get("actionable_owner_gap_count")), + "stale_count": _as_int(atlas_summary.get("stale_count")), + "actionable_stale_count": _as_int(atlas_summary.get("actionable_stale_count")), + "drift_signal_count": _as_int(atlas_summary.get("drift_signal_count")), + "actionable_drift_signal_count": _as_int(atlas_summary.get("actionable_drift_signal_count")), + "no_route_opportunity_count": _as_int(atlas_summary.get("no_route_opportunity_count")), + }, + "release_state": { + "release_lock_ready": summary["release_lock_ready"], + "evidence_consistency_ok": summary["evidence_consistency_ok"], + "public_world_class_ready": summary["public_world_class_ready"], + "world_class_pending_count": summary["world_class_pending_count"], + }, + "actions": actions, + "failures": failures, + "source_reports": { + "daily_dir": "reports/skillops/daily", + "skill_atlas": "reports/skill_atlas.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", + "evidence_consistency": "reports/evidence_consistency.json", + "world_class_ledger": "reports/world_class_evidence_ledger.json", + }, + "artifacts": { + "json": str(default_weekly_path(skill_dir, generated_at, "json").relative_to(skill_dir)), + "markdown": str(default_weekly_path(skill_dir, generated_at, "md").relative_to(skill_dir)), + }, + } + return report + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Weekly SkillOps Curator Report", + "", + f"Generated at: `{report['generated_at']}`", + f"Week: `{summary['week_id']}`", + "", + "## Summary", + "", + ] + for key in [ + "decision", + "daily_report_count", + "unique_opportunity_count", + "ready_for_approval_review_count", + "proposal_review_count", + "top_score", + "skill_count", + "actionable_portfolio_issue_count", + "release_lock_ready", + "evidence_consistency_ok", + "public_world_class_ready", + "world_class_pending_count", + ]: + lines.append(f"- {key}: `{summary[key]}`") + lines.extend( + [ + "", + "This report is a weekly curator cockpit for generated SkillOps evidence. It does not scan private logs, write source files, apply patches, or count as world-class evidence.", + "", + "## Curator Boundary", + "", + ] + ) + contract = report["curator_contract"] + for key in [ + "raw_content_stored", + "redacted_or_generated_evidence_only", + "proposal_only", + "writes_source_files", + "auto_patch_enabled", + "approval_required_for_writes", + "counts_as_world_class_evidence", + ]: + lines.append(f"- {key}: `{str(contract[key]).lower()}`") + lines.extend(["", "## Actions", ""]) + for action in report["actions"]: + lines.append(f"- `{action['priority']}` {action['action']}") + lines.extend(["", "## Curator Queue", ""]) + if not report["curator_queue"]: + lines.append("- No weekly curator opportunities.") + for item in report["curator_queue"]: + lines.extend( + [ + f"### {item.get('title', '')}", + "", + f"- ID: `{item.get('opportunity_id', '')}`", + f"- Action: `{item.get('action_type', '')}`", + f"- Decision: `{item.get('decision', '')}`", + f"- Score: `{item.get('score', 0)}`", + f"- Risk: `{item.get('risk_level', '')}`", + f"- Source day: `{item.get('daily_report_date', '')}`", + f"- Policy: {item.get('policy_reason', '')}", + "", + ] + ) + lines.extend(["## Portfolio Signals", ""]) + for key, value in report["portfolio"].items(): + lines.append(f"- {key}: `{value}`") + lines.extend(["", "## Evidence", ""]) + for label, path in report["source_reports"].items(): + lines.append(f"- {label}: `{path}`") + if report["failures"]: + lines.extend(["", "## Failures", ""]) + lines.extend(f"- {failure}" for failure in report["failures"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a weekly SkillOps curator report from generated SkillOps evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--daily-json", action="append", default=[], help="Explicit Daily SkillOps JSON report to include.") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--generated-at", default=utc_now()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, args.generated_at, [Path(item) for item in args.daily_json]) + output_json = resolve_output(skill_dir, args.output_json) if args.output_json else default_weekly_path(skill_dir, args.generated_at, "json") + output_md = resolve_output(skill_dir, args.output_md) if args.output_md else default_weekly_path(skill_dir, args.generated_at, "md") + report["artifacts"] = { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + } + write_json(output_json, report) + write_text(output_md, render_markdown(report)) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_claim_guard.py b/scripts/render_world_class_claim_guard.py new file mode 100644 index 00000000..b809caf3 --- /dev/null +++ b/scripts/render_world_class_claim_guard.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_ledger import build_ledger + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists." + +FORBIDDEN_WHEN_PENDING = [ + { + "key": "ready-to-claim-true", + "pattern": re.compile(r"ready[\s_-]*to[\s_-]*claim[\s_-]*world[\s_-]*class\s*[:=]\s*`?true`?", re.IGNORECASE), + "reason": "ready_to_claim_world_class cannot be true while ledger evidence is pending.", + }, + { + "key": "world-class-ready-true", + "pattern": re.compile(r"(public\s+)?world[\s_-]*class[\s_-]*ready\s*[:=]\s*`?true`?", re.IGNORECASE), + "reason": "world-class readiness cannot be claimed before accepted external and human evidence exists.", + }, + { + "key": "json-world-class-ready-true", + "pattern": re.compile(r'"(?:ready_to_claim_world_class|world_class_ready|public_world_class_ready)"\s*:\s*true', re.IGNORECASE), + "reason": "machine-readable claim fields must remain false until the ledger is ready.", + }, + { + "key": "completion-phrase", + "pattern": re.compile(r"(public\s+)?world[\s_-]*class\s+(?:complete|completed|achieved|certified)", re.IGNORECASE), + "reason": "completion language is blocked until the world-class ledger is accepted.", + }, + { + "key": "zh-completion-phrase", + "pattern": re.compile(r"世界级(?:已完成|完成|已达成|达成|准备就绪|可声明|认证通过)"), + "reason": "中文完成态表述必须等到 ledger ready 后才能出现。", + }, +] + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def resolve_claim_surface(raw_path: str, skill_dir: Path) -> Path: + path = Path(raw_path) + if not path.is_absolute(): + path = skill_dir / path + return path.resolve() + + +def should_scan_claim_surface(path: Path) -> bool: + parts = path.parts + if "release_snapshots" in parts: + return False + if path.name in {"world_class_claim_guard.md", "world_class_claim_guard.html"}: + return False + if "dist" in parts and "install-simulation" in parts: + return False + if "evidence" in parts and "world_class" in parts and "submissions" in parts: + return False + return True + + +def default_claim_surfaces(skill_dir: Path) -> list[Path]: + paths: list[Path] = [] + for rel in ["README.md", "SKILL.md", "manifest.json"]: + path = skill_dir / rel + if path.exists(): + paths.append(path) + for directory, patterns in [ + (skill_dir / "agents", ["*.yaml", "*.yml", "*.json"]), + (skill_dir / "docs", ["*.md"]), + (skill_dir / "evidence" / "world_class", ["*.md", "*.json"]), + (skill_dir / "dist", ["*.json", "*.md"]), + (skill_dir / "reports", ["*.md", "*.html", "*.json"]), + (skill_dir / "registry", ["*.json"]), + (skill_dir / "security", ["*.md", "*.json"]), + (skill_dir / "skill-ir", ["*.json"]), + ]: + if not directory.exists(): + continue + for pattern in patterns: + paths.extend( + path + for path in directory.rglob(pattern) + if path.is_file() and should_scan_claim_surface(path) + ) + return sorted(set(paths), key=lambda item: rel_path(item, skill_dir)) + + +def line_matches(path: Path, ledger_ready: bool) -> list[dict[str, Any]]: + if ledger_ready: + return [] + violations: list[dict[str, Any]] = [] + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return violations + for lineno, line in enumerate(lines, start=1): + for rule in FORBIDDEN_WHEN_PENDING: + if rule["pattern"].search(line): + violations.append( + { + "line": lineno, + "rule": rule["key"], + "reason": rule["reason"], + "excerpt": line.strip()[:240], + } + ) + return violations + + +def escape_markdown_table_cell(value: Any) -> str: + return str(value).replace("|", "\\|") + + +def build_guard(skill_dir: Path, generated_at: str, claim_surfaces: list[Path] | None = None) -> dict[str, Any]: + ledger = build_ledger(skill_dir, generated_at) + ledger_ready = ledger.get("summary", {}).get("ready_to_claim_world_class") is True + surfaces = claim_surfaces or default_claim_surfaces(skill_dir) + scanned = [] + violations = [] + for path in surfaces: + matches = line_matches(path, ledger_ready) + rel = rel_path(path, skill_dir) + scanned.append({"path": rel, "violation_count": len(matches)}) + for match in matches: + violations.append({"path": rel, **match}) + ok = len(violations) == 0 + json_surface_count = sum(1 for item in scanned if item["path"].endswith(".json")) + metadata_surface_count = sum( + 1 for item in scanned if item["path"].endswith((".json", ".yaml", ".yml")) + ) + package_surface_count = sum( + 1 + for item in scanned + if item["path"] == "manifest.json" + or item["path"].startswith(("agents/", "dist/", "security/")) + ) + return { + "schema_version": "1.0", + "ok": ok, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "ledger_ready_to_claim_world_class": ledger_ready, + "ledger_pending_count": ledger.get("summary", {}).get("pending_count", 0), + "claim_surface_count": len(scanned), + "json_claim_surface_count": json_surface_count, + "metadata_claim_surface_count": metadata_surface_count, + "package_claim_surface_count": package_surface_count, + "violation_count": len(violations), + "overclaim_guard_active": True, + "decision": ( + "claim-ready" + if ledger_ready and ok + else ("claim-blocked-overclaim" if violations else "claim-guard-pass-evidence-pending") + ), + }, + "rules": [ + {"key": item["key"], "reason": item["reason"], "pattern": item["pattern"].pattern} + for item in FORBIDDEN_WHEN_PENDING + ], + "scanned_surfaces": scanned, + "violations": violations, + "source_ledger": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + }, + "artifacts": { + "json": "reports/world_class_claim_guard.json", + "markdown": "reports/world_class_claim_guard.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Claim Guard", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ledger ready to claim world-class: `{str(summary['ledger_ready_to_claim_world_class']).lower()}`", + f"- ledger pending evidence: `{summary['ledger_pending_count']}`", + f"- claim surfaces scanned: `{summary['claim_surface_count']}`", + f"- JSON claim surfaces scanned: `{summary['json_claim_surface_count']}`", + f"- metadata claim surfaces scanned: `{summary['metadata_claim_surface_count']}`", + f"- package/runtime claim surfaces scanned: `{summary['package_claim_surface_count']}`", + f"- violations: `{summary['violation_count']}`", + f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`", + "", + "This guard scans public claim surfaces, machine-readable reports, and package/runtime metadata for completion language that would contradict the world-class evidence ledger. It allows evidence planning and pending-state language, but blocks completion claims until the ledger is ready.", + "", + "## Violations", + "", + "| Path | Line | Rule | Excerpt |", + "| --- | ---: | --- | --- |", + ] + if report["violations"]: + for item in report["violations"]: + excerpt = str(item["excerpt"]).replace("|", "\\|") + lines.append(f"| `{item['path']}` | {item['line']} | `{item['rule']}` | {excerpt} |") + else: + lines.append("| `none` | 0 | `none` | none |") + lines.extend( + [ + "", + "## Rules", + "", + "| Rule | Reason |", + "| --- | --- |", + ] + ) + for rule in report["rules"]: + reason = escape_markdown_table_cell(rule["reason"]) + lines.append(f"| `{rule['key']}` | {reason} |") + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Guard public world-class completion claims against the evidence ledger.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--claim-surface", action="append", default=[]) + parser.add_argument("--output-json", default="reports/world_class_claim_guard.json") + parser.add_argument("--output-md", default="reports/world_class_claim_guard.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + claim_surfaces = [resolve_claim_surface(item, skill_dir) for item in args.claim_surface] if args.claim_surface else None + report = build_guard(skill_dir, args.generated_at, claim_surfaces) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_evidence_intake.py b/scripts/render_world_class_evidence_intake.py new file mode 100644 index 00000000..328c3b94 --- /dev/null +++ b/scripts/render_world_class_evidence_intake.py @@ -0,0 +1,354 @@ +#!/usr/bin/env python3 +import argparse +import json +import shlex +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_ledger import build_ledger +from world_class_evidence_contract import load_json, rel_path, validate_payload + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Validates world-class human and external evidence intake packets before ledger review." + +def template_paths(skill_dir: Path, keys: list[str]) -> dict[str, Path]: + template_dir = skill_dir / "evidence" / "world_class" / "templates" + return {key: template_dir / f"{key}.intake.json" for key in keys} + + +def submission_paths(submissions_dir: Path) -> list[Path]: + if not submissions_dir.exists(): + return [] + ignored_names = {"submission_manifest.json"} + return sorted(path for path in submissions_dir.glob("*.json") if path.is_file() and path.name not in ignored_names) + + +def find_entry(entries: list[dict[str, Any]], key: str) -> dict[str, Any] | None: + for entry in entries: + if entry.get("key") == key: + return entry + return None + + +def first_by_key(items: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + by_key: dict[str, dict[str, Any]] = {} + for item in items: + key = str(item.get("evidence_key", "")) + if key and key not in by_key: + by_key[key] = item + return by_key + + +def shell_path(path: Path, root: Path) -> str: + return shlex.quote(rel_path(path, root)) + + +def checklist_readiness( + entry: dict[str, Any], + template_result: dict[str, Any] | None, + submission_result: dict[str, Any] | None, +) -> tuple[str, str]: + if entry.get("status") == "accepted": + return "accepted", "Ledger already accepts this evidence item." + if template_result is None or template_result.get("status") != "pass": + return "fix-template", "The intake template is missing or invalid." + if submission_result is None: + return "awaiting-submission", "No real evidence submission has been provided yet." + if submission_result.get("status") == "pass": + observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + if entry.get("source_accepted") is True or observed.get("accepted") is True: + return "ready-for-ledger-review", "Submission passes intake validation and is ready for ledger review." + return "source-evidence-incomplete", "Submission passes intake validation, but the source evidence checks still do not pass." + return "fix-submission", "Submission exists but failed intake validation." + + +def build_operator_checklist( + skill_dir: Path, + ledger: dict[str, Any], + template_results: list[dict[str, Any]], + submission_results: list[dict[str, Any]], + submissions_dir: Path, +) -> list[dict[str, Any]]: + templates_by_key = first_by_key(template_results) + submissions_by_key = first_by_key(submission_results) + checklist = [] + for entry in ledger.get("entries", []): + key = str(entry.get("key", "")) + template_path = skill_dir / "evidence" / "world_class" / "templates" / f"{key}.intake.json" + submission_path = submissions_dir / f"{key}.json" + template_result = templates_by_key.get(key) + submission_result = submissions_by_key.get(key) + readiness, blocking_reason = checklist_readiness(entry, template_result, submission_result) + submissions_dir_arg = shell_path(submissions_dir, skill_dir) + checklist.append( + { + "evidence_key": key, + "label": entry.get("label", key), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "readiness": readiness, + "blocking_reason": blocking_reason, + "template_status": template_result.get("status", "missing") if template_result else "missing", + "submission_status": submission_result.get("status", "missing") if submission_result else "missing", + "source_accepted": entry.get("source_accepted") is True, + "observed_state": entry.get("observed_state", {}), + "template_path": rel_path(template_path, skill_dir), + "submission_path": rel_path(submission_path, skill_dir), + "commands": { + "prepare_submission": ( + "python3 scripts/yao.py world-class-submission-kit . " + f"--evidence-key {shlex.quote(key)} --output-dir {shell_path(submission_path.parent, skill_dir)}" + ), + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {submissions_dir_arg}", + "submission_review": ( + f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {submissions_dir_arg}" + ), + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {submissions_dir_arg}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + }, + "must_collect": { + "provenance_requirements": entry.get("provenance_requirements", []), + "runbook": entry.get("runbook", []), + "success_checks": entry.get("success_checks", []), + "evidence_artifacts": entry.get("evidence_artifacts", []), + "privacy_contract": entry.get("privacy_contract", []), + }, + "anti_overclaim": entry.get("anti_overclaim", {}), + "next_action": entry.get("next_action", ""), + } + ) + return checklist + + +def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + ledger = build_ledger(skill_dir, generated_at) + entries = ledger.get("entries", []) + keys = [str(entry.get("key", "")) for entry in entries] + schema_path = skill_dir / "evidence" / "world_class" / "intake.schema.json" + template_results = [] + for key, path in template_paths(skill_dir, keys).items(): + entry = find_entry(entries, key) or {"key": key, "category": "external"} + payload = load_json(path) + if not payload: + template_results.append( + { + "path": rel_path(path, skill_dir), + "evidence_key": key, + "status": "fail", + "template_only": True, + "errors": ["template missing or invalid JSON"], + } + ) + continue + template_results.append(validate_payload(payload, entry, path=path, root=skill_dir, template_expected=True)) + + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + submission_results = [] + for path in submission_paths(submissions_dir): + payload = load_json(path) + key = str(payload.get("evidence_key", "")) + entry = find_entry(entries, key) + if not payload or entry is None: + submission_results.append( + { + "path": rel_path(path, skill_dir), + "evidence_key": key or "unknown", + "status": "fail", + "template_only": False, + "errors": ["submission missing, invalid JSON, or unknown evidence_key"], + } + ) + continue + submission_results.append(validate_payload(payload, entry, path=path, root=skill_dir, template_expected=False)) + + template_pass_count = sum(1 for item in template_results if item["status"] == "pass") + valid_submission_count = sum(1 for item in submission_results if item["status"] == "pass") + invalid_submission_count = sum(1 for item in submission_results if item["status"] == "fail") + schema_exists = schema_path.exists() + intake_ready = schema_exists and template_pass_count == len(keys) and invalid_submission_count == 0 + operator_checklist = build_operator_checklist(skill_dir, ledger, template_results, submission_results, submissions_dir) + ready_checklist_count = sum(1 for item in operator_checklist if item["readiness"] in {"accepted", "ready-for-ledger-review"}) + source_incomplete_count = sum(1 for item in operator_checklist if item["readiness"] == "source-evidence-incomplete") + return { + "schema_version": "1.0", + "ok": intake_ready, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "schema_present": schema_exists, + "ledger_entry_count": len(keys), + "template_count": len(template_results), + "template_pass_count": template_pass_count, + "submission_count": len(submission_results), + "valid_submission_count": valid_submission_count, + "invalid_submission_count": invalid_submission_count, + "operator_checklist_count": len(operator_checklist), + "operator_checklist_ready_count": ready_checklist_count, + "valid_packet_source_incomplete_count": source_incomplete_count, + "ready_for_external_collection": intake_ready, + "ready_for_ledger_review": ready_checklist_count > 0, + "ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True, + "overclaim_guard_active": True, + "decision": ( + "fix-intake" + if not intake_ready + else ( + "intake-ready-for-ledger-review" + if ready_checklist_count + else ("source-evidence-incomplete" if source_incomplete_count else "awaiting-submissions") + ) + ), + }, + "schema": rel_path(schema_path, skill_dir), + "templates": template_results, + "submissions": submission_results, + "operator_checklist": operator_checklist, + "source_ledger": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + "pending_count": ledger.get("summary", {}).get("pending_count", 0), + }, + "artifacts": { + "json": "reports/world_class_evidence_intake.json", + "markdown": "reports/world_class_evidence_intake.md", + }, + } + + +def render_table(items: list[dict[str, Any]]) -> list[str]: + lines = ["| Evidence | Status | Path | Artifacts | Errors |", "| --- | --- | --- | --- | --- |"] + if not items: + lines.append("| `none` | `n/a` | none | none | none |") + return lines + for item in items: + errors = "; ".join(item.get("errors", [])) or "none" + safe_errors = errors.replace("|", "\\|") + integrity = item.get("artifact_integrity", {}) if isinstance(item.get("artifact_integrity", {}), dict) else {} + artifact_summary = ( + f"{integrity.get('artifact_existing_count', 0)} existing / " + f"{integrity.get('artifact_sha256_verified_count', 0)} sha256 verified / " + f"{integrity.get('required_artifact_verified_count', 0)} required verified / " + f"{integrity.get('artifact_ref_count', 0)} refs" + ) + lines.append( + f"| `{item['evidence_key']}` | `{item['status']}` | `{item['path']}` | {artifact_summary} | {safe_errors} |" + ) + return lines + + +def render_operator_checklist(items: list[dict[str, Any]]) -> list[str]: + lines = [ + "| Evidence | Readiness | Submission | Next action |", + "| --- | --- | --- | --- |", + ] + if not items: + lines.append("| `none` | `accepted` | none | none |") + return lines + for item in items: + action = str(item.get("next_action", "")).replace("|", "\\|") + lines.append( + f"| `{item['evidence_key']}` | `{item['readiness']}` | `{item['submission_status']}` | {action} |" + ) + for item in items: + lines.extend(["", f"### {item['label']}", ""]) + lines.append(f"- readiness: `{item['readiness']}`") + lines.append(f"- blocking reason: {item['blocking_reason']}") + lines.append(f"- owner: {item['owner']}") + lines.append(f"- template: `{item['template_path']}`") + lines.append(f"- submission: `{item['submission_path']}`") + lines.extend(["", "#### Commands", ""]) + commands = item.get("commands", {}) + for label in ["prepare_submission", "validate_intake", "submission_review", "refresh_ledger", "guard_claim"]: + if commands.get(label): + lines.append(f"- {label}: `{commands[label]}`") + must_collect = item.get("must_collect", {}) + lines.extend(["", "#### Must Collect", ""]) + for label in ["provenance_requirements", "success_checks", "evidence_artifacts", "privacy_contract"]: + values = must_collect.get(label, []) + if values: + lines.append(f"- {label}:") + lines.extend(f" - {value}" for value in values) + runbook = must_collect.get("runbook", []) + if runbook: + lines.extend(["", "#### Source Runbook", ""]) + lines.extend(f"- `{value}`" if str(value).startswith("python3 ") or "=" in str(value) else f"- {value}" for value in runbook) + return lines + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Evidence Intake", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- schema present: `{str(summary['schema_present']).lower()}`", + f"- templates: `{summary['template_pass_count']}` / `{summary['template_count']}`", + f"- submissions: `{summary['valid_submission_count']}` valid / `{summary['submission_count']}` total", + f"- invalid submissions: `{summary['invalid_submission_count']}`", + f"- valid packet but source incomplete: `{summary['valid_packet_source_incomplete_count']}`", + f"- operator checklist: `{summary['operator_checklist_ready_count']}` ready / `{summary['operator_checklist_count']}` total", + f"- ready for external collection: `{str(summary['ready_for_external_collection']).lower()}`", + f"- ready for ledger review: `{str(summary['ready_for_ledger_review']).lower()}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`", + "", + "This report validates the intake contract for human and external evidence. A valid intake packet means the evidence is ready for ledger review; it does not by itself make a world-class claim true.", + "", + "## Templates", + "", + *render_table(report["templates"]), + "", + "## Submissions", + "", + *render_table(report["submissions"]), + "", + "## Operator Checklist", + "", + *render_operator_checklist(report["operator_checklist"]), + "", + "## Boundary", + "", + "- Templates and planned work do not count as accepted evidence.", + "- Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests.", + "- Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata.", + "- Local command-runner output does not count as provider-backed model evidence.", + "- Metadata fallback does not count as native permission enforcement.", + "- Pending reviewer work does not count as human adjudication.", + ] + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate world-class evidence intake packets.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_evidence_intake.json") + parser.add_argument("--output-md", default="reports/world_class_evidence_intake.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_intake(skill_dir, args.generated_at, submissions_dir) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_evidence_ledger.py b/scripts/render_world_class_evidence_ledger.py new file mode 100644 index 00000000..18911c57 --- /dev/null +++ b/scripts/render_world_class_evidence_ledger.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from output_review_privacy import forbidden_raw_content_field_paths +from render_world_class_evidence_plan import build_plan +from world_class_evidence_contract import load_json, load_json_with_status, rel_path, validate_payload +from world_class_source_checks import build_source_checklist, summarize_source_checklist + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a machine-checkable ledger for world-class external and human evidence gaps." + + +def provider_state(skill_dir: Path) -> dict[str, Any]: + summary = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {}) + return { + "model_executed_count": summary.get("model_executed_count", 0), + "timing_observed_count": summary.get("timing_observed_count", 0), + "token_observed_count": summary.get("token_observed_count", 0), + "accepted": ( + summary.get("model_executed_count", 0) > 0 + and summary.get("timing_observed_count", 0) > 0 + and summary.get("token_observed_count", 0) > 0 + ), + } + + +def human_state(skill_dir: Path) -> dict[str, Any]: + adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json") + summary = adjudication.get("summary", {}) + raw_content_paths = forbidden_raw_content_field_paths(adjudication, "output_review_adjudication") + return { + "pair_count": summary.get("pair_count", 0), + "judgment_count": summary.get("judgment_count", 0), + "pending_count": summary.get("pending_count", 0), + "invalid_decision_count": summary.get("invalid_decision_count", 0), + "answer_revealed_count": summary.get("answer_revealed_count", 0), + "reviewer_metadata_present": summary.get("reviewer_metadata_present", False), + "reason_required": summary.get("reason_required", False), + "blind_review_attested": summary.get("blind_review_attested", False), + "raw_content_excluded_attested": summary.get("raw_content_excluded_attested", False), + "reviewer_reason_required_attested": summary.get("reviewer_reason_required_attested", False), + "ready_for_human_evidence": summary.get("ready_for_human_evidence", False), + "raw_content_allowed": bool(raw_content_paths), + "raw_content_path_count": len(raw_content_paths), + "accepted": ( + summary.get("ready_for_human_evidence") is True + and not raw_content_paths + ), + } + + +def permission_state(skill_dir: Path) -> dict[str, Any]: + summary = load_json(skill_dir / "reports" / "runtime_permission_probes.json").get("summary", {}) + return { + "native_enforcement_count": summary.get("native_enforcement_count", 0), + "metadata_fallback_count": summary.get("metadata_fallback_count", 0), + "installer_enforcement_pass_count": summary.get("installer_enforcement_pass_count", 0), + "installer_permission_failure_count": summary.get("installer_permission_failure_count", 0), + "installer_enforcement_ready": summary.get("installer_enforcement_ready", False), + "residual_risk_count": summary.get("residual_risk_count", 0), + "failure_count": summary.get("failure_count", 0), + "accepted": summary.get("native_enforcement_count", 0) > 0 and summary.get("failure_count", 0) == 0, + } + + +def telemetry_state(skill_dir: Path) -> dict[str, Any]: + payload = load_json(skill_dir / "reports" / "adoption_drift_report.json") + summary = payload.get("summary", {}) + source_types = summary.get("source_types", {}) if isinstance(summary.get("source_types", {}), dict) else {} + privacy = payload.get("privacy_contract", {}) + return { + "external_source_events": source_types.get("external", 0), + "adoption_sample_count": summary.get("adoption_sample_count", 0), + "raw_content_allowed": privacy.get("raw_content_allowed"), + "risk_band": summary.get("risk_band", "missing"), + "accepted": ( + source_types.get("external", 0) > 0 + and summary.get("adoption_sample_count", 0) > 0 + and privacy.get("raw_content_allowed") is False + ), + } + + +STATE_LOADERS = { + "provider-holdout": provider_state, + "human-adjudication": human_state, + "native-permission-enforcement": permission_state, + "native-client-telemetry": telemetry_state, +} + +PROVENANCE_REQUIREMENTS = { + "provider-holdout": ["provider-backed model run", "observed timing", "observed token metadata"], + "human-adjudication": ["real reviewer identity", "blind A/B decisions", "answer key unopened until decisions exist"], + "native-permission-enforcement": ["real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", "residual risk retained for fallback targets"], + "native-client-telemetry": ["real external client source", "metadata-only event", "local-first import path"], +} + +TOP_LEVEL_SUMMARY_FIELDS = [ + "decision", + "ready_to_claim_world_class", + "ledger_entry_count", + "source_accepted_count", + "accepted_count", + "pending_count", + "human_pending_count", + "external_pending_count", + "submitted_entry_count", + "reviewer_approved_submission_count", + "missing_submission_count", + "invalid_submission_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "submitted_but_pending_count", + "source_accepted_without_valid_submission_count", + "overclaim_guard_active", +] + + +def top_level_summary_mirrors(summary: dict[str, Any]) -> dict[str, Any]: + return {key: summary[key] for key in TOP_LEVEL_SUMMARY_FIELDS if key in summary} + + +def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]: + key = str(task.get("key", "")) + path = submissions_dir / f"{key}.json" + payload, load_status = load_json_with_status(path) + if load_status != "present": + return { + "status": load_status, + "path": rel_path(path, skill_dir), + "artifact_ref_count": 0, + "attested_real_evidence": False, + "privacy_contract_satisfied": False, + "ledger_reviewer_approved": False, + "ledger_reviewer": "", + "ledger_reviewed_at": "", + "ledger_counts_as_completion": False, + } + validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False) + refs = payload.get("artifact_refs", []) + attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {} + status = "submitted" if validation["status"] == "pass" else "invalid-contract" + artifact_integrity = validation.get("artifact_integrity", {}) + return { + "status": status, + "path": rel_path(path, skill_dir), + "submitted_by": payload.get("submitted_by", ""), + "submitted_at": payload.get("submitted_at", ""), + "artifact_ref_count": len(refs) if isinstance(refs, list) else 0, + "artifact_existing_count": artifact_integrity.get("artifact_existing_count", 0), + "artifact_sha256_verified_count": artifact_integrity.get("artifact_sha256_verified_count", 0), + "attested_real_evidence": attestation.get("real_external_or_human_evidence") is True, + "reviewer_or_operator_identity_present": attestation.get("reviewer_or_operator_identity_present") is True, + "privacy_contract_satisfied": attestation.get("privacy_contract_satisfied") is True, + "ledger_reviewer_approved": attestation.get("ledger_reviewer_approved") is True, + "ledger_reviewer": str(attestation.get("ledger_reviewer", "")).strip(), + "ledger_reviewed_at": str(attestation.get("ledger_reviewed_at", "")).strip(), + "errors": validation.get("errors", []), + "ledger_counts_as_completion": False, + } + + +def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]: + state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir) + submission = submission_state(skill_dir, task, submissions_dir) + source_checklist = build_source_checklist([{"key": task["key"], "observed_state": state}]) + source_summary = summarize_source_checklist(source_checklist) + source_accepted = bool(source_checklist) and source_summary["source_blocked_count"] == 0 + accepted = source_accepted and submission.get("status") == "submitted" and submission.get("ledger_reviewer_approved") is True + return { + "key": task["key"], + "label": task["label"], + "category": task["category"], + "owner": task["owner"], + "status": "accepted" if accepted else "pending", + "source_status": task["status"], + "source_accepted": source_accepted, + "current": task["current"], + "objective": task["objective"], + "runbook": task.get("runbook", []), + "provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]), + "success_checks": task["success_checks"], + "evidence_artifacts": task["evidence_artifacts"], + "privacy_contract": task["privacy_contract"], + "observed_state": state, + "source_checklist": source_checklist, + **source_summary, + "submission_state": submission, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "next_action": task["audit_next_action"], + } + + +def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + plan = build_plan(skill_dir, generated_at) + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + evidence_requirements = plan.get("evidence_requirements") or plan.get("tasks", []) + entries = [build_entry(skill_dir, task, submissions_dir) for task in evidence_requirements] + source_rows = [row for entry in entries for row in entry.get("source_checklist", [])] + source_summary = summarize_source_checklist(source_rows) + source_accepted_count = sum(1 for entry in entries if entry.get("source_accepted") is True) + accepted_count = sum(1 for entry in entries if entry["status"] == "accepted") + pending_count = len(entries) - accepted_count + human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending") + external_pending_count = sum(1 for entry in entries if entry["category"] == "external" and entry["status"] == "pending") + submitted_entry_count = sum(1 for entry in entries if entry["submission_state"]["status"] == "submitted") + reviewer_approved_submission_count = sum( + 1 for entry in entries if entry["submission_state"].get("ledger_reviewer_approved") is True + ) + missing_submission_count = sum(1 for entry in entries if entry["submission_state"]["status"] == "missing") + invalid_submission_count = sum(1 for entry in entries if entry["submission_state"]["status"] in {"invalid-json", "invalid-contract"}) + submitted_but_pending_count = sum( + 1 for entry in entries if entry["submission_state"]["status"] == "submitted" and entry["status"] == "pending" + ) + source_accepted_without_valid_submission_count = sum( + 1 + for entry in entries + if entry.get("source_accepted") is True and entry["submission_state"]["status"] != "submitted" + ) + ready = bool(entries) and pending_count == 0 and plan["summary"].get("world_class_ready") is True + summary = { + "ledger_entry_count": len(entries), + "source_accepted_count": source_accepted_count, + "accepted_count": accepted_count, + "pending_count": pending_count, + "human_pending_count": human_pending_count, + "external_pending_count": external_pending_count, + "submitted_entry_count": submitted_entry_count, + "reviewer_approved_submission_count": reviewer_approved_submission_count, + "missing_submission_count": missing_submission_count, + "invalid_submission_count": invalid_submission_count, + **source_summary, + "submitted_but_pending_count": submitted_but_pending_count, + "source_accepted_without_valid_submission_count": source_accepted_without_valid_submission_count, + "overclaim_guard_active": True, + "ready_to_claim_world_class": ready, + "decision": "ready-for-completion-audit" if ready else "evidence-pending", + } + return { + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + **top_level_summary_mirrors(summary), + "summary": summary, + "report_contract": { + "schema_version": "1.0", + "contract": "world-class-evidence-ledger", + "top_level_mirrors_summary": True, + "summary_fields": TOP_LEVEL_SUMMARY_FIELDS, + "source_of_truth": "summary", + }, + "entries": entries, + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": plan["summary"].get("task_count", 0), + "evidence_requirement_count": len(evidence_requirements), + }, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "ledger_counts_submission_as_completion": False, + "source_pass_requires_valid_submission": True, + }, + "artifacts": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + "intake": "reports/world_class_evidence_intake.md", + }, + } + + +def render_markdown(ledger: dict[str, Any]) -> str: + summary = ledger["summary"] + lines = [ + "# World-Class Evidence Ledger", + "", + f"Generated at: `{ledger['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- entries: `{summary['ledger_entry_count']}`", + f"- source accepted: `{summary.get('source_accepted_count', 0)}`", + f"- source checks: `{summary.get('source_pass_count', 0)}` pass / `{summary.get('source_check_count', 0)}` total", + f"- source blocked: `{summary.get('source_blocked_count', 0)}`", + f"- accepted: `{summary['accepted_count']}`", + f"- pending: `{summary['pending_count']}`", + f"- human pending: `{summary['human_pending_count']}`", + f"- external pending: `{summary['external_pending_count']}`", + f"- submitted entries: `{summary['submitted_entry_count']}`", + f"- reviewer approved submissions: `{summary.get('reviewer_approved_submission_count', 0)}`", + f"- submitted but pending: `{summary['submitted_but_pending_count']}`", + f"- source accepted without valid submission: `{summary.get('source_accepted_without_valid_submission_count', 0)}`", + f"- invalid submissions: `{summary['invalid_submission_count']}`", + f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`", + "", + "This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.", + "", + "## Ledger", + "", + "| Evidence | Status | Submission | Category | Current | Next action |", + "| --- | --- | --- | --- | --- | --- |", + ] + for entry in ledger["entries"]: + current = str(entry["current"]).replace("|", "\\|") + action = str(entry["next_action"]).replace("|", "\\|") + submission = entry.get("submission_state", {}).get("status", "missing") + lines.append(f"| `{entry['key']}` | `{entry['status']}` | `{submission}` | `{entry['category']}` | {current} | {action} |") + if not ledger["entries"]: + lines.append("| `none` | `accepted` | `none` | all evidence collected | none |") + for entry in ledger["entries"]: + lines.extend(["", f"## {entry['label']}", ""]) + lines.append(f"- objective: {entry['objective']}") + lines.append(f"- source status: `{entry['source_status']}`") + lines.append(f"- observed state: `{json.dumps(entry['observed_state'], ensure_ascii=False)}`") + lines.append( + f"- source checks: `{entry.get('source_pass_count', 0)}` pass / " + f"`{entry.get('source_check_count', 0)}` total" + ) + lines.append(f"- submission state: `{json.dumps(entry.get('submission_state', {}), ensure_ascii=False)}`") + lines.extend(["", "### Provenance Requirements", ""]) + lines.extend(f"- {item}" for item in entry["provenance_requirements"]) + lines.extend(["", "### Source Runbook", ""]) + lines.extend(f"- `{item}`" if str(item).startswith("python3 ") or "=" in str(item) else f"- {item}" for item in entry.get("runbook", [])) + lines.extend( + [ + "", + "### Source Evidence Checks", + "", + "| Check | Current | Expected | Status |", + "| --- | --- | --- | --- |", + ] + ) + source_checks = entry.get("source_checklist", []) + if source_checks: + for row in source_checks: + lines.append( + f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` |" + ) + else: + lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` |") + lines.extend(["", "### Completion Assertions", ""]) + lines.extend(f"- {item}" for item in entry["success_checks"]) + lines.extend(["", "### Privacy Contract", ""]) + lines.extend(f"- {item}" for item in entry["privacy_contract"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a world-class evidence ledger.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/world_class_evidence_ledger.json") + parser.add_argument("--output-md", default="reports/world_class_evidence_ledger.md") + parser.add_argument("--submissions-dir") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + ledger = build_ledger(skill_dir, args.generated_at, submissions_dir=submissions_dir) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(ledger), encoding="utf-8") + print(json.dumps(ledger, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_evidence_plan.py b/scripts/render_world_class_evidence_plan.py new file mode 100644 index 00000000..30ebc620 --- /dev/null +++ b/scripts/render_world_class_evidence_plan.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from render_skill_os2_audit import build_audit +from world_class_evidence_contract import load_json_with_status, validate_payload + + +ROOT = Path(__file__).resolve().parent.parent + + +TASK_TEMPLATES: dict[str, dict[str, Any]] = { + "provider-holdout": { + "category": "external", + "owner": "operator with provider credentials", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + "For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60", + "For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass", + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts.", + ], + }, + "human-adjudication": { + "category": "human", + "owner": "human reviewer", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.", + "python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/output_review_adjudication.json summary.reviewer_metadata_present is true", + "reports/output_review_adjudication.json summary.blind_review_attested is true", + "reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json", + "reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text", + "reports/output_review_adjudication.json summary.ready_for_human_evidence is true", + "reports/skill_os2_audit.json item human-adjudication status becomes pass", + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "scripts/import_output_review_decisions.py", + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.", + "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", + "The adjudication evidence stores prompt_sha256 instead of raw prompt text.", + "The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.", + "Keep the answer key separate until after decisions are recorded.", + ], + }, + "native-permission-enforcement": { + "category": "external", + "owner": "target client or installer integrator", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass", + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement.", + ], + }, + "native-client-telemetry": { + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass", + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.", + ], + }, +} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def build_task(item: dict[str, Any]) -> dict[str, Any]: + template = TASK_TEMPLATES.get( + item["key"], + { + "category": "review", + "owner": "release reviewer", + "objective": item.get("target", "Collect stronger evidence for this non-pass audit item."), + "runbook": ["Open reports/skill_os2_audit.md and resolve the listed next action."], + "success_checks": [f"reports/skill_os2_audit.json item {item['key']} status becomes pass"], + "evidence_artifacts": [entry["path"] for entry in item.get("evidence", []) if entry.get("exists")], + "privacy_contract": ["Do not add raw private user content to release evidence."], + }, + ) + intake_runbook = [ + f"Copy evidence/world_class/templates/{item['key']}.intake.json to evidence/world_class/submissions/{item['key']}.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + ] + intake_artifacts = [ + "evidence/world_class/intake.schema.json", + f"evidence/world_class/templates/{item['key']}.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md", + ] + return { + "key": item["key"], + "label": item["label"], + "status": item["status"], + "category": template["category"], + "owner": template["owner"], + "current": item["current"], + "objective": template["objective"], + "runbook": [*template["runbook"], *intake_runbook], + "success_checks": template["success_checks"], + "evidence_artifacts": [*template["evidence_artifacts"], *intake_artifacts], + "privacy_contract": template["privacy_contract"], + "audit_next_action": item["next_action"], + } + + +def has_accepted_ledger_submission(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> bool: + path = submissions_dir / f"{task['key']}.json" + payload, load_status = load_json_with_status(path) + if load_status != "present": + return False + validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False) + return validation.get("status") == "pass" + + +def build_plan(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + audit = build_audit(skill_dir, generated_at) + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + evidence_keys = set(TASK_TEMPLATES) + evidence_requirements = [build_task(item) for item in audit["items"] if item["key"] in evidence_keys] + tasks = [ + task + for task in evidence_requirements + if task["status"] != "pass" or not has_accepted_ledger_submission(skill_dir, task, submissions_dir) + ] + category_counts: dict[str, int] = {} + for task in tasks: + category_counts[task["category"]] = category_counts.get(task["category"], 0) + 1 + audit_world_class_ready = audit["summary"].get("world_class_ready") is True + return { + "schema_version": "1.0", + "ok": audit["ok"], + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "audit_decision": audit["summary"]["decision"], + "world_class_ready": bool(audit["summary"]["world_class_ready"]), + "audit_world_class_ready": audit_world_class_ready, + "ready_to_claim_world_class": False, + "ledger_completion_required": True, + "evidence_requirement_count": len(evidence_requirements), + "task_count": len(tasks), + "human_task_count": category_counts.get("human", 0), + "external_task_count": category_counts.get("external", 0), + "review_task_count": category_counts.get("review", 0), + "decision": "audit-ready-ledger-required" if audit_world_class_ready and not tasks else "collect-external-evidence", + }, + "tasks": tasks, + "evidence_requirements": evidence_requirements, + "source_audit": { + "json": "reports/skill_os2_audit.json", + "markdown": "reports/skill_os2_audit.md", + "open_gap_count": audit["summary"]["open_gap_count"], + }, + "artifacts": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "ledger": "reports/world_class_evidence_ledger.md", + "intake": "reports/world_class_evidence_intake.md", + }, + } + + +def render_markdown(plan: dict[str, Any]) -> str: + summary = plan["summary"] + lines = [ + "# World-Class Evidence Plan", + "", + f"Generated at: `{plan['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- audit decision: `{summary['audit_decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- ledger completion required: `{str(summary.get('ledger_completion_required', True)).lower()}`", + f"- evidence requirements: `{summary.get('evidence_requirement_count', 0)}`", + f"- tasks: `{summary['task_count']}`", + f"- human tasks: `{summary['human_task_count']}`", + f"- external tasks: `{summary['external_task_count']}`", + "", + "This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions.", + "", + "## Task Table", + "", + "| Task | Status | Category | Owner | Current |", + "| --- | --- | --- | --- | --- |", + ] + for task in plan["tasks"]: + current = str(task["current"]).replace("|", "\\|") + lines.append( + f"| `{task['key']}` | `{task['status']}` | `{task['category']}` | {task['owner']} | {current} |" + ) + if not plan["tasks"]: + lines.append("| `none` | `pass` | `none` | none | audit gaps closed; ledger validation still required |") + for task in plan["tasks"]: + lines.extend( + [ + "", + f"## {task['label']}", + "", + f"- objective: {task['objective']}", + f"- audit next action: {task['audit_next_action']}", + "", + "### Runbook", + "", + ] + ) + for command in task["runbook"]: + lines.append(f"- `{command}`" if command.startswith("python3 ") or "=" in command else f"- {command}") + lines.extend(["", "### Success Checks", ""]) + lines.extend(f"- {check}" for check in task["success_checks"]) + lines.extend(["", "### Evidence Artifacts", ""]) + lines.extend(f"- `{artifact}`" for artifact in task["evidence_artifacts"]) + lines.extend(["", "### Privacy Contract", ""]) + lines.extend(f"- {item}" for item in task["privacy_contract"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a world-class evidence collection plan.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/world_class_evidence_plan.json") + parser.add_argument("--output-md", default="reports/world_class_evidence_plan.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + plan = build_plan(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(plan, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(plan), encoding="utf-8") + print(json.dumps(plan, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_operator_runbook.py b/scripts/render_world_class_operator_runbook.py new file mode 100644 index 00000000..71077c89 --- /dev/null +++ b/scripts/render_world_class_operator_runbook.py @@ -0,0 +1,651 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from html_rendering import html_text +from render_world_class_evidence_intake import build_intake +from render_world_class_evidence_ledger import build_ledger +from render_world_class_preflight import build_preflight +from render_world_class_submission_review import build_submission_review +from world_class_operator_runbook_coordination import build_coordination_plan, build_release_gate + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders an operator runbook for collecting pending world-class evidence without accepting evidence." + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def by_key(items: list[dict[str, Any]], key_name: str) -> dict[str, dict[str, Any]]: + return {str(item.get(key_name, "")): item for item in items if item.get(key_name)} + + +def output_path(path: str, skill_dir: Path) -> str: + candidate = Path(path) + if candidate.is_absolute(): + return rel_path(candidate, skill_dir) + return path + + +def build_runbook_item( + entry: dict[str, Any], + checklist: dict[str, Any], + review_item: dict[str, Any], + preflight_item: dict[str, Any], +) -> dict[str, Any]: + accepted = entry.get("status") == "accepted" or review_item.get("review_state") == "accepted" + commands = checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {} + must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {} + source_checklist = review_item.get("source_checklist", []) + blocked_source_checks = [ + row for row in source_checklist if isinstance(row, dict) and row.get("status") != "pass" + ] + next_source_actions = [] + for row in blocked_source_checks: + action = str(row.get("next_action", "")).strip() + if action and action not in next_source_actions: + next_source_actions.append(action) + repair_checklist = ( + preflight_item.get("repair_checklist", []) if isinstance(preflight_item.get("repair_checklist", []), list) else [] + ) + phase_queue = preflight_item.get("phase_queue", []) if isinstance(preflight_item.get("phase_queue", []), list) else [] + if accepted: + repair_checklist = [] + phase_queue = [] + return { + "evidence_key": entry.get("key", ""), + "label": entry.get("label", entry.get("key", "")), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "ledger_status": entry.get("status", "pending"), + "intake_readiness": checklist.get("readiness", "missing"), + "review_state": review_item.get("review_state", "missing"), + "source_accepted": review_item.get("source_accepted") is True, + "objective": entry.get("objective", ""), + "current": entry.get("current", ""), + "execution_runbook": entry.get("runbook", []), + "blocking_reason": review_item.get("blocking_reason") or checklist.get("blocking_reason") or entry.get("next_action", ""), + "submission_path": checklist.get("submission_path") or entry.get("submission_state", {}).get("path", ""), + "template_path": checklist.get("template_path", ""), + "commands": { + "prepare_submission": commands.get("prepare_submission", ""), + "validate_intake": commands.get("validate_intake", ""), + "review_queue": commands.get("submission_review", ""), + "refresh_ledger": commands.get("refresh_ledger", "python3 scripts/yao.py world-class-ledger ."), + "guard_claim": commands.get("guard_claim", "python3 scripts/yao.py world-class-claim-guard ."), + }, + "must_collect": { + "provenance_requirements": must_collect.get("provenance_requirements", entry.get("provenance_requirements", [])), + "success_checks": must_collect.get("success_checks", entry.get("success_checks", [])), + "privacy_contract": must_collect.get("privacy_contract", entry.get("privacy_contract", [])), + }, + "evidence_artifacts": entry.get("evidence_artifacts", []), + "observed_state": entry.get("observed_state", {}), + "source_checklist": source_checklist, + "blocked_source_check_count": len(blocked_source_checks), + "next_source_actions": next_source_actions, + "repair_checklist": repair_checklist, + "repair_blocked_count": sum( + 1 + for row in repair_checklist + if isinstance(row, dict) and row.get("status") != "ready" + ), + "repair_counts_as_completion": False, + "phase_queue": phase_queue, + "phase_queue_blocked_count": sum( + 1 + for row in phase_queue + if isinstance(row, dict) and row.get("status") != "ready" + ), + "phase_queue_counts_as_completion": False, + "submission_state": entry.get("submission_state", {}), + "anti_overclaim": entry.get("anti_overclaim", {}), + } + + +def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir) + intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir) + review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir) + preflight = build_preflight(skill_dir, generated_at, submissions_dir=submissions_dir) + checklist_by_key = by_key(intake.get("operator_checklist", []), "evidence_key") + review_by_key = by_key(review.get("items", []), "evidence_key") + preflight_by_key = by_key(preflight.get("items", []), "evidence_key") + items = [ + build_runbook_item( + entry, + checklist_by_key.get(str(entry.get("key", "")), {}), + review_by_key.get(str(entry.get("key", "")), {}), + preflight_by_key.get(str(entry.get("key", "")), {}), + ) + for entry in ledger.get("entries", []) + ] + coordination_plan = build_coordination_plan(items) + release_gate = build_release_gate(skill_dir, ledger) + summary = ledger.get("summary", {}) + review_summary = review.get("summary", {}) + preflight_summary = preflight.get("summary", {}) if isinstance(preflight.get("summary", {}), dict) else {} + return { + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "evidence_item_count": len(items), + "pending_count": summary.get("pending_count", 0), + "accepted_count": summary.get("accepted_count", 0), + "awaiting_submission_count": review_summary.get("awaiting_submission_count", 0), + "ready_for_ledger_review_count": review_summary.get("ready_for_ledger_review_count", 0), + "valid_packet_source_incomplete_count": review_summary.get("valid_packet_source_incomplete_count", 0), + "invalid_submission_count": review_summary.get("invalid_submission_count", 0), + "source_check_count": review_summary.get("source_check_count", 0), + "source_pass_count": review_summary.get("source_pass_count", 0), + "source_blocked_count": review_summary.get("source_blocked_count", 0), + "repair_checklist_count": preflight_summary.get("repair_checklist_count", 0), + "repair_blocked_count": preflight_summary.get("repair_blocked_count", 0), + "phase_queue_count": preflight_summary.get("phase_queue_count", 0), + "phase_queue_blocked_count": preflight_summary.get("phase_queue_blocked_count", 0), + "phase_queue_row_count": preflight_summary.get("phase_queue_row_count", 0), + "phase_queue_next_phase": preflight_summary.get("phase_queue_next_phase", ""), + "phase_queue_next_action_id": preflight_summary.get("phase_queue_next_action_id", ""), + "phase_queue_next_command": preflight_summary.get("phase_queue_next_command", ""), + "phase_queue_counts_as_completion": False, + "coordination_step_count": len(coordination_plan), + "coordination_user_required_step_count": sum(1 for step in coordination_plan if step.get("requires_user_input")), + "coordination_pending_evidence_keys": sorted( + key + for key in {str(step.get("evidence_key", "")) for step in coordination_plan} + if key + ), + "coordination_counts_as_completion": False, + "release_gate_ready": release_gate["ready"], + "release_gate_blocked_count": release_gate["blocked_count"], + "release_gate_check_count": release_gate["check_count"], + "release_gate_counts_as_completion": False, + "ready_to_claim_world_class": summary.get("ready_to_claim_world_class") is True, + "runbook_counts_as_completion": False, + "decision": "ready-for-completion-audit" if summary.get("ready_to_claim_world_class") is True else "collect-evidence", + }, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "runbook_counts_submission_as_completion": False, + }, + "items": items, + "coordination_plan": coordination_plan, + "release_gate": release_gate, + "repair_checklist": preflight.get("repair_checklist", []) + if isinstance(preflight.get("repair_checklist", []), list) + else [], + "phase_queue": preflight.get("phase_queue", []) if isinstance(preflight.get("phase_queue", []), list) else [], + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "preflight": "reports/world_class_evidence_preflight.json", + "submission_review": "reports/world_class_submission_review.json", + "claim_guard": "reports/world_class_claim_guard.json", + }, + "artifacts": { + "json": "reports/world_class_operator_runbook.json", + "markdown": "reports/world_class_operator_runbook.md", + "html": "reports/world_class_operator_runbook.html", + }, + } + + +def list_lines(values: list[Any], empty: str) -> list[str]: + if not values: + return [f"- {empty}"] + return [f"- {value}" for value in values] + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Operator Runbook", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- runbook counts as completion: `{str(summary['runbook_counts_as_completion']).lower()}`", + f"- evidence items: `{summary['evidence_item_count']}`", + f"- pending: `{summary['pending_count']}`", + f"- awaiting submission: `{summary['awaiting_submission_count']}`", + f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`", + f"- phase queue: `{summary['phase_queue_blocked_count']}` blocked / `{summary['phase_queue_count']}` phases", + f"- phase queue rows: `{summary['phase_queue_row_count']}`", + f"- phase queue counts as completion: `{str(summary['phase_queue_counts_as_completion']).lower()}`", + f"- coordination steps: `{summary['coordination_user_required_step_count']}` user-required / `{summary['coordination_step_count']}` total", + f"- coordination pending keys: `{', '.join(summary['coordination_pending_evidence_keys'])}`", + f"- coordination counts as completion: `{str(summary['coordination_counts_as_completion']).lower()}`", + f"- release gate ready: `{str(summary['release_gate_ready']).lower()}`", + f"- release gate blocked checks: `{summary['release_gate_blocked_count']}` / `{summary['release_gate_check_count']}`", + f"- release gate counts as completion: `{str(summary['release_gate_counts_as_completion']).lower()}`", + "", + "This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.", + "", + "## Fast Path", + "", + "1. Run the real external or human work for one evidence item.", + "2. Generate the matching submission draft.", + "3. Replace template-only fields with aggregate evidence and provenance.", + "4. Validate intake, review the queue, refresh the ledger, then run the claim guard.", + "", + ] + lines.extend( + [ + "## Coordination Plan", + "", + "| Step | Evidence | Owner | Needs user | User action | Assistant action | Command | Pass condition |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + ) + for step in report.get("coordination_plan", []): + lines.append( + f"| `{step.get('step_id', '')}` | `{step.get('evidence_key', '') or 'all'}` | {step.get('owner', '')} | " + f"`{str(step.get('requires_user_input') is True).lower()}` | {step.get('user_action', '')} | " + f"{step.get('assistant_action', '')} | `{step.get('command', '')}` | {step.get('pass_condition', '')} |" + ) + lines.append("") + lines.extend( + [ + "## Phase Queue", + "", + "| Phase | Status | Rows | Blocked | Owners | Next action | Verify |", + "| --- | --- | ---: | ---: | --- | --- | --- |", + ] + ) + for row in report.get("phase_queue", []): + owners = ", ".join(str(owner) for owner in row.get("owners", [])) + lines.append( + f"| `{row.get('phase', '')}` | `{row.get('status', '')}` | `{row.get('row_count', 0)}` | " + f"`{row.get('blocked_count', 0)}` | {owners} | {row.get('next_action', '')} | `{row.get('verification_command', '')}` |" + ) + lines.append("") + lines.extend( + [ + "## Evidence Items", + "", + "| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |", + "| --- | --- | --- | --- | ---: | --- | --- |", + ] + ) + for item in report["items"]: + next_action = item.get("next_source_actions", ["none"])[0] if item.get("next_source_actions") else "none" + lines.append( + f"| `{item['evidence_key']}` | `{item['ledger_status']}` | `{item['intake_readiness']}` | " + f"`{item['review_state']}` | `{item.get('blocked_source_check_count', 0)}` | {next_action} | {item['owner']} |" + ) + for item in report["items"]: + lines.extend( + [ + "", + f"## {item['label']}", + "", + f"- objective: {item['objective']}", + f"- blocking reason: {item['blocking_reason']}", + f"- blocked source checks: `{item.get('blocked_source_check_count', 0)}`", + f"- repair rows: `{item.get('repair_blocked_count', 0)}` blocked", + f"- phase queue: `{item.get('phase_queue_blocked_count', 0)}` blocked phases", + f"- submission: `{item['submission_path'] or 'missing'}`", + f"- template: `{item['template_path'] or 'missing'}`", + "", + "### Phase Queue", + "", + "| Phase | Status | Rows | Blocked | Next action |", + "| --- | --- | ---: | ---: | --- |", + ] + ) + item_phase_queue = item.get("phase_queue", []) + if item_phase_queue: + for row in item_phase_queue: + lines.append( + f"| `{row.get('phase', '')}` | `{row.get('status', '')}` | `{row.get('row_count', 0)}` | " + f"`{row.get('blocked_count', 0)}` | {row.get('next_action', '')} |" + ) + else: + lines.append("| No phase queue listed. | `n/a` | `0` | `0` | n/a |") + lines.extend( + [ + "", + "### Source Runbook", + "", + *list_lines(item.get("execution_runbook", []), "No source runbook listed."), + "", + "### Commands", + "", + ] + ) + for label, command in item["commands"].items(): + lines.append(f"- {label}: `{command}`") + lines.extend(["", "### Must Collect", ""]) + lines.extend(list_lines(item["must_collect"].get("provenance_requirements", []), "No provenance requirements listed.")) + lines.extend(["", "### Success Checks", ""]) + lines.extend(list_lines(item["must_collect"].get("success_checks", []), "No success checks listed.")) + lines.extend(["", "### Privacy Contract", ""]) + lines.extend(list_lines(item["must_collect"].get("privacy_contract", []), "No privacy contract listed.")) + lines.extend(["", "### Evidence Artifacts", ""]) + lines.extend(list_lines(item.get("evidence_artifacts", []), "No evidence artifacts listed.")) + lines.extend(["", "### Next Source Actions", ""]) + lines.extend(list_lines(item.get("next_source_actions", []), "No blocked source checks.")) + lines.extend( + [ + "", + "### Source Evidence Snapshot", + "", + "| Check | Current | Expected | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + source_rows = item.get("source_checklist", []) + if source_rows: + for row in source_rows: + lines.append( + f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` | {row.get('next_action', '')} |" + ) + else: + lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` | n/a |") + release_gate = report.get("release_gate", {}) + lines.extend( + [ + "", + "## Release Gate", + "", + f"- decision: `{release_gate.get('decision', 'missing')}`", + f"- ready: `{str(release_gate.get('ready') is True).lower()}`", + f"- blocked checks: `{release_gate.get('blocked_count', 0)}` / `{release_gate.get('check_count', 0)}`", + f"- counts as completion: `{str(release_gate.get('counts_as_completion') is True).lower()}`", + f"- final manual check: {release_gate.get('final_manual_check', '')}", + "", + "| Check | Current | Expected | Status | Artifact |", + "| --- | --- | --- | --- | --- |", + ] + ) + for check in release_gate.get("checks", []): + status = "pass" if check.get("passed") is True else "blocked" + lines.append( + f"| {check.get('label', '')} | `{check.get('current', '')}` | `{check.get('expected', '')}` | " + f"`{status}` | `{check.get('artifact', '')}` |" + ) + lines.extend( + [ + "", + "## Boundary", + "", + "- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.", + "- Valid intake means ready for submission review; ledger review still requires passing source evidence.", + "- The world-class ledger and claim guard remain the source of truth.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def html_list(values: list[Any], empty: str) -> str: + if not values: + return f"
  • {html_text(empty)}
    • " + return "".join(f"
  • {html_text(value)}
    • " for value in values) + + +def html_source_checks(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No source checks listed.

    " + items = [] + for row in rows: + items.append( + "
  • " + f"{html_text(row.get('label', ''))}" + f"{html_text(row.get('field', ''))}: {html_text(row.get('actual', ''))} / {html_text(row.get('expected', ''))}" + f"{html_text(row.get('next_action', ''))}" + "
    • " + ) + return "
      " + "".join(items) + "
    " + + +def html_phase_queue(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No phase queue listed.

    " + items = [] + for row in rows: + items.append( + "
  • " + f"{html_text(row.get('label', row.get('phase', '')))}" + f"{html_text(row.get('phase', ''))}: {html_text(row.get('blocked_count', 0))} / {html_text(row.get('row_count', 0))}" + f"{html_text(row.get('next_action', ''))}" + "
    • " + ) + return "
      " + "".join(items) + "
    " + + +def render_html_coordination_plan(steps: list[dict[str, Any]]) -> str: + if not steps: + return "

    No coordination plan listed.

    " + rows = [] + for step in steps: + rows.append( + "" + f"{html_text(step.get('step_id', ''))}" + f"{html_text(step.get('evidence_key', '') or 'all')}" + f"{html_text(step.get('owner', ''))}" + f"{html_text(str(step.get('requires_user_input') is True).lower())}" + f"{html_text(step.get('user_action', ''))}" + f"{html_text(step.get('assistant_action', ''))}" + f"{html_text(step.get('command', ''))}" + f"{html_text(step.get('pass_condition', ''))}" + "" + ) + return "
    " + "".join(rows) + "
    StepEvidenceOwnerUserUser actionAssistant actionCommandPass condition
    " + + +def render_html_release_gate(release_gate: dict[str, Any]) -> str: + checks = release_gate.get("checks", []) if isinstance(release_gate.get("checks", []), list) else [] + rows = [] + for check in checks: + status = "pass" if check.get("passed") is True else "blocked" + rows.append( + f"" + f"{html_text(check.get('label', ''))}" + f"{html_text(check.get('current', ''))}" + f"{html_text(check.get('expected', ''))}" + f"{html_text(status)}" + f"{html_text(check.get('artifact', ''))}" + "" + ) + check_table = ( + "
    " + + "".join(rows) + + "
    CheckCurrentExpectedStatusArtifact
    " + if rows + else "

    No release checks listed.

    " + ) + return f""" +
    +
    Decision{html_text(release_gate.get('decision', 'missing'))}
    +
    Ready{html_text(str(release_gate.get('ready') is True).lower())}
    +
    Blocked{html_text(release_gate.get('blocked_count', 0))}/{html_text(release_gate.get('check_count', 0))}
    +
    Counts{html_text(str(release_gate.get('counts_as_completion') is True).lower())}
    +
    +

    {html_text(release_gate.get('final_manual_check', ''))}

    + {check_table} + """.strip() + + +def render_html_item(item: dict[str, Any]) -> str: + commands = "".join( + f"
  • {html_text(label.replace('_', ' '))}{html_text(command)}
    • " + for label, command in item["commands"].items() + ) + must_collect = item["must_collect"] + return f""" +
    +
    {html_text(item['category'])} · {html_text(item['review_state'])}

    {html_text(item['label'])}

    +

    {html_text(item['blocking_reason'])}

    +
    +
    Owner
    {html_text(item['owner'])}
    +
    Ledger
    {html_text(item['ledger_status'])}
    +
    Blocked
    {html_text(item.get('blocked_source_check_count', 0))}
    +
    Queue
    {html_text(item.get('phase_queue_blocked_count', 0))}
    +
    Submission
    {html_text(item['submission_path'])}
    +
    +

    Phase Queue

    {html_phase_queue(item.get('phase_queue', []))}
    +

    Source Runbook

      {html_list(item.get('execution_runbook', []), 'No source runbook listed.')}
    +

    Commands

      {commands}
    +
    +

    Must Collect

      {html_list(must_collect.get('provenance_requirements', []), 'No provenance requirements listed.')}
    +

    Success Checks

      {html_list(must_collect.get('success_checks', []), 'No success checks listed.')}
    +

    Privacy

      {html_list(must_collect.get('privacy_contract', []), 'No privacy contract listed.')}
    +
    +

    Next Source Actions

      {html_list(item.get('next_source_actions', []), 'No blocked source checks.')}
    +

    Source Evidence Snapshot

    {html_source_checks(item.get('source_checklist', []))}
    +
    + """.strip() + + +def render_html(report: dict[str, Any]) -> str: + summary = report["summary"] + stats = [ + ("Pending", summary["pending_count"]), + ("Awaiting", summary["awaiting_submission_count"]), + ("Ready", summary["ready_for_ledger_review_count"]), + ("Source", f"{summary.get('source_pass_count', 0)}/{summary.get('source_check_count', 0)}"), + ("Queue", f"{summary.get('phase_queue_blocked_count', 0)}/{summary.get('phase_queue_count', 0)}"), + ("Steps", f"{summary.get('coordination_user_required_step_count', 0)}/{summary.get('coordination_step_count', 0)}"), + ("Gate", f"{summary.get('release_gate_blocked_count', 0)}/{summary.get('release_gate_check_count', 0)}"), + ("Blocked", summary.get("source_blocked_count", 0)), + ("Invalid", summary["invalid_submission_count"]), + ] + stat_html = "".join(f"
    {html_text(label)}{html_text(value)}
    " for label, value in stats) + item_html = "".join(render_html_item(item) for item in report["items"]) + coordination_html = render_html_coordination_plan(report.get("coordination_plan", [])) + release_gate_html = render_html_release_gate(report.get("release_gate", {})) + return f""" + + + + + World-Class Operator Runbook + + + + +
    +
    + Evidence Operations +

    World-Class Operator Runbook

    +

    A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

    +
    {stat_html}
    +
    +

    Fast Path

    1. Run the real external or human work for one evidence item.
    2. Generate and fill the matching submission draft.
    3. Validate intake and inspect the submission review queue.
    4. Refresh the ledger and run the claim guard before making any completion claim.
    +

    Coordination Plan

    These steps tell the assistant what to run and where the user must supply real provider, reviewer, client, or telemetry input. Every row is operational guidance only and keeps counts_as_completion false.

    {coordination_html}
    +

    Phase Queue

    {html_phase_queue(report.get('phase_queue', []))}
    +

    Evidence Items

    {item_html}
    +

    Release Gate

    {release_gate_html}
    +

    Boundary

    • Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.
    • Valid intake means ready for submission review; ledger review still requires passing source evidence.
    • The world-class ledger and claim guard remain the source of truth.
    +
    + + +""" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render an operator runbook for collecting pending world-class evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_operator_runbook.json") + parser.add_argument("--output-md", default="reports/world_class_operator_runbook.md") + parser.add_argument("--output-html", default="reports/world_class_operator_runbook.html") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_operator_runbook(skill_dir, args.generated_at, submissions_dir=submissions_dir) + outputs = { + "json": Path(args.output_json), + "markdown": Path(args.output_md), + "html": Path(args.output_html), + } + for key, path in outputs.items(): + if not path.is_absolute(): + path = skill_dir / path + path.parent.mkdir(parents=True, exist_ok=True) + report["artifacts"][key] = output_path(str(path), skill_dir) + if key == "json": + path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + elif key == "markdown": + path.write_text(render_markdown(report), encoding="utf-8") + else: + path.write_text(render_html(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_preflight.py b/scripts/render_world_class_preflight.py new file mode 100644 index 00000000..0c69dbcc --- /dev/null +++ b/scripts/render_world_class_preflight.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import shlex +from datetime import date +from pathlib import Path +from typing import Any + +from prepare_world_class_submission_kit import build_artifact_checklist +from render_world_class_evidence_intake import build_intake +from render_world_class_evidence_ledger import build_ledger +from render_world_class_submission_review import build_submission_review +from world_class_evidence_contract import rel_path +from world_class_phase_queue import build_phase_queue, summarize_phase_queue +from world_class_preflight_markdown import render_markdown +from world_class_preflight_layout import render_html +from world_class_repair_checklist import build_preflight_repair_checklist, summarize_repair_checklist +from world_class_source_checks import summarize_source_checklist + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a preflight checklist for collecting pending world-class evidence without accepting evidence." + + +PREFLIGHT_SPECS: dict[str, list[dict[str, Any]]] = { + "provider-holdout": [ + { + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "path": "evals/output/cases.jsonl", + "required": True, + "next_action": "Keep output holdout cases available before provider execution.", + }, + { + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "path": "scripts/provider_output_eval_runner.py", + "required": True, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + }, + { + "key": "provider-api-key", + "label": "Provider credential", + "kind": "env_any", + "names": ["OPENAI_API_KEY", "DEEPSEEK_API_KEY"], + "required": True, + "secret": True, + "next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.", + }, + { + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "name": "YAO_OUTPUT_EVAL_MODEL", + "required": False, + "default": "provider-specific model, or pass --provider-model", + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.", + }, + ], + "human-adjudication": [ + { + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "path": "reports/output_review_kit.html", + "required": True, + "next_action": "Open the blind review kit and record real reviewer choices with required rationale.", + }, + { + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "path": "reports/output_review_decisions.json", + "required": True, + "next_action": "Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input --blind-review-attested --run-adjudication`.", + }, + { + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "path": "scripts/import_output_review_decisions.py", + "required": True, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + }, + { + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": True, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + }, + ], + "native-permission-enforcement": [ + { + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "path": "security/permission_policy.json", + "required": True, + "next_action": "Keep approved high-permission capabilities explicit.", + }, + { + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "path": "reports/runtime_permission_probes.json", + "required": True, + "next_action": "Refresh runtime permission probes after packaging changes.", + }, + { + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": True, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + }, + ], + "native-client-telemetry": [ + { + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "path": "scripts/telemetry_native_host.py", + "required": True, + "next_action": "Use the native host to receive metadata-only client events.", + }, + { + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "path": "reports/telemetry_hook_recipes.json", + "required": True, + "next_action": "Refresh telemetry hook recipes before external client installation.", + }, + { + "key": "external-client", + "label": "External client", + "kind": "external", + "required": True, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + }, + ], +} + + +def env_present(name: str) -> bool: + return bool(os.environ.get(name)) + + +def shell_path(path: Path, root: Path) -> str: + return shlex.quote(rel_path(path, root)) + + +def build_submission_commands(skill_dir: Path, submissions_dir: Path, evidence_key: str | None = None) -> dict[str, str]: + output_dir = shell_path(submissions_dir, skill_dir) + prepare = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir}" + if evidence_key: + prepare = f"python3 scripts/yao.py world-class-submission-kit . --evidence-key {evidence_key} --output-dir {output_dir}" + prefilled_prepare = f"{prepare} --prefill-artifacts" + return { + "prepare_submission": prepare, + "prepare_prefilled_submission": prefilled_prepare, + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir}", + "submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + } + + +def build_artifact_role_contract(rows: list[dict[str, Any]]) -> dict[str, Any]: + submission_ref_rows = [row for row in rows if row.get("submission_ref_required") is True] + supporting_rows = [row for row in rows if row.get("submission_ref_required") is not True] + return { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": False, + "artifact_prefill_counts_as_evidence": False, + "submission_ref_total_count": len(submission_ref_rows), + "submission_ref_ready_count": sum(1 for row in submission_ref_rows if row.get("artifact_ref_ready")), + "supporting_evidence_total_count": len(supporting_rows), + "supporting_evidence_ready_count": sum(1 for row in supporting_rows if row.get("artifact_ref_ready")), + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": True, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs.", + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": False, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.", + }, + ], + } + + +def artifact_role_contract_for_key(contract: dict[str, Any], rows: list[dict[str, Any]], evidence_key: str) -> dict[str, Any]: + item_rows = [row for row in rows if row.get("evidence_key") == evidence_key] + role_contract = build_artifact_role_contract(item_rows) + return { + **role_contract, + "schema_version": contract.get("schema_version", "1.0"), + "role_source": contract.get("role_source", "world-class-submission-kit"), + } + + +def build_precheck(skill_dir: Path, evidence_key: str, spec: dict[str, Any]) -> dict[str, Any]: + kind = str(spec.get("kind", "")) + required = spec.get("required") is True + row = { + "evidence_key": evidence_key, + "key": spec.get("key", ""), + "label": spec.get("label", ""), + "kind": kind, + "required": required, + "next_action": spec.get("next_action", ""), + "secret_value_redacted": spec.get("secret") is True, + } + if kind == "file": + path = skill_dir / str(spec.get("path", "")) + exists = path.exists() + row.update( + { + "path": rel_path(path, skill_dir), + "status": "pass" if exists else ("missing" if required else "optional"), + "actual": "present" if exists else "missing", + } + ) + return row + if kind == "env": + name = str(spec.get("name", "")) + present = env_present(name) + row.update( + { + "env": name, + "status": "pass" if present else ("missing" if required else "optional"), + "actual": "set" if present else "not-set", + "default": spec.get("default", ""), + } + ) + return row + if kind == "env_any": + names = [str(name) for name in spec.get("names", []) if str(name).strip()] + set_names = [name for name in names if env_present(name)] + row.update( + { + "env_any": names, + "status": "pass" if set_names else ("missing" if required else "optional"), + "actual": "set" if set_names else "not-set", + "set_count": len(set_names), + "set_envs": set_names, + } + ) + return row + if kind == "human": + row.update( + { + "status": "human-required", + "actual": "external-human-action", + } + ) + return row + if kind == "external": + row.update( + { + "status": "external-required", + "actual": "external-integration-required", + } + ) + return row + row.update({"status": "unknown", "actual": "unknown"}) + return row + + +def item_status(entry: dict[str, Any], prechecks: list[dict[str, Any]], source_rows: list[dict[str, Any]]) -> str: + if entry.get("status") == "accepted": + return "accepted" + blocking = {row.get("status") for row in prechecks if row.get("required") is True} + if "missing" in blocking or "external-required" in blocking: + return "blocked" + if "human-required" in blocking: + return "ready-for-human-review" + if any(row.get("status") != "pass" for row in source_rows): + return "ready-to-collect" + return "ready-for-submission" + + +def first_next_action(prechecks: list[dict[str, Any]], source_rows: list[dict[str, Any]]) -> str: + for row in prechecks: + if row.get("required") is True and row.get("status") != "pass": + return str(row.get("next_action", "")) + for row in source_rows: + if row.get("status") != "pass": + return str(row.get("next_action", "")) + return "Validate intake, refresh the ledger, and run the claim guard." + + +def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir) + intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir) + review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir) + review_by_key = {str(item.get("evidence_key", "")): item for item in review.get("items", [])} + intake_by_key = {str(item.get("evidence_key", "")): item for item in intake.get("operator_checklist", [])} + operator_items = intake.get("operator_checklist", []) + operator_items = operator_items if isinstance(operator_items, list) else [] + artifact_checklist = build_artifact_checklist(skill_dir, operator_items) + artifact_role_contract = build_artifact_role_contract(artifact_checklist) + items: list[dict[str, Any]] = [] + precheck_rows: list[dict[str, Any]] = [] + source_rows: list[dict[str, Any]] = [] + for entry in ledger.get("entries", []): + key = str(entry.get("key", "")) + prechecks = [build_precheck(skill_dir, key, spec) for spec in PREFLIGHT_SPECS.get(key, [])] + review_item = review_by_key.get(key, {}) + item_source_rows = review_item.get("source_checklist", entry.get("source_checklist", [])) + item_source_rows = item_source_rows if isinstance(item_source_rows, list) else [] + status = item_status(entry, prechecks, item_source_rows) + item_commands = build_submission_commands(skill_dir, submissions_dir, key) + item = { + "evidence_key": key, + "label": entry.get("label", key), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "status": status, + "ledger_status": entry.get("status", "pending"), + "intake_readiness": intake_by_key.get(key, {}).get("readiness", "missing"), + "review_state": review_item.get("review_state", "missing"), + "source_accepted": entry.get("source_accepted") is True, + "prechecks": prechecks, + "source_checklist": item_source_rows, + "next_action": first_next_action(prechecks, item_source_rows), + "submission_path": intake_by_key.get(key, {}).get("submission_path", ""), + "template_path": intake_by_key.get(key, {}).get("template_path", ""), + "commands": item_commands, + "submission_kit": { + "prepare_command": item_commands["prepare_submission"], + "prefill_command": item_commands["prepare_prefilled_submission"], + "output_dir": rel_path(submissions_dir, skill_dir), + "draft_path": intake_by_key.get(key, {}).get("submission_path", ""), + "template_path": intake_by_key.get(key, {}).get("template_path", ""), + "artifact_role_contract": artifact_role_contract_for_key( + artifact_role_contract, + artifact_checklist, + key, + ), + "drafts_count_as_evidence": False, + "artifact_prefill_counts_as_evidence": False, + }, + "runbook": entry.get("runbook", []), + } + items.append(item) + precheck_rows.extend(prechecks) + source_rows.extend(item_source_rows) + repair_checklist = build_preflight_repair_checklist(items) + repairs_by_key: dict[str, list[dict[str, Any]]] = {} + for row in repair_checklist: + key = str(row.get("evidence_key", "")) + repairs_by_key.setdefault(key, []).append(row) + for item in items: + item_repair_rows = repairs_by_key.get(str(item.get("evidence_key", "")), []) + item["repair_checklist"] = item_repair_rows + item["phase_queue"] = build_phase_queue(item_repair_rows) + precheck_status_counts: dict[str, int] = {} + for row in precheck_rows: + status = str(row.get("status", "unknown")) + precheck_status_counts[status] = precheck_status_counts.get(status, 0) + 1 + source_summary = summarize_source_checklist(source_rows) + repair_summary = summarize_repair_checklist(repair_checklist) + phase_queue = build_phase_queue(repair_checklist) + phase_queue_summary = summarize_phase_queue(phase_queue) + collection_ready_count = sum(1 for item in items if item["status"] in {"ready-to-collect", "ready-for-human-review", "ready-for-submission"}) + blocked_count = sum(1 for item in items if item["status"] == "blocked") + ready_to_claim = ledger.get("summary", {}).get("ready_to_claim_world_class") is True + summary = { + "evidence_item_count": len(items), + "precheck_count": len(precheck_rows), + "precheck_pass_count": precheck_status_counts.get("pass", 0), + "precheck_missing_count": precheck_status_counts.get("missing", 0), + "precheck_optional_count": precheck_status_counts.get("optional", 0), + "precheck_human_required_count": precheck_status_counts.get("human-required", 0), + "precheck_external_required_count": precheck_status_counts.get("external-required", 0), + "collection_ready_count": collection_ready_count, + "collection_blocked_count": blocked_count, + **source_summary, + **repair_summary, + **phase_queue_summary, + "pending_count": ledger.get("summary", {}).get("pending_count", 0), + "ready_to_claim_world_class": ready_to_claim, + "credential_value_exposed": False, + "preflight_counts_as_evidence": False, + "decision": "ready-for-completion-audit" if ready_to_claim else ("collection-preflight-blocked" if blocked_count else "ready-to-collect-evidence"), + } + return { + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": summary, + "status_counts": {item["status"]: sum(1 for candidate in items if candidate["status"] == item["status"]) for item in items}, + "precheck_status_counts": precheck_status_counts, + "items": items, + "prechecks": precheck_rows, + "source_checklist": source_rows, + "repair_checklist": repair_checklist, + "phase_queue": phase_queue, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "commands": build_submission_commands(skill_dir, submissions_dir), + "submission_kit_command": build_submission_commands(skill_dir, submissions_dir)["prepare_submission"], + "submission_kit_prefill_command": build_submission_commands(skill_dir, submissions_dir)[ + "prepare_prefilled_submission" + ], + "preflight_counts_submission_as_completion": False, + "drafts_count_as_evidence": False, + "artifact_prefill_counts_as_evidence": False, + "artifact_role_contract": artifact_role_contract, + }, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "operator_runbook": "reports/world_class_operator_runbook.json", + }, + "artifacts": { + "json": "reports/world_class_evidence_preflight.json", + "markdown": "reports/world_class_evidence_preflight.md", + "html": "reports/world_class_evidence_preflight.html", + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render collection preflight checks for pending world-class evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_evidence_preflight.json") + parser.add_argument("--output-md", default="reports/world_class_evidence_preflight.md") + parser.add_argument("--output-html", default="reports/world_class_evidence_preflight.html") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_preflight(skill_dir, args.generated_at, submissions_dir=submissions_dir) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + output_html = Path(args.output_html) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + if not output_html.is_absolute(): + output_html = skill_dir / output_html + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_html.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + output_html.write_text(render_html(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_submission_review.py b/scripts/render_world_class_submission_review.py new file mode 100644 index 00000000..ae550749 --- /dev/null +++ b/scripts/render_world_class_submission_review.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_intake import build_intake +from render_world_class_evidence_ledger import build_ledger +from world_class_source_checks import build_source_checklist, summarize_source_checklist + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a read-only review queue for world-class evidence submissions before ledger acceptance." + +TOP_LEVEL_SUMMARY_FIELDS = [ + "decision", + "ready_to_claim_world_class", + "review_item_count", + "accepted_count", + "awaiting_submission_count", + "valid_packet_source_incomplete_count", + "ready_for_ledger_review_count", + "fix_submission_count", + "unmatched_submission_count", + "invalid_submission_count", + "source_check_count", + "source_pass_count", + "source_blocked_count", + "review_counts_submission_as_completion", +] + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def by_key(items: list[dict[str, Any]], key_name: str = "evidence_key") -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = {} + for item in items: + key = str(item.get(key_name, "")) + if key and key not in result: + result[key] = item + return result + + +def top_level_summary_mirrors(summary: dict[str, Any]) -> dict[str, Any]: + return {key: summary[key] for key in TOP_LEVEL_SUMMARY_FIELDS if key in summary} + + +def review_state(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> tuple[str, str]: + if entry.get("status") == "accepted": + return "accepted", "Ledger already accepts this evidence item." + + submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} + submission_status = str(submission.get("status", "missing")) + if submission_status == "missing": + return "awaiting-submission", "No evidence packet has been submitted for review." + if submission_status in {"invalid-json", "invalid-contract"}: + return "fix-submission", "Submission exists but fails the ledger submission contract." + if intake_result is None: + return "fix-submission", "Submission exists but is missing from the intake validation results." + if intake_result.get("status") != "pass": + return "fix-submission", "Submission exists but failed intake validation." + + observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + if observed.get("accepted") is True: + return "ready-for-ledger-review", "Submission and source evidence are ready for final ledger review." + return "source-evidence-incomplete", "Submission packet is valid, but the source evidence checks still do not pass." + + +def build_item(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> dict[str, Any]: + state, reason = review_state(entry, intake_result) + submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} + observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + source_checklist = build_source_checklist([entry]) + source_summary = summarize_source_checklist(source_checklist) + return { + "evidence_key": entry.get("key", ""), + "label": entry.get("label", entry.get("key", "")), + "category": entry.get("category", "external"), + "owner": entry.get("owner", "release reviewer"), + "ledger_status": entry.get("status", "pending"), + "submission_status": submission.get("status", "missing"), + "intake_status": intake_result.get("status", "missing") if intake_result else "missing", + "source_accepted": observed.get("accepted") is True, + "review_state": state, + "blocking_reason": reason, + "submission_path": submission.get("path", ""), + "submitted_by": submission.get("submitted_by", ""), + "submitted_at": submission.get("submitted_at", ""), + "artifact_ref_count": submission.get("artifact_ref_count", 0), + "intake_errors": intake_result.get("errors", []) if intake_result else [], + "observed_state": observed, + "source_checklist": source_checklist, + **source_summary, + "success_checks": entry.get("success_checks", []), + "privacy_contract": entry.get("privacy_contract", []), + "next_action": entry.get("next_action", ""), + } + + +def build_submission_review(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: + submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") + ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir) + intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir) + intake_by_key = by_key(intake.get("submissions", [])) + entries = ledger.get("entries", []) + items = [build_item(entry, intake_by_key.get(str(entry.get("key", "")))) for entry in entries] + known_keys = {str(entry.get("key", "")) for entry in entries} + unmatched = [item for item in intake.get("submissions", []) if str(item.get("evidence_key", "")) not in known_keys] + counts: dict[str, int] = {} + for item in items: + state = str(item.get("review_state", "unknown")) + counts[state] = counts.get(state, 0) + 1 + invalid_count = counts.get("fix-submission", 0) + len(unmatched) + ready_count = counts.get("ready-for-ledger-review", 0) + accepted_count = counts.get("accepted", 0) + source_incomplete_count = counts.get("source-evidence-incomplete", 0) + awaiting_count = counts.get("awaiting-submission", 0) + source_rows = [row for item in items for row in item.get("source_checklist", [])] + source_summary = summarize_source_checklist(source_rows) + if accepted_count == len(items) and items: + decision = "ledger-complete" + elif invalid_count: + decision = "fix-submissions" + elif ready_count: + decision = "review-ready" + elif source_incomplete_count: + decision = "source-evidence-incomplete" + else: + decision = "awaiting-submissions" + summary = { + "review_item_count": len(items), + "accepted_count": accepted_count, + "awaiting_submission_count": awaiting_count, + "valid_packet_source_incomplete_count": source_incomplete_count, + "ready_for_ledger_review_count": ready_count, + "fix_submission_count": counts.get("fix-submission", 0), + "unmatched_submission_count": len(unmatched), + "invalid_submission_count": invalid_count, + **source_summary, + "ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True, + "review_counts_submission_as_completion": False, + "decision": decision, + } + return { + "schema_version": "1.0", + "ok": invalid_count == 0, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + **top_level_summary_mirrors(summary), + "summary": summary, + "report_contract": { + "schema_version": "1.0", + "contract": "world-class-submission-review", + "top_level_mirrors_summary": True, + "summary_fields": TOP_LEVEL_SUMMARY_FIELDS, + "source_of_truth": "summary", + }, + "submissions": { + "directory": rel_path(submissions_dir, skill_dir), + "review_counts_submission_as_completion": False, + }, + "items": items, + "unmatched_submissions": unmatched, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + }, + "artifacts": { + "json": "reports/world_class_submission_review.json", + "markdown": "reports/world_class_submission_review.md", + }, + } + + +def render_list(values: list[Any], empty: str) -> list[str]: + if not values: + return [f"- {empty}"] + return [f"- {value}" for value in values] + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# World-Class Submission Review", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- review items: `{summary['review_item_count']}`", + f"- accepted: `{summary['accepted_count']}`", + f"- awaiting submission: `{summary['awaiting_submission_count']}`", + f"- valid packet but source incomplete: `{summary['valid_packet_source_incomplete_count']}`", + f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`", + f"- fix submission: `{summary['fix_submission_count']}`", + f"- unmatched submissions: `{summary['unmatched_submission_count']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- review counts submission as completion: `{str(summary['review_counts_submission_as_completion']).lower()}`", + "", + "This report is a read-only reviewer queue. It does not accept evidence or make world-class completion true.", + "", + "## Queue", + "", + "| Evidence | Review state | Intake | Source accepted | Submission | Next action |", + "| --- | --- | --- | --- | --- | --- |", + ] + for item in report["items"]: + source_accepted = str(item.get("source_accepted") is True).lower() + next_action = str(item.get("next_action", "")).replace("|", "\\|") + lines.append( + f"| `{item['evidence_key']}` | `{item['review_state']}` | `{item['intake_status']}` | " + f"`{source_accepted}` | `{item['submission_status']}` | {next_action} |" + ) + if report.get("unmatched_submissions"): + lines.extend(["", "## Unmatched Submissions", ""]) + for item in report["unmatched_submissions"]: + errors = "; ".join(item.get("errors", [])) or "unknown evidence key" + lines.append(f"- `{item.get('path', '')}`: {errors}") + lines.extend(["", "## Details", ""]) + for item in report["items"]: + lines.extend( + [ + f"### {item['label']}", + "", + f"- review state: `{item['review_state']}`", + f"- blocking reason: {item['blocking_reason']}", + f"- ledger status: `{item['ledger_status']}`", + f"- submission status: `{item['submission_status']}`", + f"- intake status: `{item['intake_status']}`", + f"- source accepted: `{str(item['source_accepted']).lower()}`", + f"- submission path: `{item.get('submission_path') or 'missing'}`", + "", + "#### Source Checks", + "", + *render_list( + [ + f"{row['label']}: {row['actual']} / {row['expected']} => {row['status']}" + for row in item.get("source_checklist", []) + ], + "No source checks listed.", + ), + "", + "#### Completion Assertions", + "", + *render_list(item.get("success_checks", []), "No completion assertions listed."), + "", + "#### Intake Errors", + "", + *render_list(item.get("intake_errors", []), "No intake errors."), + "", + "#### Privacy Contract", + "", + *render_list(item.get("privacy_contract", []), "No privacy contract listed."), + "", + ] + ) + lines.extend( + [ + "## Boundary", + "", + "- A valid submission packet is not accepted evidence by itself.", + "- Planned work, metadata fallback, pending human review, and local command-runner output still do not count.", + "- The world-class ledger remains the source of truth for `ready_to_claim_world_class`.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a read-only review queue for world-class evidence submissions.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--submissions-dir") + parser.add_argument("--output-json", default="reports/world_class_submission_review.json") + parser.add_argument("--output-md", default="reports/world_class_submission_review.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None + report = build_submission_review(skill_dir, args.generated_at, submissions_dir=submissions_dir) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/resource_boundary_check.py b/scripts/resource_boundary_check.py index 92d9b52e..fe7ea65a 100644 --- a/scripts/resource_boundary_check.py +++ b/scripts/resource_boundary_check.py @@ -14,6 +14,27 @@ IGNORED_RELATIVE_DIRS = { Path("tests") / "tmp_snapshot", Path("tests") / "tmp_cli", } +IGNORED_FILE_PATTERNS = { + "reports/benchmark_reproducibility*.json", + "reports/benchmark_reproducibility*.md", + "reports/context_budget*.json", + "reports/context_budget*.md", + "reports/evidence_consistency*.json", + "reports/evidence_consistency*.md", + "reports/review-studio*.html", + "reports/review-studio*.json", + "reports/review-viewer*.html", + "reports/review-viewer*.json", + "reports/skill-interpretation*.html", + "reports/skill-interpretation*.json", + "reports/skill-overview*.html", + "reports/skill-overview*.json", + "reports/world_class_evidence_preflight*.json", + "reports/world_class_evidence_preflight*.md", + "reports/world_class_evidence_preflight*.html", + "reports/*pattern-analysis*.md", + "reports/*research-plan*.md", +} CANONICAL_PATHS = ( "SKILL.md", "manifest.json", @@ -37,6 +58,14 @@ CONTEXT_BUDGETS = { } SKILL_BODY_BUFFER = 100 SKILL_BODY_WARN_RATIO = 0.85 +DEFERRED_RESOURCE_DIRS = {"references", "scripts", "evals", "templates", "assets", "input", "outputs"} +DEFERRED_RESOURCE_WARN_TOKENS = 120_000 +DEFERRED_RESOURCE_WARN_DIR_TOKENS = 80_000 +SCRIPT_GOVERNANCE_REPORTS = ( + "reports/security_trust_report.json", + "reports/architecture_maintainability.json", + "reports/python_compatibility.json", +) def has_files(path: Path) -> bool: @@ -47,6 +76,8 @@ def should_ignore(path: Path, root: Path) -> bool: rel = path.relative_to(root) if any(rel == ignored or ignored in rel.parents for ignored in IGNORED_RELATIVE_DIRS): return True + if any(rel.match(pattern) for pattern in IGNORED_FILE_PATTERNS): + return True return len(rel.parts) >= 2 and rel.parts[0] == "tests" and rel.parts[1].startswith("tmp_") @@ -56,6 +87,16 @@ def load_manifest(path: Path) -> dict: return json.loads(path.read_text(encoding="utf-8")) +def load_json(path: Path) -> dict: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + def iter_relevant_files(root: Path) -> list[Path]: files = [] for entry in CANONICAL_PATHS: @@ -103,6 +144,111 @@ def quality_signal_points(root: Path, manifest: dict, governance_score: int) -> return points +def script_resource_governance(root: Path, expected_file_count: int) -> dict: + trust = load_json(root / "reports" / "security_trust_report.json") + architecture = load_json(root / "reports" / "architecture_maintainability.json") + python_compat = load_json(root / "reports" / "python_compatibility.json") + trust_summary = trust.get("summary", {}) if isinstance(trust.get("summary", {}), dict) else {} + architecture_summary = architecture.get("summary", {}) if isinstance(architecture.get("summary", {}), dict) else {} + python_summary = python_compat.get("summary", {}) if isinstance(python_compat.get("summary", {}), dict) else {} + + reasons = [] + missing = [] + checks = [ + ( + "trust report covers scripts", + trust.get("ok") is True + and int(trust_summary.get("script_count", 0) or 0) >= expected_file_count + and int(trust_summary.get("secret_findings", 0) or 0) == 0 + and int(trust_summary.get("help_smoke_failed_count", 0) or 0) == 0, + ), + ( + "architecture report has no script hotspots or blockers", + architecture.get("ok") is True + and int(architecture_summary.get("hotspot_count", 0) or 0) == 0 + and int(architecture_summary.get("blocker_count", 0) or 0) == 0, + ), + ( + "Python compatibility report has no issues", + python_compat.get("ok") is True + and int(python_summary.get("issue_count", 0) or 0) == 0, + ), + ] + for label, ok in checks: + if ok: + reasons.append(label) + else: + missing.append(label) + return { + "status": "governed" if not missing else "needs-review", + "evidence": list(SCRIPT_GOVERNANCE_REPORTS), + "reasons": reasons, + "missing": missing, + } + + +def deferred_dir_governance( + root: Path, + dirname: str, + payload: dict, + manifest: dict, + skill_text: str, +) -> dict: + file_count = int(payload.get("file_count", 0) or 0) + if dirname == "scripts": + governance = script_resource_governance(root, file_count) + governance.update( + { + "path": dirname, + "estimated_tokens": int(payload.get("estimated_tokens", 0) or 0), + "file_count": file_count, + "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context.", + } + ) + return governance + + referenced = explicit_dir_reference(dirname, root / dirname, skill_text, manifest) + declared = dirname in (manifest.get("factory_components") or []) + governed = referenced or declared + return { + "path": dirname, + "status": "governed" if governed else "needs-review", + "estimated_tokens": int(payload.get("estimated_tokens", 0) or 0), + "file_count": file_count, + "evidence": ["SKILL.md", "manifest.json"], + "reasons": ["directory is explicitly referenced or declared as a factory component"] if governed else [], + "missing": [] if governed else ["directory is not referenced in SKILL.md or manifest factory_components"], + "rationale": "Deferred resources are acceptable when they are discoverable and intentionally part of the package contract.", + } + + +def deferred_resource_governance( + root: Path, + manifest: dict, + skill_text: str, + deferred_resource_dirs: dict[str, dict[str, int | str]], + large_deferred_resource_dirs: list[dict], +) -> dict: + governed_dirs = [ + deferred_dir_governance(root, str(item["path"]), item, manifest, skill_text) + for item in large_deferred_resource_dirs + ] + missing = [item for item in governed_dirs if item["status"] != "governed"] + return { + "status": "governed" if governed_dirs and not missing else ("not-required" if not governed_dirs else "needs-review"), + "large_dir_count": len(governed_dirs), + "governed_large_dir_count": len(governed_dirs) - len(missing), + "directories": governed_dirs, + "summary": ( + "Large deferred resources are indexed and backed by evidence." + if governed_dirs and not missing + else "No large deferred resource directory exceeds the per-dir threshold." + if not governed_dirs + else "One or more large deferred resource directories still need explicit governance evidence." + ), + } + + def analyze_skill( root: Path, max_initial_tokens: int | None = None, @@ -122,6 +268,8 @@ def analyze_skill( other_tokens = 0 initial_load_tokens = 0 total_text_tokens = 0 + deferred_resource_tokens = 0 + deferred_resource_dirs: dict[str, dict[str, int | str]] = {} for path in files: if path.suffix and path.suffix not in TEXT_EXTS and path.name != "SKILL.md": continue @@ -129,6 +277,7 @@ def analyze_skill( tokens = estimate_tokens(text) total_text_tokens += tokens rel = path.relative_to(root) + top_dir = rel.parts[0] if rel.parts else str(rel) if rel == Path("SKILL.md"): skill_body_tokens += tokens initial_load_tokens += tokens @@ -136,6 +285,14 @@ def analyze_skill( other_tokens += tokens if rel.parts[0] in {"agents"}: initial_load_tokens += tokens + if top_dir in DEFERRED_RESOURCE_DIRS: + deferred_resource_tokens += tokens + current = deferred_resource_dirs.setdefault( + top_dir, + {"path": top_dir, "estimated_tokens": 0, "file_count": 0}, + ) + current["estimated_tokens"] = int(current["estimated_tokens"]) + tokens + current["file_count"] = int(current["file_count"]) + 1 budget_tier = budget_tier_for(manifest) budget_limit = max_initial_tokens if max_initial_tokens is not None else CONTEXT_BUDGETS[budget_tier] @@ -168,6 +325,30 @@ def analyze_skill( if other_tokens and skill_body_tokens / (skill_body_tokens + other_tokens) > 0.75: warnings.append("Most text still lives in SKILL.md; consider moving detail into references/ or scripts/.") + large_deferred_resource_dirs = [ + item + for item in sorted( + deferred_resource_dirs.values(), + key=lambda payload: int(payload["estimated_tokens"]), + reverse=True, + ) + if int(item["estimated_tokens"]) > DEFERRED_RESOURCE_WARN_DIR_TOKENS + ] + deferred_governance = deferred_resource_governance( + root, + manifest, + skill_text, + deferred_resource_dirs, + large_deferred_resource_dirs, + ) + + if deferred_resource_tokens > DEFERRED_RESOURCE_WARN_TOKENS and deferred_governance["status"] != "governed": + warnings.append( + "Deferred resource footprint is high: " + f"{deferred_resource_tokens} estimated tokens across references/scripts/evals. " + "Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." + ) + frontmatter = read_frontmatter(skill_md) governance_score, _ = compute_score(root, manifest, frontmatter, skill_text, bool(manifest)) signal_points = quality_signal_points(root, manifest, governance_score) @@ -184,6 +365,15 @@ def analyze_skill( "other_text_tokens": other_tokens, "estimated_initial_load_tokens": initial_load_tokens, "estimated_total_text_tokens": total_text_tokens, + "deferred_resource_tokens": deferred_resource_tokens, + "deferred_resource_warn_threshold": DEFERRED_RESOURCE_WARN_TOKENS, + "deferred_resource_dirs": sorted( + deferred_resource_dirs.values(), + key=lambda payload: int(payload["estimated_tokens"]), + reverse=True, + ), + "large_deferred_resource_dirs": large_deferred_resource_dirs, + "deferred_resource_governance": deferred_governance, "relevant_file_count": len(files), "unused_resource_dirs": unused_resource_dirs, "quality_signal_points": signal_points, diff --git a/scripts/review_studio_action_evidence.py b/scripts/review_studio_action_evidence.py new file mode 100644 index 00000000..6fc684f9 --- /dev/null +++ b/scripts/review_studio_action_evidence.py @@ -0,0 +1,388 @@ +import html +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by review_studio_actions.py to keep world-class evidence action cards out of generic Review Studio action wiring." + + +def action_command_rows(commands: dict[str, Any]) -> list[dict[str, str]]: + command_labels = { + "prepare_submission": "准备提交", + "validate_intake": "校验入口", + "submission_review": "审查提交", + "refresh_ledger": "刷新台账", + "guard_claim": "声明守卫", + } + rows = [] + for key in ("prepare_submission", "validate_intake", "submission_review", "refresh_ledger", "guard_claim"): + command = str(commands.get(key, "")).strip() + if command: + rows.append({"key": key, "label": command_labels.get(key, key), "command": command}) + return rows + + +def first_text_items(*values: Any, limit: int = 4) -> list[str]: + for value in values: + if isinstance(value, list) and value: + return [str(item) for item in value[:limit]] + return [] + + +def artifact_role_rows(contract: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + roles = contract.get("roles", []) if isinstance(contract.get("roles", []), list) else [] + for role in roles: + if not isinstance(role, dict): + continue + role_name = str(role.get("role", "")).strip() + if role_name == "submission-ref": + ready_count = int(contract.get("submission_ref_ready_count", 0) or 0) + total_count = int(contract.get("submission_ref_total_count", 0) or 0) + elif role_name == "supporting-evidence": + ready_count = int(contract.get("supporting_evidence_ready_count", 0) or 0) + total_count = int(contract.get("supporting_evidence_total_count", 0) or 0) + else: + ready_count = 0 + total_count = 0 + rows.append( + { + "role": role_name, + "label": str(role.get("label", role_name)), + "ready_count": ready_count, + "total_count": total_count, + "copy_to_artifact_refs": role.get("copy_to_artifact_refs") is True, + "description": str(role.get("description", "")), + } + ) + return rows + + +def compact_artifact_role_contract(contract: dict[str, Any]) -> dict[str, Any]: + if not isinstance(contract, dict) or not contract: + return {} + return { + "role_source": str(contract.get("role_source", "")), + "counts_as_evidence": contract.get("counts_as_evidence") is True, + "artifact_prefill_counts_as_evidence": contract.get("artifact_prefill_counts_as_evidence") is True, + "submission_ref_ready_count": int(contract.get("submission_ref_ready_count", 0) or 0), + "submission_ref_total_count": int(contract.get("submission_ref_total_count", 0) or 0), + "supporting_evidence_ready_count": int(contract.get("supporting_evidence_ready_count", 0) or 0), + "supporting_evidence_total_count": int(contract.get("supporting_evidence_total_count", 0) or 0), + "roles": artifact_role_rows(contract), + } + + +def compact_phase_queue(rows: list[Any]) -> list[dict[str, Any]]: + compact_rows: list[dict[str, Any]] = [] + for row in rows: + if not isinstance(row, dict): + continue + compact_rows.append( + { + "phase": str(row.get("phase", "")), + "label": str(row.get("label", "")), + "priority": int(row.get("priority", 90) or 90), + "status": str(row.get("status", "blocked")), + "blocked_count": int(row.get("blocked_count", 0) or 0), + "row_count": int(row.get("row_count", 0) or 0), + "owners": [str(item) for item in row.get("owners", []) if str(item).strip()], + "evidence_keys": [str(item) for item in row.get("evidence_keys", []) if str(item).strip()], + "next_action_id": str(row.get("next_action_id", "")), + "next_action": str(row.get("next_action", "")), + "verification_command": str(row.get("verification_command", "")), + "counts_as_completion": row.get("counts_as_completion") is True, + } + ) + return compact_rows + + +def world_class_action_steps(data: dict[str, Any]) -> list[dict[str, Any]]: + ledger = data.get("world_class_evidence_ledger", {}) if isinstance(data, dict) else {} + entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] + intake = data.get("world_class_evidence_intake", {}) if isinstance(data, dict) else {} + intake_items = intake.get("operator_checklist", []) if isinstance(intake, dict) else [] + intake_by_key = { + str(item.get("evidence_key", "")): item + for item in intake_items + if isinstance(item, dict) and str(item.get("evidence_key", "")).strip() + } + preflight = data.get("world_class_evidence_preflight", {}) if isinstance(data, dict) else {} + preflight_items = preflight.get("items", []) if isinstance(preflight, dict) else [] + preflight_by_key = { + str(item.get("evidence_key", "")): item + for item in preflight_items + if isinstance(item, dict) and str(item.get("evidence_key", "")).strip() + } + steps = [] + for entry in entries: + if not isinstance(entry, dict): + continue + key = str(entry.get("key", "")).strip() + if not key: + continue + checklist = intake_by_key.get(key, {}) + preflight_item = preflight_by_key.get(key, {}) + submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} + source_checks = entry.get("source_checklist", []) if isinstance(entry.get("source_checklist", []), list) else [] + blocked_checks = [ + { + "label": str(row.get("label", "")), + "field": str(row.get("field", "")), + "actual": row.get("actual", ""), + "expected": str(row.get("expected", "")), + "status": str(row.get("status", "blocked")), + "next_action": str(row.get("next_action", "")), + } + for row in source_checks + if isinstance(row, dict) and str(row.get("status", "")) != "pass" + ] + repair_rows = [] + for row in preflight_item.get("repair_checklist", []) if isinstance(preflight_item, dict) else []: + if not isinstance(row, dict): + continue + repair_rows.append( + { + "action_id": str(row.get("action_id", "")), + "repair_type": str(row.get("repair_type", "")), + "target": str(row.get("target", "")), + "phase": str(row.get("phase", "")), + "priority": int(row.get("priority", 90) or 90), + "owner": str(row.get("owner", "")), + "status": str(row.get("status", "blocked")), + "blocking_reason": str(row.get("blocking_reason", "")), + "next_action": str(row.get("next_action", "")), + "verification_command": str(row.get("verification_command", "")), + "counts_as_completion": row.get("counts_as_completion") is True, + } + ) + derived_pass_count = sum(1 for row in source_checks if isinstance(row, dict) and str(row.get("status", "")) == "pass") + runbook = entry.get("runbook", []) + if not runbook and isinstance(checklist.get("must_collect", {}), dict): + runbook = checklist["must_collect"].get("runbook", []) + must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {} + submission_kit = ( + preflight_item.get("submission_kit", {}) + if isinstance(preflight_item.get("submission_kit", {}), dict) + else {} + ) + phase_queue = compact_phase_queue( + preflight_item.get("phase_queue", []) if isinstance(preflight_item.get("phase_queue", []), list) else [] + ) + steps.append( + { + "key": key, + "label": str(entry.get("label", key)), + "category": str(entry.get("category", "")), + "status": str(entry.get("status", "pending")), + "readiness": str(checklist.get("readiness", "")), + "current": str(entry.get("current", "")), + "next_action": str(entry.get("next_action") or checklist.get("next_action", "")), + "submission_path": str(submission.get("path") or checklist.get("submission_path", "")), + "template_path": str(checklist.get("template_path", "")), + "source_pass_count": int(entry.get("source_pass_count", 0) or derived_pass_count), + "source_blocked_count": int(entry.get("source_blocked_count", 0) or len(blocked_checks)), + "blocked_checks": blocked_checks, + "repair_rows": repair_rows, + "repair_blocked_count": sum(1 for row in repair_rows if row.get("status") != "ready"), + "repair_counts_as_completion": any(row.get("counts_as_completion") for row in repair_rows), + "phase_queue": phase_queue, + "phase_queue_blocked_count": sum(1 for row in phase_queue if row.get("status") != "ready"), + "phase_queue_counts_as_completion": any(row.get("counts_as_completion") for row in phase_queue), + "commands": action_command_rows(checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {}), + "runbook": [str(item) for item in runbook[:3]], + "provenance_requirements": first_text_items( + must_collect.get("provenance_requirements"), + entry.get("provenance_requirements"), + limit=3, + ), + "success_checks": first_text_items( + must_collect.get("success_checks"), + entry.get("success_checks"), + limit=9, + ), + "evidence_artifacts": first_text_items( + must_collect.get("evidence_artifacts"), + entry.get("evidence_artifacts"), + limit=5, + ), + "artifact_role_contract": compact_artifact_role_contract( + submission_kit.get("artifact_role_contract", {}) + if isinstance(submission_kit.get("artifact_role_contract", {}), dict) + else {} + ), + "privacy_contract": first_text_items( + must_collect.get("privacy_contract"), + entry.get("privacy_contract"), + limit=4, + ), + } + ) + return steps + + +def render_small_list(items: list[Any], empty: str, ordered: bool = False) -> str: + if not items: + return f"

    {html.escape(empty)}

    " + tag = "ol" if ordered else "ul" + rows = "".join(f"
  • {html.escape(str(item))}
    • " for item in items) + return f"<{tag}>{rows}" + + +def render_artifact_role_contract(contract: dict[str, Any]) -> str: + roles = contract.get("roles", []) if isinstance(contract, dict) else [] + if not roles: + return "

    暂无资产角色。

    " + rows = [] + for role in roles: + rows.append( + "
  • " + f"{html.escape(str(role.get('role', '')))}" + f"{html.escape(str(role.get('ready_count', 0)))} / {html.escape(str(role.get('total_count', 0)))} ready" + f"artifact_refs: {html.escape(str(role.get('copy_to_artifact_refs') is True).lower())}" + f"{html.escape(str(role.get('description', '')))}" + "
    • " + ) + source = html.escape(str(contract.get("role_source", ""))) + counts = html.escape(str(contract.get("counts_as_evidence") is True).lower()) + prefill = html.escape(str(contract.get("artifact_prefill_counts_as_evidence") is True).lower()) + return ( + f"

    source: {source}; counts as evidence: {counts}; " + f"prefill counts as evidence: {prefill}

    " + "
      " + + "".join(rows) + + "
    " + ) + + +def render_action_evidence_steps(steps: list[dict[str, Any]]) -> str: + if not steps: + return "" + cards = [] + for step in steps: + blocked_checks = [] + for check in step.get("blocked_checks", []): + blocked_checks.append( + "
  • " + f"{html.escape(str(check.get('label', '')))}" + f"{html.escape(str(check.get('field', '')))}: {html.escape(str(check.get('actual', '')))} / {html.escape(str(check.get('expected', '')))}" + f"{html.escape(str(check.get('next_action', '')))}" + "
    • " + ) + command_rows = [] + for command in step.get("commands", [])[:4]: + command_rows.append( + "
  • " + f"{html.escape(str(command.get('label', '')))}" + f"{html.escape(str(command.get('command', '')))}" + "
    • " + ) + runbook_rows = [f"
  • {html.escape(str(item))}
    • " for item in step.get("runbook", [])] + repair_rows = [] + for row in step.get("repair_rows", []): + repair_rows.append( + "
  • " + f"#{html.escape(str(row.get('priority', '')))} · {html.escape(str(row.get('phase', '')))} · {html.escape(str(row.get('repair_type', '')))}" + f"{html.escape(str(row.get('target', '')))}" + f"{html.escape(str(row.get('owner', '')))}" + f"{html.escape(str(row.get('blocking_reason', '')))}" + f"{html.escape(str(row.get('next_action', '')))}" + f"{html.escape(str(row.get('verification_command', '')))}" + "
    • " + ) + phase_rows = [] + for row in step.get("phase_queue", []): + phase_rows.append( + "
  • " + f"#{html.escape(str(row.get('priority', '')))} · {html.escape(str(row.get('phase', '')))}" + f"{html.escape(str(row.get('label', '')))}" + f"{html.escape(str(row.get('blocked_count', 0)))} / {html.escape(str(row.get('row_count', 0)))} blocked" + f"owners: {html.escape(', '.join(str(item) for item in row.get('owners', [])) or 'n/a')}" + f"evidence: {html.escape(', '.join(str(item) for item in row.get('evidence_keys', [])) or 'n/a')}" + f"{html.escape(str(row.get('next_action', '')))}" + f"{html.escape(str(row.get('verification_command', '')))}" + "
    • " + ) + cards.append( + "
    " + f"
    {html.escape(str(step.get('status', 'pending')))} · {html.escape(str(step.get('category', '')))}" + f"

    {html.escape(str(step.get('label', step.get('key', 'evidence'))))}

    " + f"

    {html.escape(str(step.get('current', '')))}

    " + f"
    提交
    {html.escape(str(step.get('submission_path', '')))}
    " + f"
    模板
    {html.escape(str(step.get('template_path', '')))}
    " + f"
    阻断
    {html.escape(str(step.get('source_blocked_count', 0)))} blocked / {html.escape(str(step.get('source_pass_count', 0)))} pass
    " + f"
    修复
    {html.escape(str(step.get('repair_blocked_count', 0)))} repair rows; counts as completion: {html.escape(str(step.get('repair_counts_as_completion') is True).lower())}
    " + f"
    阶段
    {html.escape(str(step.get('phase_queue_blocked_count', 0)))} blocked phases; counts as completion: {html.escape(str(step.get('phase_queue_counts_as_completion') is True).lower())}
    " + f"
    下一步
    {html.escape(str(step.get('next_action', '')))}
    " + "
    阻断检查
    " + + ( + "
      " + "".join(blocked_checks) + "
    " + if blocked_checks + else "

    暂无阻断检查。

    " + ) + + "
    " + "
    阶段队列" + + ( + "
      " + "".join(phase_rows) + "
    " + if phase_rows + else "

    暂无阶段队列。

    " + ) + + "
    " + "
    修复清单" + + ( + "
      " + "".join(repair_rows) + "
    " + if repair_rows + else "

    暂无修复项。

    " + ) + + "
    " + "
    操作命令" + + ( + "
      " + "".join(command_rows) + "
    " + if command_rows + else "

    暂无操作命令。

    " + ) + + "
    " + "
    首要步骤" + + ( + "
      " + "".join(runbook_rows) + "
    " + if runbook_rows + else "

    暂无首要步骤。

    " + ) + + "
    " + "
    采集契约" + "
    " + "
    来源要求
    " + + render_small_list(step.get("provenance_requirements", []), "暂无来源要求。") + + "
    " + "
    通过条件
    " + + render_small_list(step.get("success_checks", []), "暂无通过条件。") + + "
    " + "
    证据资产
    " + + render_small_list(step.get("evidence_artifacts", []), "暂无证据资产。") + + "
    " + "
    资产角色
    " + + render_artifact_role_contract(step.get("artifact_role_contract", {})) + + "
    " + "
    隐私边界
    " + + render_small_list(step.get("privacy_contract", []), "暂无隐私边界。") + + "
    " + "
    " + "
    " + "
    " + ) + return ( + "
    " + "

    证据采集

    " + "

    以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

    " + "
    " + + "".join(cards) + + "
    " + ) diff --git a/scripts/review_studio_actions.py b/scripts/review_studio_actions.py new file mode 100644 index 00000000..e0fd91b2 --- /dev/null +++ b/scripts/review_studio_actions.py @@ -0,0 +1,374 @@ +import html +import os +from pathlib import Path +from typing import Any + +from review_studio_action_evidence import render_action_evidence_steps, world_class_action_steps +from review_studio_gates import status_label + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep Review Studio action guidance and source refs out of HTML rendering." + + +def link_from(output_html: Path, target: Path) -> str: + return os.path.relpath(target.resolve(), output_html.parent.resolve()) + + +def compact_excerpt(text: str, limit: int = 180) -> str: + normalized = " ".join(text.strip().split()) + if len(normalized) <= limit: + return normalized + return normalized[: limit - 1].rstrip() + "…" + + +def find_line_anchor(path: Path, patterns: list[str] | None = None) -> dict[str, Any]: + if not path.exists(): + return {"line": None, "matched_pattern": "", "excerpt": ""} + if not patterns: + return {"line": 1, "matched_pattern": "", "excerpt": ""} + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return {"line": 1, "matched_pattern": "", "excerpt": ""} + for pattern in patterns: + for index, line in enumerate(lines, start=1): + if pattern in line: + return { + "line": index, + "matched_pattern": pattern, + "excerpt": compact_excerpt(line), + } + first_line = compact_excerpt(lines[0]) if lines else "" + return {"line": 1, "matched_pattern": "", "excerpt": first_line} + + +def find_line(path: Path, patterns: list[str] | None = None) -> int | None: + return find_line_anchor(path, patterns).get("line") + + +def source_refs( + skill_dir: Path, + output_html: Path, + specs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + refs: list[dict[str, Any]] = [] + for spec in specs: + rel_path = str(spec.get("path", "")).strip() + if not rel_path: + continue + path = skill_dir / rel_path + exists = path.exists() + anchor = find_line_anchor(path, spec.get("patterns", [])) + refs.append( + { + "path": rel_path, + "label": str(spec.get("label", rel_path)), + "kind": str(spec.get("kind", "source")), + "line": anchor.get("line"), + "matched_pattern": anchor.get("matched_pattern", ""), + "excerpt": anchor.get("excerpt", ""), + "exists": exists, + "link": link_from(output_html, path) if exists else "", + } + ) + return refs + + +ACTION_GUIDANCE: dict[str, dict[str, Any]] = { + "intent-canvas": { + "summary": "收紧真实任务、输入、输出、排除项和成功标准。", + "why": "低 intent confidence 会让后续 Skill IR、输出评测和 Review Studio 结论建立在模糊意图上。", + "source_fix": "reports/intent-dialogue.md + reports/intent-confidence.md", + "source_paths": [ + {"path": "reports/intent-dialogue.md", "label": "intent dialogue", "kind": "report", "patterns": ["# Intent"]}, + {"path": "reports/intent-confidence.md", "label": "intent confidence", "kind": "report", "patterns": ["# Intent"]}, + ], + "verification": "python3 scripts/yao.py intent-confidence .", + }, + "trigger-lab": { + "summary": "修正 route scorecard 中的误触发、漏触发或 ambiguous case。", + "why": "触发错误会让正确 Skill 失活，或让相邻 Skill 被错误调用。", + "source_fix": "SKILL.md frontmatter description + evals/*/trigger_cases.json", + "source_paths": [ + {"path": "SKILL.md", "label": "frontmatter description", "kind": "source", "patterns": ["description:"]}, + {"path": "evals/trigger_cases.json", "label": "trigger eval cases", "kind": "eval", "patterns": ["should_trigger"]}, + {"path": "reports/route_scorecard.md", "label": "route scorecard", "kind": "report", "patterns": ["# Route"]}, + ], + "verification": "python3 scripts/build_confusion_matrix.py", + }, + "output-lab": { + "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。", + "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json", + "source_paths": [ + {"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ['"case_id"', '"id"']}, + {"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]}, + {"path": "reports/output_execution_runs.md", "label": "output execution runs", "kind": "report", "patterns": ["# Output Execution"]}, + {"path": "reports/output_blind_review_pack.md", "label": "blind A/B review pack", "kind": "report", "patterns": ["# Output Blind"]}, + {"path": "reports/output_review_kit.html", "label": "reviewer cockpit", "kind": "report", "patterns": ["Output Review Kit", "Variant A"]}, + {"path": "reports/output_review_adjudication.md", "label": "review adjudication", "kind": "report", "patterns": ["# Output Review"]}, + ], + "verification": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review", + }, + "context-budget": { + "summary": "压缩或拆分高成本 deferred resources，保留最小可路由上下文。", + "why": "初始加载可以安全，但后续 references、scripts、evals 体量过大时，reviewer 仍需要看到维护和读取成本。", + "source_fix": "SKILL.md + references/ + scripts/ + evals/", + "source_paths": [ + {"path": "SKILL.md", "label": "entrypoint", "kind": "source", "patterns": ["# Yao Meta Skill"]}, + {"path": "reports/context_budget.md", "label": "context budget", "kind": "report", "patterns": ["# Context"]}, + {"path": "scripts/resource_boundary_check.py", "label": "resource boundary checker", "kind": "source", "patterns": ["DEFERRED_RESOURCE"]}, + {"path": "references/skill-engineering-method.md", "label": "skill engineering method", "kind": "method", "patterns": ["Design Principle"]}, + ], + "verification": "python3 scripts/render_context_reports.py", + }, + "runtime-matrix": { + "summary": "修复目标端结构、metadata、相对路径、fallback 或 adapter target 声明。", + "why": "runtime conformance 失败意味着包可能被目标客户端错误加载或静默降级。", + "source_fix": "agents/interface.yaml + reports/conformance_matrix.md", + "source_paths": [ + {"path": "agents/interface.yaml", "label": "portable interface", "kind": "source", "patterns": ["adapter_targets"]}, + {"path": "reports/conformance_matrix.md", "label": "conformance matrix", "kind": "report", "patterns": ["# Runtime"]}, + ], + "verification": "python3 scripts/run_conformance_suite.py .", + }, + "trust-report": { + "summary": "处理脚本 help surface、依赖 pin、network policy、secret 和权限声明。", + "why": "团队分发时，脚本和依赖是主要供应链风险面，warning 必须有明确处置。", + "source_fix": "reports/security_trust_report.md + security/*.md + scripts/", + "source_paths": [ + {"path": "reports/security_trust_report.md", "label": "trust report", "kind": "report", "patterns": ["# Security"]}, + {"path": "security/script_policy.md", "label": "script policy", "kind": "policy", "patterns": ["# Script"]}, + {"path": "security/network_policy.md", "label": "network policy", "kind": "policy", "patterns": ["# Network"]}, + ], + "verification": "python3 scripts/trust_check.py .", + }, + "python-compat": { + "summary": "修复 Python 3.11 语法兼容问题，尤其是 f-string 表达式内的反斜杠转义。", + "why": "目标运行环境可能仍停留在 Python 3.11，语法漂移会让 CLI、报告生成和 CI 在发布后直接失败。", + "source_fix": "reports/python_compatibility.md + scripts/*.py + tests/*.py", + "source_paths": [ + {"path": "reports/python_compatibility.md", "label": "Python compatibility", "kind": "report", "patterns": ["# Python"]}, + {"path": "scripts/python_compat_check.py", "label": "compatibility checker", "kind": "source", "patterns": ["SCRIPT_INTERFACE"]}, + {"path": ".github/workflows/test.yml", "label": "CI test workflow", "kind": "ci", "patterns": ["python"]}, + ], + "verification": "python3 scripts/yao.py python-compat .", + }, + "architecture-maintainability": { + "summary": "处理大文件和 CLI command surface 的维护性热点，优先拆分稳定职责边界。", + "why": "Meta Skill 的门禁、报告和 CLI 会持续增长；如果不把架构债纳入审查，后续能力会越来越难验证和迁移。", + "source_fix": "reports/architecture_maintainability.md + scripts/yao.py + scripts/render_review_studio.py", + "source_paths": [ + {"path": "reports/architecture_maintainability.md", "label": "architecture maintainability", "kind": "report", "patterns": ["# Architecture"]}, + {"path": "scripts/yao.py", "label": "Yao CLI orchestrator", "kind": "source", "patterns": ["def command_"]}, + {"path": "scripts/render_review_studio.py", "label": "Review Studio renderer", "kind": "source", "patterns": ["def render_html"]}, + {"path": "scripts/review_studio_actions.py", "label": "Review Studio actions", "kind": "source", "patterns": ["ACTION_GUIDANCE"]}, + {"path": "scripts/render_review_viewer.py", "label": "review viewer renderer", "kind": "source", "patterns": ["def "]}, + ], + "verification": "python3 scripts/yao.py architecture-audit .", + }, + "permission-gates": { + "summary": "补齐高权限能力的 reviewer、scope、reason、expires_at 和目标端 enforcement 说明。", + "why": "权限契约只有在批准人、有效期和目标端处置方式明确时，才能支撑 governed release。", + "source_fix": "security/permission_policy.json + security/permission_policy.md", + "source_paths": [ + {"path": "security/permission_policy.json", "label": "permission approvals", "kind": "policy", "patterns": ["approved"]}, + {"path": "security/permission_policy.md", "label": "permission method", "kind": "policy", "patterns": ["# Permission"]}, + ], + "verification": "python3 scripts/trust_check.py .", + }, + "permission-runtime": { + "summary": "生成并修复目标包的 runtime permission probe 报告。", + "why": "目标端即使只能提供 metadata fallback，也必须明确 native enforcement 缺口、表示位置和 operator note。", + "source_fix": "dist/targets/*/adapter.json + reports/runtime_permission_probes.md", + "source_paths": [ + {"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]}, + {"path": "dist/targets/openai/adapter.json", "label": "OpenAI adapter", "kind": "package", "patterns": ["target_permission_contract"]}, + {"path": "dist/targets/claude/adapter.json", "label": "Claude adapter", "kind": "package", "patterns": ["target_permission_contract"]}, + {"path": "dist/targets/generic/adapter.json", "label": "generic adapter", "kind": "package", "patterns": ["target_permission_contract"]}, + ], + "verification": "python3 scripts/probe_runtime_permissions.py . --package-dir dist", + }, + "skill-atlas": { + "summary": "处理 portfolio 里的路由冲突、owner 缺口、stale skill 和重复能力。", + "why": "单个 Skill 质量很高仍可能在团队 skill library 中被相邻 Skill 冲突削弱。", + "source_fix": "reports/skill_atlas.html + skill_atlas/catalog.json", + "source_paths": [ + {"path": "skill_atlas/catalog.json", "label": "skill atlas catalog", "kind": "atlas", "patterns": ["summary"]}, + {"path": "skill_atlas/policy.json", "label": "atlas scope policy", "kind": "policy", "patterns": ["scope"]}, + {"path": "reports/skill_atlas.html", "label": "skill atlas report", "kind": "report", "patterns": ["Skill Atlas"]}, + ], + "verification": "python3 scripts/build_skill_atlas.py --workspace-root .", + }, + "operations-loop": { + "summary": "记录 metadata-only 使用事件，或明确当前 release 缺少真实使用信号。", + "why": "没有运营回路时，reviewer 无法判断采用率、误触发、坏输出和 review overdue 的真实影响。", + "source_fix": "reports/adoption_drift_report.md", + "source_paths": [ + {"path": "reports/adoption_drift_report.md", "label": "adoption drift report", "kind": "report", "patterns": ["# Adoption"]}, + {"path": "references/telemetry-drift-method.md", "label": "telemetry method", "kind": "method", "patterns": ["# Telemetry"]}, + ], + "verification": "python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted", + }, + "review-waivers": { + "summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "why": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "source_fix": "reports/review_waivers.md", + "source_paths": [ + {"path": "reports/review_waivers.md", "label": "waiver ledger", "kind": "report", "patterns": ["# Review"]}, + {"path": "references/review-waiver-method.md", "label": "waiver method", "kind": "method", "patterns": ["# Review"]}, + ], + "verification": "python3 scripts/render_review_waivers.py .", + }, + "world-class-evidence": { + "summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。", + "why": "世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。", + "source_fix": "reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md", + "source_paths": [ + {"path": "reports/world_class_evidence_ledger.md", "label": "world-class evidence ledger", "kind": "report", "patterns": ["# World-Class Evidence Ledger"]}, + {"path": "reports/world_class_evidence_plan.md", "label": "world-class evidence plan", "kind": "report", "patterns": ["# World-Class Evidence Plan"]}, + {"path": "reports/world_class_evidence_intake.md", "label": "world-class evidence intake", "kind": "report", "patterns": ["# World-Class Evidence Intake"]}, + {"path": "reports/world_class_evidence_preflight.md", "label": "world-class evidence preflight", "kind": "report", "patterns": ["# World-Class Evidence Preflight"]}, + {"path": "reports/world_class_evidence_preflight.html", "label": "world-class preflight HTML", "kind": "report", "patterns": ["World-Class Evidence Preflight"]}, + {"path": "reports/world_class_submission_review.md", "label": "world-class submission review", "kind": "report", "patterns": ["# World-Class Submission Review"]}, + {"path": "reports/world_class_claim_guard.md", "label": "world-class claim guard", "kind": "report", "patterns": ["# World-Class Claim Guard"]}, + {"path": "evidence/world_class/intake.schema.json", "label": "evidence intake schema", "kind": "schema", "patterns": ["Yao World-Class Evidence Intake"]}, + {"path": "evidence/world_class/templates/provider-holdout.intake.json", "label": "provider intake template", "kind": "template", "patterns": ["provider-holdout"]}, + {"path": "evidence/world_class/templates/human-adjudication.intake.json", "label": "human intake template", "kind": "template", "patterns": ["human-adjudication"]}, + {"path": "evidence/world_class/templates/native-permission-enforcement.intake.json", "label": "permission intake template", "kind": "template", "patterns": ["native-permission-enforcement"]}, + {"path": "evidence/world_class/templates/native-client-telemetry.intake.json", "label": "telemetry intake template", "kind": "template", "patterns": ["native-client-telemetry"]}, + {"path": "reports/skill_os2_audit.md", "label": "Skill OS 2.0 audit", "kind": "report", "patterns": ["# Skill OS"]}, + {"path": "reports/output_review_decisions.json", "label": "human review decisions", "kind": "report", "patterns": ["winner_variant"]}, + {"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]}, + {"path": "reports/adoption_drift_report.md", "label": "adoption drift", "kind": "report", "patterns": ["# Adoption"]}, + ], + "verification": ( + "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions " + "&& python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions " + "&& python3 scripts/yao.py review-studio ." + ), + }, + "registry-audit": { + "summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。", + "why": "分发元数据不完整时，团队无法安全安装、升级或追溯包体来源。", + "source_fix": "registry/ + reports/registry_audit.md", + "source_paths": [ + {"path": "registry/packages/yao-meta-skill.json", "label": "registry package", "kind": "registry", "patterns": ["version"]}, + {"path": "reports/registry_audit.md", "label": "registry audit", "kind": "report", "patterns": ["# Registry"]}, + {"path": "reports/install_simulation.md", "label": "install simulation", "kind": "report", "patterns": ["# Install"]}, + ], + "verification": "python3 scripts/registry_audit.py .", + }, + "release-notes": { + "summary": "确认 promotion、upgrade diff、breaking changes、migration guide 和 known limitations。", + "why": "发布说明不完整会让使用者无法判断升级风险和迁移动作。", + "source_fix": "reports/upgrade_check.md + docs/migration-v2.md", + "source_paths": [ + {"path": "reports/upgrade_check.md", "label": "upgrade check", "kind": "report", "patterns": ["# Upgrade"]}, + {"path": "docs/migration-v2.md", "label": "migration guide", "kind": "docs", "patterns": ["# Migration"]}, + {"path": "reports/promotion_decisions.md", "label": "promotion decisions", "kind": "report", "patterns": ["# Promotion"]}, + ], + "verification": "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json", + }, +} + + +def build_review_actions( + gates: list[dict[str, str]], + skill_dir: Path, + output_html: Path, + data: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + actions: list[dict[str, Any]] = [] + for gate_item in gates: + if gate_item["status"] == "pass": + continue + guidance = ACTION_GUIDANCE.get( + gate_item["key"], + { + "summary": "打开证据报告，修复当前 gate 暴露的问题。", + "why": "该 gate 未通过，release reviewer 需要明确处置动作。", + "source_fix": gate_item.get("evidence", ""), + "source_paths": [], + "verification": "python3 scripts/render_review_studio.py .", + }, + ) + refs = source_refs(skill_dir, output_html, guidance.get("source_paths", [])) + actions.append( + { + "action_id": f"review-action-{gate_item['key']}", + "gate_key": gate_item["key"], + "title": gate_item["label"], + "label": gate_item["label"], + "status": gate_item["status"], + "priority": "blocker" if gate_item["status"] == "block" else "warning", + "next_step": guidance["summary"], + "summary": guidance["summary"], + "reason": guidance["why"], + "why": guidance["why"], + "source_fix": guidance["source_fix"], + "source_refs": refs, + "evidence": gate_item.get("evidence", ""), + "evidence_link": gate_item.get("link", ""), + "verification_command": guidance["verification"], + "evidence_steps": world_class_action_steps(data or {}) + if gate_item["key"] == "world-class-evidence" + else [], + } + ) + return actions + + +def render_action_source_refs(refs: list[dict[str, Any]]) -> str: + if not refs: + return "

    暂无结构化 source refs；请先打开证据报告。

    " + items = [] + for ref in refs: + line_suffix = f":{ref['line']}" if ref.get("line") else "" + label = f"{ref['path']}{line_suffix}" + if ref.get("exists") and ref.get("link"): + path_html = f"{html.escape(label)}" + else: + path_html = f"{html.escape(label)} · missing" + pattern = str(ref.get("matched_pattern", "")).strip() + excerpt = str(ref.get("excerpt", "")).strip() + anchor_bits = [html.escape(str(ref.get("label", "source"))), html.escape(str(ref.get("kind", "source")))] + if pattern: + anchor_bits.append("pattern: " + html.escape(pattern)) + meta_html = " · ".join(anchor_bits) + excerpt_html = f"
    {html.escape(excerpt)}
    " if excerpt else "" + items.append( + "
  • " + f"{path_html}" + f"{meta_html}" + f"{excerpt_html}" + "
    • " + ) + return "
      " + "".join(items) + "
    " + + +def render_review_actions(actions: list[dict[str, Any]]) -> str: + if not actions: + return "

    当前没有 blocker 或 warning。保持现有证据链即可。

    " + cards = [] + for item in actions: + link_html = f"打开证据" if item.get("evidence_link") else "" + source_refs_html = render_action_source_refs(item.get("source_refs", [])) + evidence_steps_html = render_action_evidence_steps(item.get("evidence_steps", [])) + card_class = "action-card " + html.escape(item["status"]) + if item.get("evidence_steps"): + card_class += " with-evidence" + cards.append( + "
    " + f"
    {html.escape(status_label(item['status']))}

    {html.escape(item['label'])}

    " + f"

    {html.escape(item['summary'])}

    " + f"{html.escape(item['why'])}" + f"
    修复位置
    {html.escape(item['source_fix'])}
    " + f"
    验证命令
    {html.escape(item['verification_command'])}
    " + f"{evidence_steps_html}" + f"{source_refs_html}" + f"
    {html.escape(item['evidence'])} {link_html}
    " + "
    " + ) + return "".join(cards) diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py new file mode 100644 index 00000000..7aca3047 --- /dev/null +++ b/scripts/review_studio_data.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Data loading and insight card helpers for Review Studio.""" + +import json +from pathlib import Path +from typing import Any + +from skill_ir_paths import find_skill_ir_path as find_skill_ir_path_for_name + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to load Review Studio source reports and metric cards." + +ROOT = Path(__file__).resolve().parent.parent + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def latest_json_report(directory: Path) -> dict[str, Any]: + if not directory.exists(): + return {} + candidates = sorted(path for path in directory.glob("*.json") if path.is_file()) + if not candidates: + return {} + payload = load_json(candidates[-1]) + if payload: + try: + payload.setdefault("source_path", str(candidates[-1].resolve().relative_to(directory.parent.parent.parent.resolve()))) + except ValueError: + payload.setdefault("source_path", str(candidates[-1])) + return payload + + +def latest_report_path(skill_dir: Path, directory: Path, suffix: str) -> str: + if not directory.exists(): + return "" + candidates = sorted(path for path in directory.glob(f"*{suffix}") if path.is_file()) + if not candidates: + return "" + try: + return str(candidates[-1].resolve().relative_to(skill_dir.resolve())) + except ValueError: + return str(candidates[-1]) + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def parse_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + if not lines or lines[0].strip() != "---": + return {} + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {} + text = "\n".join(lines[1:end_index]) + if yaml is not None: + payload = yaml.safe_load(text) or {} + return payload if isinstance(payload, dict) else {} + data = {} + for line in text.splitlines(): + if ":" in line: + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data + + +def find_skill_ir_path(skill_dir: Path) -> str: + frontmatter = parse_frontmatter(skill_dir / "SKILL.md") + name = str(frontmatter.get("name") or skill_dir.name) + return find_skill_ir_path_for_name(skill_dir, name) + + +def evidence_paths(skill_dir: Path) -> dict[str, str]: + rels = { + "skill_overview": "reports/skill-overview.html", + "review_viewer": "reports/review-viewer.html", + "output_eval": "reports/output_quality_scorecard.md", + "output_execution": "reports/output_execution_runs.md", + "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_kit": "reports/output_review_kit.md", + "output_review_kit_html": "reports/output_review_kit.html", + "output_review_decisions": "reports/output_review_decisions.json", + "output_review_adjudication": "reports/output_review_adjudication.md", + "benchmark_reproducibility": "reports/benchmark_reproducibility.md", + "skill_os2_coverage": "reports/skill_os2_coverage.md", + "runtime_conformance": "reports/conformance_matrix.md", + "trust_report": "reports/security_trust_report.md", + "python_compatibility": "reports/python_compatibility.md", + "architecture_maintainability": "reports/architecture_maintainability.md", + "permission_policy": "security/permission_policy.md", + "runtime_permissions": "reports/runtime_permission_probes.md", + "skill_atlas": "reports/skill_atlas.html", + "compiled_targets": "reports/compiled_targets.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "adaptation_proposals": "reports/adaptation_proposals.md", + "adaptation_approval_ledger": "reports/adaptation_approval_ledger.json", + "adaptation_regression": "reports/adaptation_regression_report.md", + "world_class_evidence_plan": "reports/world_class_evidence_plan.md", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", + "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_evidence_preflight": "reports/world_class_evidence_preflight.md", + "world_class_evidence_preflight_html": "reports/world_class_evidence_preflight.html", + "world_class_submission_review": "reports/world_class_submission_review.md", + "world_class_operator_runbook": "reports/world_class_operator_runbook.md", + "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", + "world_class_claim_guard": "reports/world_class_claim_guard.md", + "registry_audit": "reports/registry_audit.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "upgrade_check": "reports/upgrade_check.md", + "migration": "docs/migration-v2.md", + } + skill_ir_path = find_skill_ir_path(skill_dir) + if skill_ir_path: + rels["skill_ir"] = skill_ir_path + daily_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "daily", ".md") + weekly_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "weekly", ".md") + if daily_md: + rels["daily_skillops"] = daily_md + if weekly_md: + rels["weekly_curator"] = weekly_md + return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()} + + +def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: + reports = skill_dir / "reports" + return { + "overview": load_json(reports / "skill-overview.json"), + "intent_confidence": load_json(reports / "intent-confidence.json"), + "intent_dialogue": load_json(reports / "intent-dialogue.json"), + "route_scorecard": load_json(reports / "route_scorecard.json"), + "output_quality": load_json(reports / "output_quality_scorecard.json"), + "output_execution": load_json(reports / "output_execution_runs.json"), + "output_blind_review": load_json(reports / "output_blind_review_pack.json"), + "output_review_kit": load_json(reports / "output_review_kit.json"), + "output_review_adjudication": load_json(reports / "output_review_adjudication.json"), + "benchmark_reproducibility": load_json(reports / "benchmark_reproducibility.json"), + "skill_os2_coverage": load_json(reports / "skill_os2_coverage.json"), + "compiled_targets": load_json(reports / "compiled_targets.json"), + "conformance": load_json(reports / "conformance_matrix.json"), + "runtime_permissions": load_json(reports / "runtime_permission_probes.json"), + "trust": load_json(reports / "security_trust_report.json"), + "python_compatibility": load_json(reports / "python_compatibility.json"), + "architecture_maintainability": load_json(reports / "architecture_maintainability.json"), + "context_budget": load_json(reports / "context_budget.json"), + "promotion": load_json(reports / "promotion_decisions.json"), + "atlas": load_json(reports / "skill_atlas.json"), + "adoption_drift": load_json(reports / "adoption_drift_report.json"), + "daily_skillops": latest_json_report(reports / "skillops" / "daily"), + "weekly_curator": latest_json_report(reports / "skillops" / "weekly"), + "review_waivers": load_json(reports / "review_waivers.json"), + "review_annotations": load_json(reports / "review_annotations.json"), + "adaptation_proposals": load_json(reports / "adaptation_proposals.json"), + "adaptation_approval_ledger": load_json(reports / "adaptation_approval_ledger.json"), + "adaptation_regression": load_json(reports / "adaptation_regression_report.json"), + "world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"), + "world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"), + "world_class_evidence_preflight": load_json(reports / "world_class_evidence_preflight.json"), + "world_class_submission_review": load_json(reports / "world_class_submission_review.json"), + "world_class_operator_runbook": load_json(reports / "world_class_operator_runbook.json"), + "world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"), + "registry": load_json(reports / "registry_audit.json"), + "package_verification": load_json(reports / "package_verification.json"), + "install_simulation": load_json(reports / "install_simulation.json"), + "upgrade_check": load_json(reports / "upgrade_check.json"), + "manifest": load_json(skill_dir / "manifest.json"), + "frontmatter": parse_frontmatter(skill_dir / "SKILL.md"), + "interface": load_yaml(skill_dir / "agents" / "interface.yaml"), + } + + +def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: + overview = data["overview"] + output = data["output_quality"].get("summary", {}) + output_execution = data["output_execution"].get("summary", {}) + output_blind = data["output_blind_review"].get("summary", {}) + output_review_kit = data["output_review_kit"].get("summary", {}) + output_review = data["output_review_adjudication"].get("summary", {}) + benchmark = data["benchmark_reproducibility"].get("summary", {}) + blueprint = data["skill_os2_coverage"].get("summary", {}) + extension_partial = int(blueprint.get("extension_partial_count", 0) or 0) + extension_planned = int(blueprint.get("extension_planned_count", 0) or 0) + compiled = data["compiled_targets"].get("summary", {}) + conformance = data["conformance"].get("summary", {}) + runtime_permissions = data["runtime_permissions"].get("summary", {}) + trust = data["trust"].get("summary", {}) + python_compat = data["python_compatibility"].get("summary", {}) + architecture = data["architecture_maintainability"].get("summary", {}) + atlas = data["atlas"].get("summary", {}) + adoption = data["adoption_drift"].get("summary", {}) + daily_skillops = data["daily_skillops"].get("summary", {}) + weekly_curator = data["weekly_curator"].get("summary", {}) + waivers = data["review_waivers"].get("summary", {}) + annotations = data["review_annotations"].get("summary", {}) + intake = data["world_class_evidence_intake"].get("summary", {}) + claim_guard = data["world_class_claim_guard"].get("summary", {}) + registry = data["registry"].get("package", {}) + package_verification = data["package_verification"].get("summary", {}) + install_simulation = data["install_simulation"].get("summary", {}) + upgrade = data["upgrade_check"].get("summary", {}) + cards = [ + { + "label": "Skill IR", + "value": str(overview.get("skill_ir", {}).get("schema_version", "missing")), + "detail": f"{overview.get('skill_ir', {}).get('target_count', 0)} targets in platform-neutral contract", + }, + { + "label": "Compiler", + "value": f"{compiled.get('pass_count', 0)}/{compiled.get('target_count', 0)}", + "detail": "target contracts compiled from Skill IR", + }, + { + "label": "Output Delta", + "value": str(output.get("delta", "n/a")), + "detail": f"{output.get('case_count', 0)} cases; {output.get('file_backed_case_count', 0)} file-backed", + }, + { + "label": "Exec Runs", + "value": str(output_execution.get("variant_run_count", 0)), + "detail": ( + f"command {output_execution.get('command_executed_count', 0)}; " + f"model {output_execution.get('model_executed_count', 0)}; " + f"recorded {output_execution.get('recorded_fixture_count', 0)}" + ), + }, + { + "label": "Blind A/B", + "value": str(output_blind.get("pair_count", 0)), + "detail": "review pairs hide baseline vs with-skill labels", + }, + { + "label": "Review Kit", + "value": f"{output_review_kit.get('ready_for_adjudication_count', 0)}/{output_review_kit.get('case_count', 0)}", + "detail": f"pending {output_review_kit.get('pending_decision_count', 0)}; answer key hidden", + }, + { + "label": "Review A/B", + "value": f"{output_review.get('judgment_count', 0)}/{output_review.get('pair_count', 0)}", + "detail": f"adjudication decisions; pending {output_review.get('pending_count', 0)}", + }, + { + "label": "Public Claim", + "value": "ready" if benchmark.get("public_claim_ready") is True else "blocked", + "detail": f"{benchmark.get('public_claim_blocker_count', 0)} blockers; local reproducible {str(benchmark.get('reproducibility_ready', False)).lower()}", + }, + { + "label": "Blueprint", + "value": f"{blueprint.get('pass_count', 0)}/{blueprint.get('item_count', 0)}", + "detail": ( + f"2.0 coverage; extensions partial {extension_partial}, " + f"planned {extension_planned}; evidence pending {blueprint.get('world_class_evidence_pending_count', 0)}" + ), + }, + { + "label": "Runtime", + "value": f"{conformance.get('pass_count', 0)}/{conformance.get('target_count', 0)}", + "detail": "target conformance pass rate", + }, + { + "label": "Perm Probe", + "value": f"{runtime_permissions.get('metadata_fallback_count', 0)}/{runtime_permissions.get('target_count', 0)}", + "detail": ( + f"{runtime_permissions.get('native_enforcement_count', 0)} native; " + f"{runtime_permissions.get('installer_enforcement_pass_count', 0)} installer-enforced" + ), + }, + { + "label": "Trust", + "value": str(trust.get("secret_findings", 0)), + "detail": f"{trust.get('script_count', 0)} scripts scanned; secrets found", + }, + { + "label": "Py Compat", + "value": str(python_compat.get("issue_count", 0)), + "detail": f"{python_compat.get('file_count', 0)} files scanned for Python {python_compat.get('target_python', '3.11')}", + }, + { + "label": "Arch Debt", + "value": str(architecture.get("hotspot_count", 0)), + "detail": ( + f"{architecture.get('largest_file_lines', 0)} largest lines; " + f"{architecture.get('watchlist_count', 0)} watchlist; " + f"{architecture.get('early_watchlist_count', 0)} early; " + f"{architecture.get('command_handler_count', 0)} CLI handlers; " + f"{architecture.get('entrypoint_command_handler_count', 0)} in entrypoint" + ), + }, + { + "label": "Atlas", + "value": str(atlas.get("route_collision_count", 0)), + "detail": f"{atlas.get('skill_count', 0)} scanned skills; route collisions", + }, + { + "label": "Drift", + "value": str(adoption.get("risk_band", "n/a")), + "detail": f"{adoption.get('event_count', 0)} metadata events; {adoption.get('missed_trigger_count', 0)} missed triggers", + }, + { + "label": "Daily Ops", + "value": str(daily_skillops.get("proposal_count", 0)), + "detail": ( + f"{daily_skillops.get('decision', 'missing')}; " + f"approval {daily_skillops.get('approval_count', 0)}; " + f"release lock {str(daily_skillops.get('release_lock_ready', False)).lower()}" + ), + }, + { + "label": "Weekly Queue", + "value": str(weekly_curator.get("unique_opportunity_count", 0)), + "detail": ( + f"{weekly_curator.get('decision', 'missing')}; " + f"ready {weekly_curator.get('ready_for_approval_review_count', 0)}; " + f"top score {weekly_curator.get('top_score', 0)}" + ), + }, + { + "label": "Waivers", + "value": str(waivers.get("active_count", 0)), + "detail": f"{waivers.get('covered_gate_count', 0)} gates covered; human risk decisions", + }, + { + "label": "Intake", + "value": f"{intake.get('template_pass_count', 0)}/{intake.get('template_count', 0)}", + "detail": ( + f"{intake.get('valid_submission_count', 0)} valid submissions; " + f"{intake.get('invalid_submission_count', 0)} invalid" + ), + }, + { + "label": "Claim Guard", + "value": str(claim_guard.get("violation_count", 0)), + "detail": f"{claim_guard.get('claim_surface_count', 0)} public surfaces scanned", + }, + { + "label": "Notes", + "value": f"{annotations.get('open_count', 0)}/{annotations.get('annotation_count', 0)}", + "detail": f"{annotations.get('open_blocker_count', 0)} open blocker annotations", + }, + { + "label": "Registry", + "value": str(registry.get("version", "n/a")), + "detail": f"{len(registry.get('targets', []))} targets; {registry.get('license', 'no license')} license", + }, + { + "label": "Archive", + "value": "pass" if data["package_verification"].get("ok") else "n/a", + "detail": f"{package_verification.get('archive_entry_count', 0)} zip entries; package verification", + }, + { + "label": "Install", + "value": "pass" if data["install_simulation"].get("ok") else "n/a", + "detail": ( + f"{install_simulation.get('adapter_count', 0)} adapters; " + f"{install_simulation.get('installer_permission_enforced_count', 0)} permissions enforced; " + f"{install_simulation.get('installer_permission_failure_count', 0)} permission failures" + ), + }, + { + "label": "Upgrade", + "value": str(upgrade.get("recommended_bump", "n/a")), + "detail": f"declared {upgrade.get('declared_bump', 'n/a')}; {upgrade.get('breaking_change_count', 0)} breaking changes", + }, + ] + return cards diff --git a/scripts/review_studio_formatting.py b/scripts/review_studio_formatting.py index f7195fea..a6d31349 100644 --- a/scripts/review_studio_formatting.py +++ b/scripts/review_studio_formatting.py @@ -19,22 +19,48 @@ LABELS = { "archive_entry_count": "Zip 条目", "archive_present": "归档存在", "archive_sha256": "归档哈希", + "actual_gate_count": "实际 Gate", + "answer_revealed_count": "答案揭示", "baseline_pass_rate": "Baseline", "breaking_change_count": "破坏变更", "case_count": "案例数", + "claim_surface_count": "声明面", "command_executed_count": "命令执行", "compatibility_pass_count": "兼容通过", "covered_gate_count": "覆盖 Gate", + "daily_report_count": "日报数", "declared_bump": "声明版本", + "decision": "决策", "delta": "增益", + "duplicate_gate_keys": "重复 Gate", "event_count": "事件数", + "expected_gate_count": "期望 Gate", "failure_count": "失败数", + "file_count": "文件数", + "fstring_311_violation_count": "F-string 3.11", "gate_pass": "Gate", "help_smoke_failed_count": "Help 失败", + "human_review_complete": "人审完成", "install_simulated": "安装模拟", + "installer_enforcement_pass_count": "安装拦截", + "installer_enforcement_ready": "安装拦截就绪", + "installer_enforcement_source_status": "安装证据", + "installer_enforcement_target_count": "安装拦截目标", + "installer_permission_capability_count": "安装权限能力", + "installer_permission_enforced_count": "安装权限", + "installer_permission_failure_count": "安装权限失败", + "invalid_submission_count": "无效提交", + "item_count": "项目数", + "issue_count": "问题数", + "ledger_pending_count": "台账待补", + "ledger_ready_to_claim_world_class": "台账可声明", "license": "License", + "local_blueprint_ready": "本地蓝图", "metadata_fallback_count": "Metadata fallback", "missed_trigger_count": "漏触发", + "missing_count": "缺失数", + "missing_gate_keys": "缺失 Gate", + "module_count": "模块数", "model_executed_count": "模型执行", "name": "名称", "native_enforcement_count": "原生执行", @@ -46,24 +72,53 @@ LABELS = { "package_sha256": "包体哈希", "pass_count": "通过数", "pending_count": "待审", + "pending_answer_hidden_count": "答案隐藏", + "permission_capability_count": "权限能力", + "permission_target_count": "权限目标", + "provider_evidence_complete": "Provider 证据", + "public_claim_blocker_count": "声明阻断", + "public_claim_ready": "可公开声明", + "public_world_class_ready": "世界级", + "proposal_count": "提案数", + "proposal_review_count": "提案复核", + "reproducibility_ready": "本地复现", "recorded_fixture_count": "记录样本", + "ready_for_approval_review_count": "待批准复核", + "ready_for_external_collection": "可收证据", + "ready_for_ledger_review": "可审台账", + "recommended_pr_count": "建议 PR", "recommended_bump": "建议版本", + "release_lock_ready": "发布锁", + "rendered_gate_keys": "渲染 Gate", "residual_risk_count": "残余风险", "risk_band": "风险带", "route_collision_count": "路由冲突", "script_count": "脚本数", + "schema_present": "Schema", "secret_findings": "Secret", "skill_count": "Skill 数", + "submission_count": "提交数", + "syntax_error_count": "语法错误", "target_count": "目标数", + "target_python": "目标 Python", "targets": "目标平台", + "template_count": "模板数", + "template_pass_count": "模板通过", "timing_observed_count": "计时样本", "token_estimated_count": "估算 Token", "token_observed_count": "真实 Token", "trust_level": "信任级别", + "unknown_gate_keys": "未知 Gate", + "unweighted_gate_keys": "未加权 Gate", + "unique_opportunity_count": "唯一机会", "variant_run_count": "运行数", + "valid_submission_count": "有效提交", "version": "版本", + "violation_count": "违规数", "warning_count": "警告数", "with_skill_pass_rate": "With Skill", + "world_class_evidence_pending_count": "待补证据", + "world_class_ready": "世界级就绪", } @@ -114,6 +169,22 @@ def render_kv_grid( return "
    " + "".join(rows) + "
    " +def render_gate_contract_panel(contract: dict[str, Any]) -> str: + return render_kv_grid( + contract, + [ + "ok", + "expected_gate_count", + "actual_gate_count", + "missing_gate_keys", + "unknown_gate_keys", + "duplicate_gate_keys", + "unweighted_gate_keys", + ], + "gate contract missing", + ) + + def registry_package_summary(package: dict[str, Any]) -> dict[str, Any]: if not package: return {} diff --git a/scripts/review_studio_gate_contract.py b/scripts/review_studio_gate_contract.py new file mode 100644 index 00000000..d1ee04be --- /dev/null +++ b/scripts/review_studio_gate_contract.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Review Studio gate keys, scoring, and contract checks.""" + +from collections import Counter +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by review_studio_gates.py for stable gate scoring and contract checks." + + +GATE_WEIGHTS = { + "trigger-lab": 15, + "output-lab": 20, + "context-budget": 10, + "runtime-matrix": 10, + "trust-report": 10, + "python-compat": 10, + "architecture-maintainability": 10, + "permission-gates": 10, + "permission-runtime": 10, + "skill-atlas": 10, + "operations-loop": 10, + "review-waivers": 10, + "world-class-evidence": 10, + "registry-audit": 10, + "release-notes": 10, + "intent-canvas": 10, +} +REVIEW_STUDIO_GATE_KEYS = frozenset(GATE_WEIGHTS) + + +def status_label(status: str) -> str: + return {"pass": "通过", "warn": "关注", "block": "阻断"}.get(status, status) + + +def add_blockers_from_gate(gates: list[dict[str, str]]) -> tuple[list[dict[str, str]], list[dict[str, str]]]: + blockers = [item for item in gates if item["status"] == "block"] + warnings = [item for item in gates if item["status"] == "warn"] + return blockers, warnings + + +def gate_contract(gates: list[dict[str, str]]) -> dict[str, Any]: + rendered_gate_keys = [str(item.get("key", "")) for item in gates] + rendered_gate_set = set(rendered_gate_keys) + expected_gate_set = set(REVIEW_STUDIO_GATE_KEYS) + duplicate_gate_keys = sorted(key for key, count in Counter(rendered_gate_keys).items() if count > 1) + missing_gate_keys = sorted(expected_gate_set - rendered_gate_set) + unknown_gate_keys = sorted(rendered_gate_set - expected_gate_set) + unweighted_gate_keys = sorted(rendered_gate_set - set(GATE_WEIGHTS)) + return { + "ok": not (missing_gate_keys or unknown_gate_keys or duplicate_gate_keys or unweighted_gate_keys), + "expected_gate_count": len(expected_gate_set), + "actual_gate_count": len(rendered_gate_keys), + "expected_gate_keys": sorted(expected_gate_set), + "rendered_gate_keys": rendered_gate_keys, + "missing_gate_keys": missing_gate_keys, + "unknown_gate_keys": unknown_gate_keys, + "duplicate_gate_keys": duplicate_gate_keys, + "unweighted_gate_keys": unweighted_gate_keys, + } + + +def min_output_cases(maturity: str) -> int: + if maturity in {"library", "governed"}: + return 5 + if maturity == "production": + return 3 + return 1 + + +def weighted_score(gates: list[dict[str, str]]) -> int: + earned = 0.0 + total = 0.0 + for item in gates: + weight = GATE_WEIGHTS.get(item["key"], 5) + total += weight + if item["status"] == "pass": + earned += weight + elif item["status"] == "warn": + earned += weight * 0.6 + return int(round(earned / total * 100)) if total else 0 diff --git a/scripts/review_studio_gate_helpers.py b/scripts/review_studio_gate_helpers.py new file mode 100644 index 00000000..6ae78425 --- /dev/null +++ b/scripts/review_studio_gate_helpers.py @@ -0,0 +1,140 @@ +"""Helper functions for Review Studio gate evaluation.""" + +import json +import os +from pathlib import Path +from typing import Any + +try: + from trust_check import permission_governance_status as compute_permission_governance_status + from trust_check import script_inventory as trust_script_inventory +except ImportError: # pragma: no cover + compute_permission_governance_status = None + trust_script_inventory = None + +from review_studio_gate_contract import min_output_cases + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by review_studio_gates.py to keep reusable Review Studio gate helpers out of the main gate sequence." + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def link_from(output_html: Path, target: Path) -> str: + return os.path.relpath(target.resolve(), output_html.parent.resolve()) + + +def report_link(output_html: Path, skill_dir: Path, rel_path: str) -> str: + return link_from(output_html, skill_dir / rel_path) + + +def gate(key: str, label: str, status: str, detail: str, evidence: str, link: str = "") -> dict[str, str]: + return { + "key": key, + "label": label, + "status": status, + "detail": detail, + "evidence": evidence, + "link": link, + } + + +def target_maturity(skill_dir: Path, overview: dict[str, Any]) -> str: + manifest = load_json(skill_dir / "manifest.json") + if manifest.get("maturity_tier"): + return str(manifest["maturity_tier"]) + metadata = overview.get("metadata", {}) if isinstance(overview, dict) else {} + if metadata.get("maturity_tier"): + return str(metadata["maturity_tier"]) + return "scaffold" + + +def fallback_permission_governance(skill_dir: Path) -> dict[str, Any]: + if compute_permission_governance_status is None or trust_script_inventory is None: + return {} + try: + scripts = trust_script_inventory(skill_dir) + return compute_permission_governance_status(skill_dir, scripts) + except Exception: + return {} + + +def build_output_lab_gate( + skill_dir: Path, + output_html: Path, + maturity: str, + output: dict[str, Any], + output_execution: dict[str, Any], + output_blind: dict[str, Any], + output_review: dict[str, Any], +) -> dict[str, str]: + output_summary = output.get("summary", {}) + output_execution_summary = output_execution.get("summary", {}) + output_blind_summary = output_blind.get("summary", {}) + output_review_summary = output_review.get("summary", {}) + required_cases = min_output_cases(maturity) + case_count = int(output_summary.get("case_count", 0) or 0) + file_backed = int(output_summary.get("file_backed_case_count", 0) or 0) + near_neighbor = int(output_summary.get("near_neighbor_case_count", 0) or 0) + boundary = int(output_summary.get("boundary_case_count", 0) or 0) + blind_pair_count = int(output_blind_summary.get("pair_count", 0) or 0) + execution_variant_count = int(output_execution_summary.get("variant_run_count", 0) or 0) + execution_command_count = int(output_execution_summary.get("command_executed_count", 0) or 0) + execution_model_count = int(output_execution_summary.get("model_executed_count", 0) or 0) + execution_recorded_count = int(output_execution_summary.get("recorded_fixture_count", 0) or 0) + review_pair_count = int(output_review_summary.get("pair_count", 0) or 0) + review_judgment_count = int(output_review_summary.get("judgment_count", 0) or 0) + review_pending_count = int(output_review_summary.get("pending_count", 0) or 0) + review_invalid_count = int(output_review_summary.get("invalid_decision_count", 0) or 0) + production_like = maturity in {"production", "library", "governed"} + blind_missing = production_like and (not output_blind or blind_pair_count < case_count) + review_missing = production_like and case_count > 0 and not output_review + review_pending = production_like and bool(output_review) and review_pending_count > 0 + execution_failed = bool(output_execution) and ( + not output_execution.get("ok", True) or int(output_execution_summary.get("failure_count", 0) or 0) > 0 + ) + review_invalid = bool(output_review) and (not output_review.get("ok", True) or review_invalid_count > 0) + output_blocked = ( + not output.get("ok", False) + or not output_summary.get("gate_pass", False) + or case_count < required_cases + or execution_failed + or review_invalid + ) + output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing or review_missing or review_pending + if not output: + output_status = "warn" + output_detail = "output eval scorecard is missing; generate it before production review" + else: + output_status = "block" if output_blocked else ("warn" if output_warn else "pass") + output_detail = ( + f"{case_count}/{required_cases} cases; with-skill {output_summary.get('with_skill_pass_rate', 0)}; " + f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}; " + f"blind A/B {blind_pair_count}" + + ( + f"; exec {execution_variant_count}; command {execution_command_count}; " + f"model {execution_model_count}; recorded {execution_recorded_count}" + if output_execution + else "" + ) + + (f"; reviewed {review_judgment_count}/{review_pair_count}" if output_review else "") + + (f"; review pending {review_pending_count}" if review_pending else "") + + ("; review adjudication missing" if review_missing else "") + ) + return gate( + "output-lab", + "输出实验", + output_status, + output_detail, + "reports/output_quality_scorecard.json", + report_link(output_html, skill_dir, "reports/output_quality_scorecard.md"), + ) diff --git a/scripts/review_studio_gates.py b/scripts/review_studio_gates.py new file mode 100644 index 00000000..a253d6c5 --- /dev/null +++ b/scripts/review_studio_gates.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 +"""Gate evaluation contract for Review Studio 2.0.""" + +from pathlib import Path +from typing import Any + +from review_studio_gate_contract import ( + GATE_WEIGHTS, + REVIEW_STUDIO_GATE_KEYS, + add_blockers_from_gate, + gate_contract, + min_output_cases, + status_label, + weighted_score, +) +from review_studio_gate_helpers import ( + build_output_lab_gate, + fallback_permission_governance, + gate, + report_link, + target_maturity, +) + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep Review Studio gate evaluation separate from HTML rendering." + + +ROOT = Path(__file__).resolve().parent.parent + + +def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: + overview = data["overview"] + maturity = target_maturity(skill_dir, overview) + gates: list[dict[str, str]] = [] + + intent = data["intent_confidence"] + intent_score = int(intent.get("score", 0) or 0) + intent_status = "pass" if intent.get("gate_passed") or intent_score >= 75 else "warn" + gates.append( + gate( + "intent-canvas", + "意图画布", + intent_status, + f"intent confidence {intent_score}/100; {intent.get('recommended_action', 'review current intent frame')}", + "reports/intent-confidence.json", + report_link(output_html, skill_dir, "reports/intent-confidence.md"), + ) + ) + + route = data["route_scorecard"] + route_summary = route.get("summary", {}) + misroutes = int(route_summary.get("misroute_count", len(route.get("misroutes", []))) or 0) + ambiguous = int(route_summary.get("ambiguous_case_count", len(route.get("ambiguous_cases", []))) or 0) + if not route: + route_status = "warn" + route_detail = "route scorecard is missing; run route-scorecard before release review" + else: + route_status = "block" if misroutes else ("warn" if ambiguous else "pass") + route_detail = f"{route_summary.get('total_cases', 0)} trigger cases; {misroutes} misroutes; {ambiguous} ambiguous" + gates.append( + gate( + "trigger-lab", + "触发实验", + route_status, + route_detail, + "reports/route_scorecard.json", + report_link(output_html, skill_dir, "reports/route_scorecard.md"), + ) + ) + + gates.append( + build_output_lab_gate( + skill_dir, + output_html, + maturity, + data["output_quality"], + data["output_execution"], + data["output_blind_review"], + data["output_review_adjudication"], + ) + ) + + context = data["context_budget"] + context_stats = context.get("stats", {}) + context_status = "pass" if context.get("ok") else "block" + if context.get("warnings"): + context_status = "warn" if context_status == "pass" else context_status + if not context: + context_status = "warn" + large_deferred_dirs = context_stats.get("large_deferred_resource_dirs", []) or [] + top_deferred = "none" + if large_deferred_dirs: + first = large_deferred_dirs[0] + top_deferred = f"{first.get('path', 'resource')} {first.get('estimated_tokens', 'n/a')}" + deferred_governance = context_stats.get("deferred_resource_governance", {}) if isinstance(context_stats, dict) else {} + governance_status = deferred_governance.get("status", "unknown") if isinstance(deferred_governance, dict) else "unknown" + context_detail = ( + f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/" + f"{context_stats.get('context_budget_limit', 'n/a')}; " + f"deferred {context_stats.get('deferred_resource_tokens', 'n/a')}/" + f"{context_stats.get('deferred_resource_warn_threshold', 'n/a')}; " + f"top deferred {top_deferred}; " + f"resource governance {governance_status}; " + f"quality density {context_stats.get('quality_density', 'n/a')}" + ) + gates.append( + gate( + "context-budget", + "上下文", + context_status, + context_detail, + "reports/context_budget.json", + report_link(output_html, skill_dir, "reports/context_budget.md"), + ) + ) + + conformance = data["conformance"] + conformance_summary = conformance.get("summary", {}) + fail_count = int(conformance_summary.get("fail_count", 0) or 0) + if not conformance: + conformance_status = "warn" + conformance_detail = "runtime conformance matrix is missing" + else: + conformance_status = "block" if fail_count else "pass" + conformance_detail = f"{conformance_summary.get('pass_count', 0)} / {conformance_summary.get('target_count', 0)} targets pass" + gates.append( + gate( + "runtime-matrix", + "运行矩阵", + conformance_status, + conformance_detail, + "reports/conformance_matrix.json", + report_link(output_html, skill_dir, "reports/conformance_matrix.md"), + ) + ) + + trust = data["trust"] + trust_summary = trust.get("summary", {}) + if not trust: + trust_status = "warn" + trust_detail = "security trust report is missing" + else: + trust_status = "block" if trust.get("failures") else ("warn" if trust.get("warnings") else "pass") + trust_detail = ( + f"{trust_summary.get('secret_findings', 0)} secrets; " + f"{trust_summary.get('script_count', 0)} scripts; " + f"{trust_summary.get('network_script_count', 0)} network-capable scripts; " + f"{trust_summary.get('help_smoke_failed_count', 0)} help smoke failures" + ) + gates.append( + gate( + "trust-report", + "信任报告", + trust_status, + trust_detail, + "reports/security_trust_report.json", + report_link(output_html, skill_dir, "reports/security_trust_report.md"), + ) + ) + + python_compat = data["python_compatibility"] + python_compat_summary = python_compat.get("summary", {}) + if not python_compat: + python_compat_status = "warn" + python_compat_detail = "Python compatibility report is missing" + else: + issue_count = int(python_compat_summary.get("issue_count", 0) or 0) + syntax_error_count = int(python_compat_summary.get("syntax_error_count", 0) or 0) + fstring_violation_count = int(python_compat_summary.get("fstring_311_violation_count", 0) or 0) + python_compat_status = ( + "block" + if not python_compat.get("ok", True) or issue_count or syntax_error_count or fstring_violation_count + else "pass" + ) + python_compat_detail = ( + f"Python {python_compat_summary.get('target_python', '3.11')}; " + f"{python_compat_summary.get('file_count', 0)} files; " + f"{issue_count} compatibility issues; " + f"{syntax_error_count} syntax; " + f"{fstring_violation_count} f-string 3.11 hazards" + ) + gates.append( + gate( + "python-compat", + "Python 兼容", + python_compat_status, + python_compat_detail, + "reports/python_compatibility.json", + report_link(output_html, skill_dir, "reports/python_compatibility.md"), + ) + ) + + architecture = data["architecture_maintainability"] + architecture_summary = architecture.get("summary", {}) + if not architecture: + if maturity in {"library", "governed"}: + architecture_status = "warn" + architecture_detail = "architecture maintainability report is missing for a reusable package" + else: + architecture_status = "pass" + architecture_detail = "architecture maintainability report is optional until code surface grows" + else: + hotspot_count = int(architecture_summary.get("hotspot_count", 0) or 0) + watchlist_count = int(architecture_summary.get("watchlist_count", 0) or 0) + early_watchlist_count = int(architecture_summary.get("early_watchlist_count", 0) or 0) + blocker_count = int(architecture_summary.get("blocker_count", 0) or 0) + architecture_status = "block" if not architecture.get("ok", True) or blocker_count else ("warn" if hotspot_count else "pass") + hotspot_label = "hotspot" if hotspot_count == 1 else "hotspots" + watchlist_label = "watchlist file" if watchlist_count == 1 else "watchlist files" + early_label = "early watch file" if early_watchlist_count == 1 else "early watch files" + blocker_label = "blocker" if blocker_count == 1 else "blockers" + architecture_detail = ( + f"{architecture_summary.get('python_file_count', 0)} Python files; " + f"{hotspot_count} {hotspot_label}; " + f"{watchlist_count} {watchlist_label}; " + f"{early_watchlist_count} {early_label}; " + f"{blocker_count} {blocker_label}; " + f"largest {architecture_summary.get('largest_file_lines', 0)} lines; " + f"{architecture_summary.get('command_handler_count', 0)} CLI handlers; " + f"{architecture_summary.get('entrypoint_command_handler_count', 0)} in entrypoint" + ) + gates.append( + gate( + "architecture-maintainability", + "架构维护", + architecture_status, + architecture_detail, + "reports/architecture_maintainability.json", + report_link(output_html, skill_dir, "reports/architecture_maintainability.md"), + ) + ) + + permission_governance = trust.get("permission_governance", {}) if isinstance(trust.get("permission_governance", {}), dict) else {} + if trust and not permission_governance: + permission_governance = fallback_permission_governance(skill_dir) + if not trust: + permission_status = "warn" + permission_detail = "permission governance evidence is missing because trust report is missing" + elif not permission_governance: + permission_status = "warn" + permission_detail = "permission governance evidence is missing from trust report" + else: + required = int(permission_governance.get("required_count", 0) or 0) + approved = int(permission_governance.get("approval_count", 0) or 0) + gaps = ( + int(permission_governance.get("missing_count", 0) or 0) + + int(permission_governance.get("invalid_count", 0) or 0) + + int(permission_governance.get("expired_count", 0) or 0) + ) + if gaps: + permission_status = "block" if maturity == "governed" else "warn" + else: + permission_status = "pass" + required_names = ", ".join(permission_governance.get("required_capabilities", []) or []) or "none" + permission_detail = f"{approved}/{required} permissions approved; gaps {gaps}; required {required_names}" + gates.append( + gate( + "permission-gates", + "权限批准", + permission_status, + permission_detail, + "reports/security_trust_report.json + security/permission_policy.json", + report_link(output_html, skill_dir, "security/permission_policy.md"), + ) + ) + + runtime_permissions = data["runtime_permissions"] + runtime_permissions_summary = runtime_permissions.get("summary", {}) + if not runtime_permissions: + runtime_permission_status = "block" if maturity == "governed" else "warn" + runtime_permission_detail = "runtime permission probe report is missing" + elif runtime_permissions.get("failures"): + runtime_permission_status = "block" + runtime_permission_detail = f"{runtime_permissions_summary.get('failure_count', len(runtime_permissions.get('failures', [])))} runtime permission probe failures" + else: + runtime_permission_status = "pass" + runtime_permission_detail = ( + f"{runtime_permissions_summary.get('pass_count', 0)}/{runtime_permissions_summary.get('target_count', 0)} targets probed; " + f"native {runtime_permissions_summary.get('native_enforcement_count', 0)}; " + f"metadata fallback {runtime_permissions_summary.get('metadata_fallback_count', 0)}; " + f"installer {runtime_permissions_summary.get('installer_enforcement_pass_count', 0)}; " + f"residual risks {runtime_permissions_summary.get('residual_risk_count', 0)}" + ) + gates.append( + gate( + "permission-runtime", + "权限探针", + runtime_permission_status, + runtime_permission_detail, + "reports/runtime_permission_probes.json", + report_link(output_html, skill_dir, "reports/runtime_permission_probes.md"), + ) + ) + + atlas = data["atlas"] + atlas_summary = atlas.get("summary", {}) + actionable_route_collisions = int( + atlas_summary.get("actionable_route_collision_count", atlas_summary.get("route_collision_count", 0)) or 0 + ) + actionable_owner_gaps = int(atlas_summary.get("actionable_owner_gap_count", atlas_summary.get("owner_gap_count", 0)) or 0) + actionable_stale = int(atlas_summary.get("actionable_stale_count", atlas_summary.get("stale_count", 0)) or 0) + actionable_drift = int(atlas_summary.get("actionable_drift_signal_count", atlas_summary.get("drift_signal_count", 0)) or 0) + atlas_issues = actionable_route_collisions + actionable_owner_gaps + actionable_stale + actionable_drift + if not atlas: + atlas_status = "warn" + atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown" + else: + atlas_status = "warn" if atlas_issues else "pass" + atlas_detail = ( + f"{atlas_summary.get('skill_count', 0)} skills, " + f"{atlas_summary.get('actionable_skill_count', atlas_summary.get('skill_count', 0))} actionable; " + f"{actionable_route_collisions} actionable route collisions; " + f"{actionable_owner_gaps} actionable owner gaps; " + f"{actionable_stale} actionable stale; " + f"{actionable_drift} actionable drift; " + f"{atlas_summary.get('non_actionable_issue_count', 0)} scoped non-actionable issues" + ) + gates.append( + gate( + "skill-atlas", + "组合治理", + atlas_status, + atlas_detail, + "reports/skill_atlas.json", + report_link(output_html, skill_dir, "reports/skill_atlas.html"), + ) + ) + + adoption = data["adoption_drift"] + adoption_summary = adoption.get("summary", {}) + daily_skillops = data.get("daily_skillops", {}) + weekly_curator = data.get("weekly_curator", {}) + daily_summary = daily_skillops.get("summary", {}) if isinstance(daily_skillops, dict) else {} + weekly_summary = weekly_curator.get("summary", {}) if isinstance(weekly_curator, dict) else {} + if not adoption: + adoption_status = "warn" + adoption_detail = "adoption drift report is missing; real usage impact is unknown" + elif adoption.get("failures"): + adoption_status = "block" + adoption_detail = f"telemetry privacy or schema failures: {len(adoption.get('failures', []))}" + else: + risk_band = adoption_summary.get("risk_band", "no-data") + adoption_status = "warn" if risk_band in {"no-data", "medium", "high"} else "pass" + adoption_detail = ( + f"{adoption_summary.get('event_count', 0)} metadata events; " + f"adoption {adoption_summary.get('adoption_rate', 0)}; " + f"missed {adoption_summary.get('missed_trigger_count', 0)}; " + f"bad-output {adoption_summary.get('bad_output_count', 0)}; " + f"risk {risk_band}" + ) + skillops_parts = [] + skillops_blocked = False + for label, report, summary in [ + ("daily", daily_skillops, daily_summary), + ("weekly", weekly_curator, weekly_summary), + ]: + if not report: + continue + writes_source = bool(summary.get("writes_source_files") or report.get("writes_source_files")) + auto_patch = bool(summary.get("auto_patch_enabled") or report.get("auto_patch_enabled")) + failure_count = int(summary.get("failure_count", 0) or 0) + if writes_source or auto_patch or failure_count: + skillops_blocked = True + if label == "daily": + skillops_parts.append( + f"daily proposals {summary.get('proposal_count', 0)}; " + f"daily decision {summary.get('decision', report.get('decision', 'unknown'))}; " + f"daily release lock {str(summary.get('release_lock_ready', False)).lower()}" + ) + else: + skillops_parts.append( + f"weekly queue {summary.get('unique_opportunity_count', 0)} unique; " + f"weekly ready {summary.get('ready_for_approval_review_count', 0)}; " + f"weekly top {summary.get('top_score', 0)}; " + f"weekly release lock {str(summary.get('release_lock_ready', False)).lower()}" + ) + if skillops_blocked: + adoption_status = "block" + if skillops_parts: + adoption_detail = adoption_detail + "; " + "; ".join(skillops_parts) + gates.append( + gate( + "operations-loop", + "运营回路", + adoption_status, + adoption_detail, + "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", + report_link(output_html, skill_dir, "reports/adoption_drift_report.md"), + ) + ) + + waiver = data["review_waivers"] + waiver_summary = waiver.get("summary", {}) + active_covered = set(waiver_summary.get("covered_gate_keys", []) or []) + prior_blockers = [item for item in gates if item["status"] == "block"] + prior_warnings = [item for item in gates if item["status"] == "warn"] + unwaived_warnings = [item for item in prior_warnings if item["key"] not in active_covered] + if not waiver: + waiver_status = "warn" + waiver_detail = "review waiver ledger is missing; warning acceptance is not auditable" + elif waiver.get("failures"): + waiver_status = "block" + waiver_detail = f"{len(waiver.get('failures', []))} invalid waiver records" + elif prior_blockers: + waiver_status = "block" + waiver_detail = f"{len(prior_blockers)} blocker gates cannot be waived in v0" + elif unwaived_warnings: + waiver_status = "warn" + waiver_detail = ( + f"{waiver_summary.get('active_count', 0)} active waivers; " + f"{len(unwaived_warnings)} warning gates still need reviewer decision" + ) + else: + waiver_status = "pass" + waiver_detail = f"{waiver_summary.get('active_count', 0)} active waivers cover current warnings" + gates.append( + gate( + "review-waivers", + "人工批准", + waiver_status, + waiver_detail, + "reports/review_waivers.json", + report_link(output_html, skill_dir, "reports/review_waivers.md"), + ) + ) + + world_class_ledger = data.get("world_class_evidence_ledger", {}) + world_class_summary = world_class_ledger.get("summary", {}) if isinstance(world_class_ledger, dict) else {} + source_check_count = int(world_class_summary.get("source_check_count", 0) or 0) + source_pass_count = int(world_class_summary.get("source_pass_count", 0) or 0) + source_blocked_count = int(world_class_summary.get("source_blocked_count", 0) or 0) + source_check_detail = ( + f"source checks {source_pass_count}/{source_check_count} pass; {source_blocked_count} blocked" + if source_check_count + else "source checks unavailable" + ) + if not world_class_ledger and maturity == "governed": + world_class_status = "warn" + world_class_detail = "world-class evidence ledger is missing; public world-class readiness cannot be claimed" + elif not world_class_ledger: + world_class_status = "pass" + world_class_detail = "world-class evidence ledger is optional until governed or public world-class readiness is claimed" + elif world_class_summary.get("ready_to_claim_world_class") is True: + world_class_status = "pass" + world_class_detail = ( + f"{world_class_summary.get('accepted_count', 0)} accepted evidence entries; " + f"{world_class_summary.get('pending_count', 0)} pending; " + f"{source_check_detail}" + ) + else: + world_class_status = "warn" + world_class_detail = ( + f"{world_class_summary.get('pending_count', 0)} pending world-class evidence entries; " + f"{world_class_summary.get('human_pending_count', 0)} human pending; " + f"{world_class_summary.get('external_pending_count', 0)} external pending; " + f"{source_check_detail}; " + f"overclaim guard {str(world_class_summary.get('overclaim_guard_active', False)).lower()}" + ) + gates.append( + gate( + "world-class-evidence", + "世界证据", + world_class_status, + world_class_detail, + "reports/world_class_evidence_ledger.json", + report_link(output_html, skill_dir, "reports/world_class_evidence_ledger.md"), + ) + ) + + registry = data["registry"] + install = data["install_simulation"] + if not registry: + if maturity in {"library", "governed"}: + registry_status = "warn" + registry_detail = "registry audit is missing; package metadata is not reviewable" + else: + registry_status = "pass" + registry_detail = "registry audit is optional until team distribution is required" + else: + compatibility = registry.get("package", {}).get("compatibility", {}) + pass_count = sum(1 for status in compatibility.values() if status == "pass") + registry_status = "block" if registry.get("failures") else ("warn" if registry.get("warnings") else "pass") + registry_detail = ( + f"{registry.get('package', {}).get('name', 'package')} " + f"{registry.get('package', {}).get('version', 'n/a')}; " + f"{pass_count}/{len(compatibility)} compatibility entries pass" + ) + if install: + install_summary = install.get("summary", {}) + permission_failures = int(install_summary.get("installer_permission_failure_count", 0) or 0) + if install.get("failures") or permission_failures: + registry_status = "block" + registry_detail += ( + f"; install {'pass' if install.get('ok') else 'fail'}" + f" with {install_summary.get('adapter_count', 0)} adapters" + f"; installer permissions {install_summary.get('installer_permission_enforced_count', 0)} enforced" + f" / {permission_failures} failures" + ) + gates.append( + gate( + "registry-audit", + "注册审计", + registry_status, + registry_detail, + "reports/registry_audit.json + reports/install_simulation.json", + report_link(output_html, skill_dir, "reports/registry_audit.md"), + ) + ) + + promotion = data["promotion"] + migration_path = ROOT / "docs" / "migration-v2.md" + if promotion: + promotion_summary = promotion.get("summary", {}) + blocked = int(promotion_summary.get("blocked", 0) or 0) + release_status = "block" if blocked else "pass" + release_detail = f"{promotion_summary.get('promote', 0)} promote; {promotion_summary.get('keep_current', 0)} keep current; {blocked} blocked" + else: + release_status = "warn" + release_detail = "promotion decisions are missing; release notes need reviewer confirmation" + upgrade = data["upgrade_check"] + if upgrade: + upgrade_summary = upgrade.get("summary", {}) + if upgrade.get("failures"): + release_status = "block" + elif upgrade.get("warnings") and release_status == "pass": + release_status = "warn" + release_detail += ( + f"; upgrade {upgrade_summary.get('declared_bump', 'n/a')}" + f" declared / {upgrade_summary.get('recommended_bump', 'n/a')} recommended" + ) + gates.append( + gate( + "release-notes", + "发布路线", + release_status, + release_detail, + "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", + report_link(output_html, skill_dir, "reports/promotion_decisions.md") if promotion else str(migration_path), + ) + ) + + return gates diff --git a/scripts/review_studio_layout.py b/scripts/review_studio_layout.py index 08e560ef..95bfe4a3 100644 --- a/scripts/review_studio_layout.py +++ b/scripts/review_studio_layout.py @@ -2,12 +2,17 @@ """Static layout contract for Review Studio HTML.""" import html +from pathlib import Path SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep Review Studio layout and CSS out of gate logic." +ROOT = Path(__file__).resolve().parent.parent +ASSET_DIR = ROOT / "assets" + + REVIEW_STUDIO_NAV = [ ("#overview", "审查总览"), ("#intent", "意图画布"), @@ -22,11 +27,16 @@ REVIEW_STUDIO_NAV = [ ("#atlas", "组合治理"), ("#telemetry", "运营回路"), ("#waivers", "人工批准"), + ("#world-class", "世界证据"), ("#registry", "注册审计"), ("#release", "发布路线"), ] +def read_layout_asset(filename: str) -> str: + return (ASSET_DIR / filename).read_text(encoding="utf-8").strip() + + def render_review_nav(nav_items: list[tuple[str, str]] | None = None) -> str: items = REVIEW_STUDIO_NAV if nav_items is None else nav_items return "".join( @@ -36,224 +46,4 @@ def render_review_nav(nav_items: list[tuple[str, str]] | None = None) -> str: def review_studio_css() -> str: - return """ - :root { - --ink: #1B365D; - --text: #24201d; - --muted: #746d66; - --line: #e7ded2; - --soft: #faf8f5; - --pass: #1e6b52; - --warn: #9a6718; - --block: #9b2c2c; - } - * { box-sizing: border-box; } - html { scroll-behavior: smooth; } - body { - margin: 0; - background: #ffffff; - color: var(--text); - font-family: Georgia, "Times New Roman", "Songti SC", serif; - line-height: 1.58; - } - nav { - position: sticky; - top: 0; - z-index: 10; - display: flex; - gap: 4px; - justify-content: center; - flex-wrap: wrap; - padding: 10px 16px; - background: rgba(255,255,255,0.94); - border-bottom: 1px solid var(--line); - backdrop-filter: blur(10px); - } - nav a { - color: var(--ink); - text-decoration: none; - font-size: 14px; - padding: 7px 10px; - border-radius: 6px; - } - nav a:hover { background: var(--soft); } - main { max-width: 1180px; margin: 0 auto; padding: 44px 28px 76px; } - header { border-bottom: 1px solid var(--line); padding-bottom: 28px; margin-bottom: 28px; } - .eyebrow { color: var(--ink); font-size: 14px; letter-spacing: .08em; text-transform: uppercase; } - h1, h2, h3 { color: var(--text); font-weight: 500; margin: 0; letter-spacing: 0; } - h1 { font-size: clamp(34px, 5vw, 64px); line-height: 1.03; max-width: 920px; margin-top: 12px; } - h2 { font-size: 30px; margin-bottom: 14px; } - h3 { font-size: 19px; } - p { margin: 0; } - .lede { max-width: 820px; color: var(--muted); font-size: 20px; margin-top: 18px; } - .decision { - display: inline-flex; - align-items: baseline; - gap: 12px; - margin-top: 24px; - padding: 12px 16px; - border: 1px solid var(--line); - border-radius: 8px; - color: var(--ink); - background: var(--soft); - } - .decision strong { font-size: 28px; } - section { padding: 30px 0; border-bottom: 1px solid var(--line); scroll-margin-top: 76px; } - .metrics, .gates { - display: grid; - grid-template-columns: repeat(5, minmax(0, 1fr)); - gap: 14px; - } - .gates { grid-template-columns: repeat(4, minmax(0, 1fr)); } - .metric, .gate { - border: 1px solid var(--line); - border-radius: 8px; - padding: 16px; - background: #fff; - min-width: 0; - } - .metric span, .gate span { display: block; color: var(--muted); font-size: 13px; } - .metric strong { display: block; color: var(--ink); font-size: 34px; line-height: 1.1; margin: 8px 0; } - .metric p, .gate p, .gate footer, .issues span, .evidence span { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; } - .gate { display: flex; flex-direction: column; gap: 10px; } - .gate.pass { border-top: 4px solid var(--pass); } - .gate.warn { border-top: 4px solid var(--warn); } - .gate.block { border-top: 4px solid var(--block); } - .gate footer { border-top: 1px solid var(--line); padding-top: 10px; } - a { color: var(--ink); } - .twocol { - display: grid; - grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); - gap: 22px; - align-items: start; - } - .panel { - border: 1px solid var(--line); - border-radius: 8px; - padding: 18px; - background: #fff; - } - .panel p { color: var(--muted); } - .kv-grid { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 10px 12px; - margin: 2px 0 0; - } - .kv-grid div { - min-width: 0; - padding: 9px 0 0; - border-top: 1px solid var(--line); - } - .kv-grid dt { - color: var(--muted); - font-size: 13px; - } - .kv-grid dd { - margin: 2px 0 0; - color: var(--ink); - font-size: 15px; - overflow-wrap: anywhere; - } - .issues, .evidence { - list-style: none; - padding: 0; - margin: 0; - display: grid; - gap: 12px; - } - .issues li, .evidence li { - border-left: 3px solid var(--line); - padding-left: 12px; - display: grid; - gap: 3px; - } - .muted { color: var(--muted); } - .actions-grid { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 16px; - } - .annotations-grid { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 16px; - } - .action-card { - border: 1px solid var(--line); - border-radius: 8px; - padding: 18px; - background: #fff; - display: grid; - gap: 10px; - min-width: 0; - } - .action-card.warn { border-left: 4px solid var(--warn); } - .action-card.block { border-left: 4px solid var(--block); } - .annotation-card { - border: 1px solid var(--line); - border-radius: 8px; - padding: 18px; - background: #fff; - display: grid; - gap: 10px; - min-width: 0; - } - .annotation-card.warning { border-left: 4px solid var(--warn); } - .annotation-card.blocker { border-left: 4px solid var(--block); } - .annotation-card.resolved { opacity: .72; } - .action-card span, - .annotation-card span, - .action-card small, - .annotation-card small, - .action-card footer, - .annotation-card footer, - .action-card dd { - color: var(--muted); - font-size: 14px; - overflow-wrap: anywhere; - } - .annotation-card dd { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; } - .action-card dl, .annotation-card dl { - display: grid; - grid-template-columns: 80px minmax(0, 1fr); - gap: 6px 10px; - margin: 0; - } - .action-card dt, .annotation-card dt { color: var(--ink); font-size: 14px; } - .action-card dd, .annotation-card dd { margin: 0; } - .source-ref-list { - list-style: none; - padding: 0; - margin: 0; - display: grid; - gap: 6px; - } - .source-ref-list li { - display: grid; - gap: 2px; - min-width: 0; - padding: 8px 0 0; - border-top: 1px solid var(--line); - } - .source-ref-list a, - .source-ref-list span { - overflow-wrap: anywhere; - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 13px; - } - .source-ref-list small { - font-size: 12px; - color: var(--muted); - } - code { - font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; - font-size: 13px; - } - @media (max-width: 980px) { - .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .kv-grid { grid-template-columns: 1fr; } - main { padding: 32px 18px 60px; } - nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; } - nav a { flex: 0 0 auto; } - } - """.strip() + return read_layout_asset("review-studio.css") diff --git a/scripts/review_studio_output_review.py b/scripts/review_studio_output_review.py new file mode 100644 index 00000000..5e494e4b --- /dev/null +++ b/scripts/review_studio_output_review.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Output review HTML helpers for Review Studio.""" + +import html +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to render output review checklist cards." + + +def render_required_fields(fields: dict[str, Any]) -> str: + if not fields: + return "

    暂无字段要求。

    " + items = [] + for key, value in fields.items(): + items.append(f"
  • {html.escape(str(key))}{html.escape(str(value))}
    • ") + return "
      " + "".join(items) + "
    " + + +def render_command_list(commands: dict[str, Any]) -> str: + if not commands: + return "

    暂无命令。

    " + items = [] + for label, command in commands.items(): + if not command: + continue + items.append( + "
  • " + f"{html.escape(str(label))}" + f"{html.escape(str(command))}" + "
    • " + ) + return "
      " + "".join(items) + "
    " if items else "

    暂无命令。

    " + + +def render_contract(items: list[Any]) -> str: + if not items: + return "

    暂无边界。

    " + return "
      " + "".join(f"
    • {html.escape(str(item))}
      • " for item in items) + "
    " + + +def render_output_review_checklist(adjudication: dict[str, Any]) -> str: + checklist = adjudication.get("reviewer_checklist", []) if isinstance(adjudication, dict) else [] + if not checklist: + return "

    当前没有输出评审操作清单。

    " + cards = [] + for item in checklist: + readiness = str(item.get("readiness", "awaiting-decision")) + answer_key = "可见" if item.get("answer_key_visible") else "隐藏" + cards.append( + "
    " + f"
    {html.escape(readiness)} · 答案{html.escape(answer_key)}" + f"

    {html.escape(str(item.get('case_id', 'case')))}

    " + f"

    {html.escape(str(item.get('blocking_reason', '')))}

    " + f"
    盲评包
    {html.escape(str(item.get('blind_pack_path', '')))}
    " + f"
    决策表
    {html.escape(str(item.get('decisions_path', '')))}
    " + f"
    提示哈希
    {html.escape(str(item.get('prompt_sha256', '')))}
    " + "
    " + "

    操作命令

    " + + render_command_list(item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {}) + + "
    " + "

    字段要求

    " + + render_required_fields(item.get("required_fields", {}) if isinstance(item.get("required_fields", {}), dict) else {}) + + "
    " + "

    隐私边界

    " + + render_contract(item.get("privacy_contract", []) if isinstance(item.get("privacy_contract", []), list) else []) + + "
    " + "
    " + "
    " + ) + return "
    " + "".join(cards) + "
    " + + +def render_output_review_section(adjudication: dict[str, Any]) -> str: + return ( + "
    " + "

    评审清单

    " + "

    先打开 reports/output_review_kit.md；每张卡片对应一个 blind A/B case，说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。

    " + f"{render_output_review_checklist(adjudication)}" + "
    " + ) diff --git a/scripts/review_studio_panels.py b/scripts/review_studio_panels.py new file mode 100644 index 00000000..7addbb1d --- /dev/null +++ b/scripts/review_studio_panels.py @@ -0,0 +1,88 @@ +import html +from typing import Any + +from review_studio_gates import status_label + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep small Review Studio panel renderers out of the main page composer." + + +def render_gate_list(gates: list[dict[str, str]]) -> str: + items = [] + for item in gates: + link_html = f"证据" if item.get("link") else "" + items.append( + "
    " + f"
    {html.escape(status_label(item['status']))}

    {html.escape(item['label'])}

    " + f"

    {html.escape(item['detail'])}

    " + f"
    {html.escape(item['evidence'])} {link_html}
    " + "
    " + ) + return "".join(items) + + +def render_insights(cards: list[dict[str, str]]) -> str: + return "".join( + ( + "
    " + f"{html.escape(item['label'])}" + f"{html.escape(item['value'])}" + f"

    {html.escape(item['detail'])}

    " + "
    " + ) + for item in cards + ) + + +def render_issue_list(title: str, items: list[dict[str, str]]) -> str: + if not items: + return f"

    {html.escape(title)}

    无。

    " + body = "".join( + ( + "
  • " + f"{html.escape(item['label'])}" + f"{html.escape(item['detail'])}" + "
    • " + ) + for item in items + ) + return f"

    {html.escape(title)}

      {body}
    " + + +def render_review_annotations_panel(annotations_report: dict[str, Any]) -> str: + annotations = annotations_report.get("annotations", []) if isinstance(annotations_report, dict) else [] + if not annotations: + return "

    当前没有 reviewer 批注。

    " + cards = [] + for item in annotations: + line_suffix = f":{item['line']}" if item.get("line") else "" + target_label = f"{item.get('target_path', '')}{line_suffix}" + meta = " · ".join( + part + for part in [ + str(item.get("gate_key", "")), + str(item.get("reviewer", "")), + str(item.get("created_at", "")), + ] + if part + ) + cards.append( + "
    " + f"
    {html.escape(str(item.get('severity', 'note')))} · {html.escape(str(item.get('status', 'open')))}" + f"

    {html.escape(str(item.get('id', 'annotation')))}

    " + f"

    {html.escape(str(item.get('body', '')))}

    " + f"
    位置
    {html.escape(target_label)}
    " + f"
    Gate
    {html.escape(str(item.get('gate_key', '')))}
    " + f"
    建议
    {html.escape(str(item.get('suggested_action', '') or '无'))}
    " + f"{html.escape(meta)}" + f"
    {html.escape(str(item.get('source_excerpt', '')))}
    " + "
    " + ) + return "".join(cards) diff --git a/scripts/review_studio_skillops.py b/scripts/review_studio_skillops.py new file mode 100644 index 00000000..4ebdc22f --- /dev/null +++ b/scripts/review_studio_skillops.py @@ -0,0 +1,48 @@ +from typing import Any + +from review_studio_formatting import render_kv_grid + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep SkillOps summary panels out of the main HTML renderer." + + +def render_skillops_section(data: dict[str, Any]) -> str: + daily_summary = data["daily_skillops"].get("summary", {}) + weekly_summary = data["weekly_curator"].get("summary", {}) + daily_panel = render_kv_grid( + daily_summary, + [ + "decision", + "proposal_count", + "approval_count", + "pending_review_count", + "release_lock_ready", + "public_world_class_ready", + "writes_source_files", + "auto_patch_enabled", + ], + "no daily SkillOps summary", + ) + weekly_panel = render_kv_grid( + weekly_summary, + [ + "decision", + "week_id", + "daily_report_count", + "unique_opportunity_count", + "ready_for_approval_review_count", + "proposal_review_count", + "top_score", + "release_lock_ready", + "writes_source_files", + "auto_patch_enabled", + ], + "no weekly curator summary", + ) + return f""" +
    +

    日常运维

    {daily_panel}
    +

    周度队列

    {weekly_panel}
    +
    +""" diff --git a/scripts/review_studio_waivers.py b/scripts/review_studio_waivers.py new file mode 100644 index 00000000..92d05176 --- /dev/null +++ b/scripts/review_studio_waivers.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Review Studio waiver candidate rendering helpers.""" + +import html +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep waiver candidate layout out of the main renderer." + + +def render_waiver_candidates_panel(review_waivers: dict[str, Any]) -> str: + candidates = review_waivers.get("waiver_candidates", []) if isinstance(review_waivers, dict) else [] + if not candidates: + return "

    当前没有需要 reviewer 处理的批准候选。

    " + cards = [] + for item in candidates: + allowed = bool(item.get("waiver_allowed")) + allowed_label = "可批准" if allowed else "不可批准" + status = str(item.get("status", "unknown")) + required_review = item.get("required_review", []) + required_html = "".join( + f"
  • {html.escape(str(review_item))}
    • " + for review_item in required_review + ) or "
  • 证据不足，需要先补充审查条件。
    • " + decision_options = item.get("decision_options", []) + decision_options_label = ", ".join(str(option) for option in decision_options) if decision_options else "无" + cards.append( + "
    " + f"
    {html.escape(allowed_label)} · {html.escape(status)}" + f"

    {html.escape(str(item.get('label') or item.get('gate_key') or 'Waiver Candidate'))}

    " + f"

    {html.escape(str(item.get('risk_summary') or '证据不足'))}

    " + "
    " + f"
    Gate
    {html.escape(str(item.get('gate_key', '')))}
    " + f"
    证据
    {html.escape(str(item.get('suggested_evidence', '')))}
    " + f"
    选项
    {html.escape(decision_options_label)}
    " + f"
    边界
    {html.escape(str(item.get('world_class_boundary', '')))}
    " + "
    " + "

    审查条件

    " + f"
      {required_html}
    " + "
    建议命令" + f"{html.escape(str(item.get('suggested_command', '')))}
    " + "
    " + ) + return "
    " + "".join(cards) + "
    " diff --git a/scripts/review_studio_world_class.py b/scripts/review_studio_world_class.py new file mode 100644 index 00000000..099e72d2 --- /dev/null +++ b/scripts/review_studio_world_class.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""World-class evidence HTML helpers for Review Studio.""" + +import html +from typing import Any + +from world_class_source_checks import build_source_checklist + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to render world-class evidence cards." + + +def render_inline_list(items: list[Any], empty_label: str) -> str: + if not items: + return f"

    {html.escape(empty_label)}

    " + return "
      " + "".join(f"
    • {html.escape(str(item))}
      • " for item in items) + "
    " + + +def render_source_checks(entry: dict[str, Any]) -> str: + rows = entry.get("source_checklist") + if not isinstance(rows, list): + rows = build_source_checklist([entry]) + if not rows: + return "

    暂无源证据检查。

    " + items = [] + for row in rows: + status = str(row.get("status", "blocked")) + items.append( + "
  • " + f"{html.escape(str(row.get('label', '')))}" + f"{html.escape(str(row.get('field', '')))}: {html.escape(str(row.get('actual', '')))} / {html.escape(str(row.get('expected', '')))}" + f"{html.escape(str(row.get('next_action', '')))}" + "
    • " + ) + return "
      " + "".join(items) + "
    " + + +def render_world_class_evidence_entries(ledger: dict[str, Any]) -> str: + entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] + if not entries: + return "

    当前没有 world-class 证据条目。

    " + cards = [] + for entry in entries: + observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + observed_summary = "; ".join(f"{key}: {value}" for key, value in observed.items()) + submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} + submission_summary = "; ".join( + f"{key}: {value}" for key, value in submission.items() if key in {"status", "path", "attested_real_evidence", "privacy_contract_satisfied"} + ) + status = str(entry.get("status", "pending")) + status_label_text = "已接受" if status == "accepted" else "待补证" + cards.append( + "
    " + f"
    {html.escape(status_label_text)} · {html.escape(str(entry.get('category', '')))}" + f"

    {html.escape(str(entry.get('label', entry.get('key', 'evidence'))))}

    " + f"

    {html.escape(str(entry.get('objective', '')))}

    " + f"
    负责人
    {html.escape(str(entry.get('owner', '')))}
    " + f"
    当前状态
    {html.escape(str(entry.get('current', '')))}
    " + f"
    下一步
    {html.escape(str(entry.get('next_action', '')))}
    " + f"
    观测值
    {html.escape(observed_summary or '无')}
    " + f"
    提交态
    {html.escape(submission_summary or 'missing')}
    " + "

    执行步骤

    " + + render_inline_list(entry.get("runbook", []), "暂无执行步骤。") + + "
    " + "
    " + "

    完成定义

    " + + render_inline_list(entry.get("success_checks", []), "暂无完成定义。") + + "
    " + "

    证据来源

    " + + render_inline_list(entry.get("evidence_artifacts", []), "暂无证据来源。") + + "
    " + "

    隐私约束

    " + + render_inline_list(entry.get("privacy_contract", []), "暂无隐私约束。") + + "
    " + "
    " + "

    源证据检查

    " + + render_source_checks(entry) + + "
    " + "
    " + ) + return "
    " + "".join(cards) + "
    " + + +def render_command_list(commands: dict[str, Any]) -> str: + if not commands: + return "

    暂无命令。

    " + items = [] + for label, command in commands.items(): + if not command: + continue + items.append( + "
  • " + f"{html.escape(str(label))}" + f"{html.escape(str(command))}" + "
    • " + ) + return "
      " + "".join(items) + "
    " if items else "

    暂无命令。

    " + + +def render_world_class_intake_checklist(intake: dict[str, Any]) -> str: + items = intake.get("operator_checklist", []) if isinstance(intake, dict) else [] + if not items: + return "

    当前没有 world-class 证据操作清单。

    " + cards = [] + for item in items: + readiness = str(item.get("readiness", "awaiting-submission")) + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + cards.append( + "
    " + f"
    {html.escape(readiness)} · {html.escape(str(item.get('category', '')))}" + f"

    {html.escape(str(item.get('label', item.get('evidence_key', 'evidence'))))}

    " + f"

    {html.escape(str(item.get('blocking_reason', '')))}

    " + f"
    负责人
    {html.escape(str(item.get('owner', '')))}
    " + f"
    模板
    {html.escape(str(item.get('template_path', '')))}
    " + f"
    提交
    {html.escape(str(item.get('submission_path', '')))}
    " + f"
    下一步
    {html.escape(str(item.get('next_action', '')))}
    " + "
    " + "

    操作命令

    " + + render_command_list(item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {}) + + "
    " + "

    收集要求

    " + + render_inline_list(must_collect.get("provenance_requirements", []), "暂无来源要求。") + + "
    " + "

    执行步骤

    " + + render_inline_list(must_collect.get("runbook", []), "暂无执行步骤。") + + "
    " + "

    通过条件

    " + + render_inline_list(must_collect.get("success_checks", []), "暂无通过条件。") + + "
    " + "

    隐私边界

    " + + render_inline_list(must_collect.get("privacy_contract", []), "暂无隐私边界。") + + "
    " + "
    " + "
    " + ) + return "
    " + "".join(cards) + "
    " diff --git a/scripts/review_viewer_data.py b/scripts/review_viewer_data.py new file mode 100644 index 00000000..d69f9dc2 --- /dev/null +++ b/scripts/review_viewer_data.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 +"""Data preparation helpers for the compact review viewer.""" + +import json +import re +from pathlib import Path + +from render_intent_confidence import render_intent_confidence +from render_intent_dialogue import render_intent_dialogue +from render_iteration_directions import render_iteration_directions +from render_artifact_design_profile import render_artifact_design_profile +from render_output_risk_profile import render_output_risk_profile +from render_prompt_quality_profile import render_prompt_quality_profile +from render_reference_scan import render_reference_scan +from render_reference_synthesis import render_reference_synthesis +from render_skill_overview import render_skill_overview + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_review_viewer.py to assemble Review Viewer data before HTML rendering." + + +def load_json(path: Path) -> dict: + if not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def load_feedback_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "feedback-log.json") + return payload if isinstance(payload, dict) else {} + + +def load_baseline_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "baseline-compare.json") + return payload if isinstance(payload, dict) else {} + + +def load_specific_compare(skill_dir: Path) -> dict: + candidates = [ + skill_dir / "reports" / "description_optimization.json", + skill_dir.parent / "optimization" / "reports" / "description_optimization.json", + ] + for path in candidates: + payload = load_json(path) + if isinstance(payload, dict) and payload: + return payload + return {} + + +def load_specific_promotion(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "promotion_decisions.json") + return payload if isinstance(payload, dict) else {} + + +def load_benchmark_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "github-benchmark-scan.json") + return payload if isinstance(payload, dict) else {} + + +def load_reference_synthesis_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "reference-synthesis.json") + return payload if isinstance(payload, dict) else {} + + +def load_output_risk_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "output-risk-profile.json") + return payload if isinstance(payload, dict) else {} + + +def load_artifact_design_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "artifact-design-profile.json") + return payload if isinstance(payload, dict) else {} + + +def load_prompt_quality_summary(skill_dir: Path) -> dict: + payload = load_json(skill_dir / "reports" / "prompt-quality-profile.json") + return payload if isinstance(payload, dict) else {} + + +def ensure_report_inputs(skill_dir: Path) -> dict: + overview_json = skill_dir / "reports" / "skill-overview.json" + intent_confidence_json = skill_dir / "reports" / "intent-confidence.json" + intent_json = skill_dir / "reports" / "intent-dialogue.json" + reference_json = skill_dir / "reports" / "reference-scan.json" + reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json" + output_risk_json = skill_dir / "reports" / "output-risk-profile.json" + artifact_design_json = skill_dir / "reports" / "artifact-design-profile.json" + prompt_quality_json = skill_dir / "reports" / "prompt-quality-profile.json" + directions_json = skill_dir / "reports" / "iteration-directions.json" + + overview_payload = load_json(overview_json) if overview_json.exists() else {} + intent_confidence_payload = load_json(intent_confidence_json) if intent_confidence_json.exists() else {} + intent_payload = load_json(intent_json) if intent_json.exists() else {} + reference_payload = load_json(reference_json) if reference_json.exists() else {} + reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {} + output_risk_payload = load_json(output_risk_json) if output_risk_json.exists() else {} + artifact_design_payload = load_json(artifact_design_json) if artifact_design_json.exists() else {} + prompt_quality_payload = load_json(prompt_quality_json) if prompt_quality_json.exists() else {} + directions_payload = load_json(directions_json) if directions_json.exists() else {} + + intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"] + intent = intent_payload or render_intent_dialogue(skill_dir)["summary"] + reference = reference_payload or render_reference_scan(skill_dir, [])["summary"] + reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"] + output_risk = output_risk_payload or render_output_risk_profile(skill_dir)["summary"] + artifact_design = artifact_design_payload or render_artifact_design_profile(skill_dir)["summary"] + prompt_quality = prompt_quality_payload or render_prompt_quality_profile(skill_dir)["summary"] + iteration_payload = directions_payload or render_iteration_directions(skill_dir) + iteration = iteration_payload.get("summary", {}) + overview = overview_payload or render_skill_overview(skill_dir)["summary"] + feedback = load_feedback_summary(skill_dir) + baseline = load_baseline_summary(skill_dir) + compare = load_specific_compare(skill_dir) + promotion = load_specific_promotion(skill_dir) + benchmark = load_benchmark_summary(skill_dir) + reference_synthesis = load_reference_synthesis_summary(skill_dir) + output_risk = load_output_risk_summary(skill_dir) or output_risk + artifact_design = load_artifact_design_summary(skill_dir) or artifact_design + prompt_quality = load_prompt_quality_summary(skill_dir) or prompt_quality + return { + "overview": overview, + "intent_confidence": intent_confidence, + "intent": intent, + "reference": reference, + "iteration": iteration_payload, + "feedback": feedback, + "baseline": baseline, + "compare": compare, + "promotion": promotion, + "benchmark": benchmark, + "reference_synthesis": reference_synthesis, + "output_risk": output_risk, + "artifact_design": artifact_design, + "prompt_quality": prompt_quality, + } + +def architecture_steps(overview: dict) -> list[dict]: + logic = overview.get("logic_steps", [])[:3] + usage = overview.get("usage_steps", [])[:2] + return [ + {"label": "Inputs", "detail": "workflow, prompt, transcript, docs, or notes"}, + {"label": "Boundary", "detail": overview.get("description", "Define the recurring job and exclusions.")}, + {"label": "Logic", "detail": "; ".join(logic) if logic else "Understand, execute, and validate."}, + {"label": "Usage", "detail": "; ".join(usage) if usage else "Load the skill and follow the workflow."}, + {"label": "Next", "detail": "Review the top iteration directions before growing the package."}, + ] + + +def compare_rows(compare: dict) -> list[dict]: + if not compare: + return [] + rows = [] + items = [ + ("Baseline", compare.get("baseline", {})), + ("Current", compare.get("current_candidate", {})), + (compare.get("winner", {}).get("label", "Winner"), compare.get("winner", {})), + ] + for label, payload in items: + if not payload: + continue + dev = payload.get("dev", {}) + holdout = payload.get("holdout", {}) + rows.append( + { + "label": label, + "tokens": payload.get("estimated_tokens", 0), + "dev_errors": dev.get("total_errors", 0), + "holdout_errors": holdout.get("total_errors", 0), + "strategy": payload.get("strategy", "existing"), + } + ) + return rows + + +def benchmark_cards(benchmark: dict) -> list[dict]: + cards = [] + for repo in benchmark.get("repositories", [])[:3]: + cards.append( + { + "name": repo.get("full_name", "Unknown repo"), + "borrow": repo.get("borrow", [])[:2], + "avoid": repo.get("avoid", [])[:1], + } + ) + return cards + + +def synthesis_cards(reference_synthesis: dict) -> list[dict]: + cards = [] + for track in reference_synthesis.get("source_tracks", [])[:3]: + cards.append( + { + "name": track.get("name", "Unknown track"), + "borrow": [track.get("borrow", "")] if track.get("borrow") else [], + "avoid": [track.get("avoid", "")] if track.get("avoid") else [], + } + ) + return cards + + +def split_sentences(text: str) -> list[str]: + if not text: + return [] + parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", " ".join(text.split())) if item.strip()] + return parts + + +def metric_delta(current: int | float, baseline: int | float) -> str: + delta = current - baseline + if delta == 0: + return "0" + return f"{delta:+}" + + +def variant_diff_cards(compare: dict) -> list[dict]: + baseline = compare.get("baseline", {}) + current = compare.get("current_candidate", {}) + winner = compare.get("winner", {}) + variants = [ + ("Baseline", baseline), + ("Current", current), + (f"Winner — {winner.get('label', 'Winner')}", winner), + ] + baseline_sentences = split_sentences(baseline.get("description", "")) + baseline_set = set(baseline_sentences) + baseline_dev = baseline.get("dev", {}).get("total_errors", 0) + baseline_holdout = baseline.get("holdout", {}).get("total_errors", 0) + cards = [] + seen = set() + for label, payload in variants: + if not payload: + continue + unique_key = (payload.get("description"), payload.get("strategy"), label) + if unique_key in seen: + continue + seen.add(unique_key) + description = payload.get("description", "") + sentences = split_sentences(description) + sentence_set = set(sentences) + added = [item for item in sentences if item not in baseline_set][:3] + removed = [item for item in baseline_sentences if item not in sentence_set][:2] + dev_errors = payload.get("dev", {}).get("total_errors", 0) + holdout_errors = payload.get("holdout", {}).get("total_errors", 0) + cards.append( + { + "label": label, + "strategy": payload.get("strategy", "existing"), + "description": description, + "tokens": payload.get("estimated_tokens", 0), + "dev_errors": dev_errors, + "holdout_errors": holdout_errors, + "token_delta": metric_delta(payload.get("estimated_tokens", 0), baseline.get("estimated_tokens", 0)), + "dev_delta": metric_delta(dev_errors, baseline_dev), + "holdout_delta": metric_delta(holdout_errors, baseline_holdout), + "added": added if label != "Baseline" else baseline_sentences[:3], + "removed": removed, + } + ) + return cards + + +def evidence_readiness(report: dict) -> dict: + intent_confidence = report.get("intent_confidence", {}) + reference_synthesis = report.get("reference_synthesis", {}) + output_risk = report.get("output_risk", {}) + artifact_design = report.get("artifact_design", {}) + prompt_quality = report.get("prompt_quality", {}) + benchmark = report.get("benchmark", {}) + synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {} + pattern_gate = synthesis.get("pattern_gate", {}) if isinstance(synthesis, dict) else {} + accepted_patterns = pattern_gate.get("accepted", []) if isinstance(pattern_gate, dict) else [] + conflicts = synthesis.get("conflicts", []) if isinstance(synthesis, dict) else [] + checks = [ + { + "label": "Intent clarity", + "status": "ready" if intent_confidence.get("gate_passed") else "needs review", + "detail": f"{intent_confidence.get('score', 0)}/100 intent confidence.", + }, + { + "label": "Benchmark coverage", + "status": "ready" if len(benchmark.get("repositories", [])) >= 2 else "needs evidence", + "detail": f"{len(benchmark.get('repositories', []))} GitHub benchmark repositories attached.", + }, + { + "label": "Pattern gate", + "status": "ready" if accepted_patterns else "needs review", + "detail": pattern_gate.get("summary", "No pattern gate summary attached."), + }, + { + "label": "Conflict handling", + "status": "ready" if not conflicts else "decision needed", + "detail": "No material conflicts detected." if not conflicts else conflicts[0].get("summary", "Conflict detected."), + }, + { + "label": "Output risk profile", + "status": "ready" if output_risk.get("risk_families") else "needs review", + "detail": f"{len(output_risk.get('risk_families', []))} output risk families attached.", + }, + { + "label": "Artifact design profile", + "status": "ready" if artifact_design.get("primary_artifact") else "needs review", + "detail": artifact_design.get("primary_artifact", {}).get("direction", "No artifact design profile attached."), + }, + { + "label": "Prompt quality profile", + "status": "ready" if prompt_quality.get("quality_matrix") else "needs review", + "detail": f"{prompt_quality.get('overall_quality_score', 0)}/100 prompt-facing quality score.", + }, + ] + ready_count = sum(1 for item in checks if item["status"] == "ready") + return { + "score": int(ready_count / len(checks) * 100), + "checks": checks, + "reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery.", + } diff --git a/scripts/run_conformance_suite.py b/scripts/run_conformance_suite.py index d51aa253..24bdf872 100644 --- a/scripts/run_conformance_suite.py +++ b/scripts/run_conformance_suite.py @@ -10,6 +10,8 @@ try: except ImportError: # pragma: no cover yaml = None +from skill_ir_paths import find_skill_ir as find_skill_ir_document + ROOT = Path(__file__).resolve().parent.parent DEFAULT_TARGETS = ["openai", "claude", "agent-skills", "vscode", "generic"] @@ -64,10 +66,7 @@ def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]: def find_skill_ir(skill_dir: Path, name: str) -> dict[str, Any]: - direct = load_json(skill_dir / "reports" / "skill-ir.json") - if direct: - return direct - return load_json(skill_dir / "skill-ir" / "examples" / f"{name}.json") + return find_skill_ir_document(skill_dir, name)[0] def add_check(checks: list[str], failures: list[str], condition: bool, passed: str, failed: str) -> None: @@ -144,7 +143,11 @@ def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict add_check(checks, failures, re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name) is not None, "name is runtime-safe", "name contains unsafe characters") if target in {"agent-skills", "vscode"}: - add_check(checks, failures, skill_dir.name == name, "directory name matches skill name", "directory name must match skill name") + add_check(checks, failures, bool(name), "package identity derives from skill name", "Missing package identity") + if skill_dir.name != name: + warnings.append( + "source checkout directory differs from skill name; package verification must enforce archive top-level identity." + ) for field in ("name", "version", "owner", "status", "maturity_tier", "review_cadence"): add_check(checks, failures, bool(manifest.get(field)), f"manifest.{field} exists", f"Missing manifest.{field}") diff --git a/scripts/run_output_execution.py b/scripts/run_output_execution.py index ad0a97af..0e623689 100644 --- a/scripts/run_output_execution.py +++ b/scripts/run_output_execution.py @@ -164,7 +164,7 @@ def command_run( execution_kind = str(payload.get("execution_kind", "command")) provider = str(payload.get("provider", "")) model = str(payload.get("model", "")) - model_executed = execution_kind == "model" or bool(model and provider) + model_executed = execution_kind == "model" and bool(model and provider) return { "case_id": request["case_id"], "variant": variant, @@ -247,7 +247,7 @@ def render_markdown(payload: dict[str, Any]) -> str: [ "No model-executed runs are recorded yet.", "", - "Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence.", + "Use `python3 scripts/yao.py output-exec --provider-runner openai` or `--runner-command` with a reviewed provider-backed runner to replace recorded fixtures with real model output evidence.", "", ] ) @@ -285,7 +285,7 @@ def render_markdown(payload: dict[str, Any]) -> str: "## Next Fixes", "", "- Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence.", - "- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior.", + "- Use `scripts/provider_output_eval_runner.py` for provider-backed holdout cases when release confidence depends on real generation behavior.", "- Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse.", ] ) diff --git a/scripts/simulate_install.py b/scripts/simulate_install.py index 03cc92c8..e6f6c107 100644 --- a/scripts/simulate_install.py +++ b/scripts/simulate_install.py @@ -4,7 +4,7 @@ import json import shutil import tempfile import zipfile -from datetime import date +from datetime import date, datetime from pathlib import Path, PurePosixPath from typing import Any @@ -91,22 +91,111 @@ def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str, failures.append(detail) -def adapter_targets(package_dir: Path) -> list[str]: - targets_dir = package_dir / "targets" +def package_name(package_manifest: dict[str, Any], skill_dir: Path) -> str: + return str(package_manifest.get("name") or skill_dir.name) + + +def adapter_targets(adapter_root: Path) -> list[str]: + targets_dir = adapter_root / "targets" if not targets_dir.exists(): return [] return sorted(path.name for path in targets_dir.iterdir() if path.is_dir()) +def sorted_strings(value: Any) -> list[str]: + return sorted(str(item) for item in value) if isinstance(value, list) else [] + + +def parse_date(value: str) -> date | None: + try: + return datetime.strptime(value, "%Y-%m-%d").date() + except ValueError: + return None + + +def permission_policy_checks( + installed_dir: Path | None, + adapter_root: Path, + targets: list[str], + generated_at: str, + checks: list[dict[str, str]], + failures: list[str], +) -> dict[str, int]: + installed = installed_dir is not None and installed_dir.exists() and installed_dir.is_dir() + policy = load_json(installed_dir / "security" / "permission_policy.json") if installed else {} + capabilities = policy.get("capabilities", {}) if isinstance(policy.get("capabilities"), dict) else {} + generated_date = parse_date(generated_at[:10]) + enforced_count = 0 + permission_failure_start = len(failures) + declared_capabilities: set[str] = set() + + add_check(checks, failures, "permission-policy-load", bool(capabilities), "Installed permission policy is readable") + for target in targets: + adapter = load_json(adapter_root / "targets" / target / "adapter.json") + target_contract = adapter.get("target_permission_contract", {}) if isinstance(adapter, dict) else {} + target_capabilities = sorted_strings(target_contract.get("declared_capabilities")) + declared_capabilities.update(target_capabilities) + add_check( + checks, + failures, + f"permission-{target}-contract", + bool(target_contract), + f"{target} adapter exposes target permission contract for installer enforcement", + ) + if not target_capabilities: + continue + for capability in target_capabilities: + approval = capabilities.get(capability, {}) if isinstance(capabilities, dict) else {} + enforcement = approval.get("target_enforcement", {}) if isinstance(approval.get("target_enforcement"), dict) else {} + expires_at = str(approval.get("expires_at", "")).strip() + expiry_date = parse_date(expires_at) if expires_at else None + common_ok = ( + approval.get("decision") == "approved" + and bool(str(approval.get("reviewer", "")).strip()) + and bool(str(approval.get("scope", "")).strip()) + and bool(str(approval.get("reason", "")).strip()) + and isinstance(approval.get("evidence"), list) + and bool(approval.get("evidence")) + and bool(enforcement) + and expiry_date is not None + and (generated_date is None or expiry_date >= generated_date) + ) + add_check( + checks, + failures, + f"permission-{target}-{capability}-approved", + bool(common_ok), + f"{target} capability {capability} has active reviewer approval", + ) + target_enforcement_ok = bool(str(enforcement.get(target, "")).strip()) + add_check( + checks, + failures, + f"permission-{target}-{capability}-target-enforcement", + target_enforcement_ok, + f"{target} capability {capability} has target enforcement note", + ) + if common_ok and target_enforcement_ok: + enforced_count += 1 + + return { + "installer_permission_enforced_count": enforced_count, + "installer_permission_failure_count": len(failures) - permission_failure_start, + "permission_target_count": len(targets), + "permission_capability_count": len(declared_capabilities), + } + + def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | None, generated_at: str) -> dict[str, Any]: skill_dir = skill_dir.resolve() package_dir = package_dir.resolve() checks: list[dict[str, str]] = [] failures: list[str] = [] warnings: list[str] = [] - archive_path = package_dir / f"{skill_dir.name}.zip" package_manifest = load_json(package_dir / "manifest.json") - installed_dir = Path("") + package_root = package_name(package_manifest, skill_dir) + archive_path = package_dir / f"{package_root}.zip" + installed_dir: Path | None = None archive_entries: list[str] = [] temp_dir: tempfile.TemporaryDirectory[str] | None = None @@ -117,7 +206,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No else: requested_root = install_root.resolve() requested_root.mkdir(parents=True, exist_ok=True) - install_base = requested_root / f"simulate-{skill_dir.name}" + install_base = requested_root / f"simulate-{package_root}" if install_base.exists(): shutil.rmtree(install_base) install_base.mkdir(parents=True, exist_ok=True) @@ -134,7 +223,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No unsafe_entries = unsafe_zip_entries(archive_entries) add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries") roots = top_level_dirs(archive_entries) - add_check(checks, failures, "single-top-level", roots == [skill_dir.name], f"Archive top-level directory is {skill_dir.name}") + add_check(checks, failures, "single-top-level", roots == [package_root], f"Archive top-level directory is {package_root}") if not unsafe_entries and roots: with zipfile.ZipFile(archive_path) as archive: archive.extractall(install_base) @@ -144,18 +233,31 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No source_manifest = load_json(installed_dir / "manifest.json") if installed_dir else {} interface_doc = load_yaml(installed_dir / "agents" / "interface.yaml") if installed_dir else {} add_check(checks, failures, "entrypoint-load", bool(frontmatter), "Installed SKILL.md frontmatter is readable") - add_check(checks, failures, "entrypoint-name", frontmatter.get("name") == skill_dir.name, "Installed SKILL.md name matches package directory") + add_check(checks, failures, "entrypoint-name", frontmatter.get("name") == package_root, "Installed SKILL.md name matches package directory") add_check(checks, failures, "entrypoint-description", bool(frontmatter.get("description")), "Installed SKILL.md description is present") add_check(checks, failures, "manifest-load", bool(source_manifest), "Installed manifest.json is readable") add_check(checks, failures, "manifest-name", source_manifest.get("name") == package_manifest.get("name"), "Installed manifest name matches package manifest") add_check(checks, failures, "manifest-version", source_manifest.get("version") == package_manifest.get("version"), "Installed manifest version matches package manifest") add_check(checks, failures, "interface-load", bool(interface_doc.get("interface")), "Installed agents/interface.yaml is readable") - add_check(checks, failures, "overview-report", (installed_dir / "reports" / "skill-overview.html").exists(), "Installed overview report is present") - add_check(checks, failures, "review-studio-report", (installed_dir / "reports" / "review-studio.html").exists(), "Installed Review Studio report is present") + add_check( + checks, + failures, + "overview-report", + installed_dir is not None and (installed_dir / "reports" / "skill-overview.html").exists(), + "Installed overview report is present", + ) + add_check( + checks, + failures, + "review-studio-report", + installed_dir is not None and (installed_dir / "reports" / "review-studio.html").exists(), + "Installed Review Studio report is present", + ) - adapters = adapter_targets(package_dir) + adapter_root = package_dir + adapters = adapter_targets(adapter_root) for target in adapters: - adapter = load_json(package_dir / "targets" / target / "adapter.json") + adapter = load_json(adapter_root / "targets" / target / "adapter.json") add_check(checks, failures, f"adapter-{target}", bool(adapter), f"{target} adapter is readable after package install simulation") add_check( checks, @@ -165,6 +267,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No f"{target} adapter name matches package manifest", ) + permission_summary = permission_policy_checks(installed_dir, adapter_root, adapters, generated_at, checks, failures) if not adapters: warnings.append("No target adapters found in package directory.") @@ -191,6 +294,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No "manifest_loaded": bool(source_manifest), "interface_loaded": bool(interface_doc.get("interface")), "adapter_count": len(adapters), + **permission_summary, "install_root_is_temp": install_root_is_temp, "failure_count": len(failures), "warning_count": len(warnings), @@ -221,6 +325,8 @@ def render_markdown(report: dict[str, Any]) -> str: f"- Manifest loaded: `{summary['manifest_loaded']}`", f"- Interface loaded: `{summary['interface_loaded']}`", f"- Adapters readable: `{summary['adapter_count']}`", + f"- Installer permissions enforced: `{summary.get('installer_permission_enforced_count', 0)}`", + f"- Installer permission failures: `{summary.get('installer_permission_failure_count', 0)}`", f"- Failures: `{summary['failure_count']}`", f"- Warnings: `{summary['warning_count']}`", "", diff --git a/scripts/skill_ir_paths.py b/scripts/skill_ir_paths.py new file mode 100644 index 00000000..b007be01 --- /dev/null +++ b/scripts/skill_ir_paths.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Shared Skill IR artifact discovery helpers.""" + +import json +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by compiler, registry, conformance, and report scripts to locate canonical Skill IR artifacts." + + +def display_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def candidate_paths(skill_dir: Path, name: str) -> list[Path]: + candidates = [ + skill_dir / "reports" / "skill-ir.json", + skill_dir / "skill-ir" / "examples" / f"{name}.json", + skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", + ] + examples_dir = skill_dir / "skill-ir" / "examples" + if examples_dir.exists(): + for path in sorted(examples_dir.glob("*.json")): + if path not in candidates: + candidates.append(path) + seen: set[Path] = set() + unique: list[Path] = [] + for path in candidates: + if path in seen: + continue + seen.add(path) + unique.append(path) + return unique + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def find_skill_ir_path(skill_dir: Path, name: str, *, require_schema: bool = False, fallback_source: str = "") -> str: + for path in candidate_paths(skill_dir, name): + payload = load_json(path) + if not payload: + continue + if require_schema and not payload.get("schema_version"): + continue + return display_path(path, skill_dir) + return fallback_source + + +def find_skill_ir( + skill_dir: Path, + name: str, + *, + require_schema: bool = False, + fallback_source: str = "", +) -> tuple[dict[str, Any], str]: + for path in candidate_paths(skill_dir, name): + payload = load_json(path) + if not payload: + continue + if require_schema and not payload.get("schema_version"): + continue + return payload, display_path(path, skill_dir) + return {}, fallback_source diff --git a/scripts/skill_os2_coverage_markdown.py b/scripts/skill_os2_coverage_markdown.py new file mode 100644 index 00000000..d8bfee12 --- /dev/null +++ b/scripts/skill_os2_coverage_markdown.py @@ -0,0 +1,101 @@ +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_skill_os2_coverage.py to keep coverage data assembly separate from Markdown rendering." + + +def render_table(items: list[dict[str, Any]]) -> list[str]: + lines = ["| Item | Status | Current | Command | Test |", "| --- | --- | --- | --- | --- |"] + for item in items: + lines.append( + "| " + + " | ".join( + [ + item["label"].replace("|", "\\|"), + f"`{item['status']}`", + item["current"].replace("|", "\\|"), + f"`{item['command']}`", + f"`{item['test']}`", + ] + ) + + " |" + ) + return lines + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Skill OS 2.0 Blueprint Coverage", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- local blueprint ready: `{str(summary['local_blueprint_ready']).lower()}`", + f"- public world-class ready: `{str(summary['public_world_class_ready']).lower()}`", + f"- pass: `{summary['pass_count']}` / `{summary['item_count']}`", + f"- missing: `{summary['missing_count']}`", + f"- warn: `{summary['warn_count']}`", + f"- reference extensions: `{summary['extension_track_count']}`", + f"- extension covered: `{summary['extension_covered_count']}`", + f"- extension partial: `{summary['extension_partial_count']}`", + f"- extension planned: `{summary['extension_planned_count']}`", + f"- adaptive extension ready: `{str(summary['adaptive_extension_ready']).lower()}`", + f"- world-class evidence pending: `{summary['world_class_evidence_pending_count']}`", + "", + "This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, commands, and tests. It does not count pending human review, provider runs, metadata fallbacks, or planned work as public world-class evidence.", + "", + "## Core Modules", + "", + *render_table(report["modules"]), + "", + "## Recommended PR Coverage", + "", + *render_table(report["recommended_prs"]), + "", + "## Reference Extension Tracks", + "", + "| Track | Status | Current | Target | Next action |", + "| --- | --- | --- | --- | --- |", + ] + for item in report["reference_extension_tracks"]: + lines.append( + "| " + + " | ".join( + [ + item["label"].replace("|", "\\|"), + f"`{item['status']}`", + item["current"].replace("|", "\\|"), + item["target"].replace("|", "\\|"), + item["next_action"].replace("|", "\\|"), + ] + ) + + " |" + ) + lines.extend( + [ + "", + "These extension tracks come from the user-supplied 2.0 reference plan. They are tracked separately from the formal Skill OS blueprint so the report can distinguish landed local architecture from planned explainer/adaptor evolution.", + "", + "## Next Highest-Leverage Moves", + "", + ] + ) + lines.extend(f"- {item}" for item in report["next_highest_leverage"]) + lines.extend(["", "## Evidence Detail", ""]) + for item in report["modules"] + report["recommended_prs"] + report["reference_extension_tracks"]: + existing = [entry["path"] for entry in item["evidence"] if entry["exists"]] + missing = [entry["path"] for entry in item["evidence"] if not entry["exists"]] + lines.append(f"### {item['label']}") + lines.append("") + lines.append(f"- objective: {item['objective']}") + lines.append(f"- status: `{item['status']}`") + lines.append(f"- existing evidence: {', '.join(f'`{path}`' for path in existing) if existing else '`none`'}") + if missing: + lines.append(f"- missing evidence: {', '.join(f'`{path}`' for path in missing)}") + lines.append(f"- next action: {item['next_action']}") + lines.append("") + return "\n".join(lines).rstrip() + "\n" diff --git a/scripts/skill_report_charts.py b/scripts/skill_report_charts.py index 69d738b5..35877961 100644 --- a/scripts/skill_report_charts.py +++ b/scripts/skill_report_charts.py @@ -2,6 +2,8 @@ import html import math +from skill_report_i18n import bi_span, en_for + SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to render inline SVG report charts." @@ -17,11 +19,26 @@ def esc(value) -> str: return html.escape(str(value)) -def figure(name: str, svg: str, caption: str) -> str: +def svg_text(zh: str, en: str | None = None, **attrs) -> str: + def attr_name(key: str) -> str: + if key.endswith("_"): + key = key[:-1] + return key.replace("_", "-") + + attr = " ".join(f'{attr_name(key)}="{esc(value)}"' for key, value in attrs.items() if value is not None) + if attr: + attr = " " + attr + return ( + f'{esc(zh)}' + f'{esc(en_for(en if en is not None else zh))}' + ) + + +def figure(name: str, svg: str, caption: str, caption_en: str | None = None) -> str: return ( f'
    ' f"{svg}" - f"
    {esc(caption)}
    " + f"
    {bi_span(caption, caption_en)}
    " "
    " ) @@ -47,12 +64,10 @@ def render_radar(scorecard: dict) -> str: data_points.append(f"{center + data_radius * math.cos(angle):.1f},{center + data_radius * math.sin(angle):.1f}") lx = center + (radius + 32) * math.cos(angle) ly = center + (radius + 32) * math.sin(angle) - label_nodes.append( - f'{esc(labels[i])}' - ) + label_nodes.append(svg_text(labels[i], x=f"{lx:.1f}", y=f"{ly:.1f}", text_anchor="middle", dominant_baseline="middle")) svg = ( '' - f'评分雷达' + + svg_text("评分雷达", x="20", y="28", class_="chart-title") + "".join(rings) + f'' + "".join(label_nodes) @@ -68,12 +83,12 @@ def render_flow(summary: dict) -> str: x = 38 + index * 210 nodes.append( f'' - f'{esc(label)}' + f'{svg_text(label, x=str(x + 75), y="99", text_anchor="middle")}' ) svg = ( '' - '交付流程' - '' + + svg_text("交付流程", x="20", y="28", class_="chart-title") + + '' + "".join(nodes) + "" ) @@ -86,15 +101,15 @@ def render_matrix(profile: dict) -> str: y = 430 - matrix.get("knowledge_density", 60) * 3.2 svg = ( '' - '能力矩阵' - f'' - f'' - f'' - '执行确定性' - '知识密度' - f'' - f'{esc(profile.get("task_family", "Skill workflow"))}' - "" + + svg_text("能力矩阵", x="20", y="30", class_="chart-title") + + f'' + + f'' + + f'' + + svg_text("执行确定性", x="260", y="424", text_anchor="middle") + + svg_text("知识密度", x="22", y="230", transform="rotate(-90 22 230)", text_anchor="middle") + + f'' + + svg_text(str(profile.get("task_family", "Skill workflow")), x=f"{x + 18:.1f}", y=f"{y + 5:.1f}") + + "" ) return figure("matrix", svg, "能力矩阵说明这个 Skill 更偏知识密集还是执行确定。") @@ -106,11 +121,11 @@ def render_layers(principle: dict) -> str: y = 55 + index * 48 blocks.append( f'' - f'{esc(layer)}' + + svg_text(layer, x="260", y=str(y + 22), text_anchor="middle") ) svg = ( '' - '分层结构' + + svg_text("分层结构", x="20", y="30", class_="chart-title") + "".join(blocks) + "" ) @@ -132,10 +147,10 @@ def render_risk_heatmap(risk: dict) -> str: ) svg = ( '' - '风险热力' + + svg_text("风险热力", x="20", y="30", class_="chart-title") + "".join(cells) - + '发生概率' - + '影响程度' + + svg_text("发生概率", x="210", y="278", text_anchor="middle") + + svg_text("影响程度", x="24", y="160", transform="rotate(-90 24 160)", text_anchor="middle") + "" ) return figure("risk_heatmap", svg, "风险热力图用影响程度和发生概率标出当前治理重点。") @@ -157,12 +172,12 @@ def render_asset_donut(assets: dict) -> str: 'pathLength="100" transform="rotate(-90 130 130)"/>' ) offset += dash - labels.append(f'{esc(item.get("label", "asset"))}') + labels.append(svg_text(str(item.get("label", "asset")), x="235", y=str(78 + index * 22))) svg = ( '' - '资产分布' + + svg_text("资产分布", x="20", y="30", class_="chart-title") + "".join(circles) - + f'{assets.get("file_count", 0)}项' + + svg_text(f'{assets.get("file_count", 0)}项', f'{assets.get("file_count", 0)} items', x="130", y="136", text_anchor="middle") + "".join(labels) + "" ) @@ -177,15 +192,18 @@ def render_timeline(roadmap: dict) -> str: title = str(item.get("title", "升级")) if len(title) > 18: title = title[:17] + "…" + title_en = en_for(title) + if len(title_en) > 24: + title_en = title_en[:23] + "..." blocks.append( f'' - f'下一步 {index + 1}' - f'{esc(title)}' + + svg_text(f"下一步 {index + 1}", f"Next {index + 1}", x=str(x), y="126", text_anchor="middle") + + svg_text(title, title_en, x=str(x), y="150", text_anchor="middle") ) svg = ( '' - '迭代时间' - f'' + + svg_text("迭代时间", x="20", y="30", class_="chart-title") + + f'' + "".join(blocks) + "" ) diff --git a/scripts/skill_report_i18n.py b/scripts/skill_report_i18n.py new file mode 100644 index 00000000..430e4f1a --- /dev/null +++ b/scripts/skill_report_i18n.py @@ -0,0 +1,386 @@ +"""Bilingual text helpers for the static skill overview report.""" + +import html +import re + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to keep bilingual report copy and fallback rules out of HTML rendering." + + +TEXT_ZH = { + "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.": "从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。", + "Understand the request.": "理解用户请求。", + "Execute the main task.": "执行核心任务。", + "Validate the result.": "校验交付结果。", + "Understand the request": "理解用户请求。", + "Execute the main task": "执行核心任务。", + "Validate the result": "校验交付结果。", + "Decide whether the request should become a skill and choose the lightest fit.": "判断请求是否应该沉淀为 Skill，并选择最轻量可靠的模式。", + "Capture job, output, exclusions, constraints, and standards.": "捕捉任务、输出、排除项、约束和质量标准。", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.": "运行参考扫描：先看外部 benchmark，再看用户材料，最后校验本地适配；只暴露不确定性或冲突。", + "Write the `description` early and test route quality before expanding the package.": "尽早写出 `description`，先测试路由质量，再扩展包体。", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.": "只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。", + "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.": "当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。", + "Turn rough requests into a compact reusable demo skill.": "把粗糙请求整理成紧凑、可复用的演示 Skill。", + "Tighten trigger and exclusions": "收紧触发与排除边界", + "Add the first execution asset": "补上第一个执行资产", + "Promote from scaffold to production-ready": "从脚手架推进到生产可用", + "Borrow one proven pattern on purpose": "有选择地借鉴一个成熟模式", + "Harden portability semantics": "加固跨环境语义", + "Create an iteration evidence loop": "建立迭代证据回路", + "补齐世界证据": "补齐世界证据", + "The package needs clearer near-neighbor exclusions before it grows.": "在继续扩展前，需要先把相邻但不应触发的场景说清楚。", + "The package is still mostly prose. Add one asset that removes repeated manual work.": "当前包体仍偏文本说明，应先增加一个能减少重复人工操作的资产。", + "The first version exists; the next gain usually comes from adding the smallest useful gates.": "第一版已经存在，下一步收益通常来自补上最小但有效的质量门禁。", + "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.": "已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。", + "The skill already signals reuse across environments, so contract clarity matters early.": "这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。", + "The package should show what changed and why after the first draft.": "第一版之后，包体应该能说明改了什么以及为什么改。", + "Add 3 to 5 should-trigger and should-not-trigger examples.": "增加 3 到 5 个应触发和不应触发的例子。", + "Refine the frontmatter description to name the recurring job and non-goals.": "精炼 frontmatter description，明确重复任务和非目标。", + "Run a first trigger evaluation pass before expanding the package.": "扩展包体前先跑一轮触发评估。", + "Move stable procedural guidance into references if users will need it repeatedly.": "如果用户会反复使用某段流程说明，把它沉淀到 references。", + "Create one deterministic helper script if a repeated step can be executed instead of described.": "如果某个重复步骤可以执行而不是描述，就沉淀成一个确定性 helper script。", + "Keep the main SKILL.md compact and route-oriented.": "保持主 SKILL.md 简洁，并围绕路由与入口组织。", + "Decide whether this skill is personal, team-reused, or library-grade.": "判断这个 Skill 是个人使用、团队复用，还是库级基础能力。", + "Add only the gates that match that risk level.": "只添加与风险等级匹配的质量门禁。", + "Record lifecycle metadata and review cadence once reuse becomes real.": "一旦进入真实复用，就记录生命周期元数据和评审节奏。", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.": "先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。", + "Record what you will not borrow so the package stays light.": "记录本轮不借鉴的内容，避免包体过重。", + "Confirm activation mode, execution context, and trust assumptions.": "确认激活模式、执行上下文和信任假设。", + "Add or review degradation strategy for non-native targets.": "补充或复核非原生目标端的降级策略。", + "Package the skill once to verify adapter expectations.": "至少打包一次 Skill，用来验证 adapter 预期。", + "Generate the HTML skill report and keep it aligned with the package.": "生成 HTML Skill 报告，并保持它与包体内容一致。", + "Record reference scan choices and non-goals.": "记录参考扫描的取舍和非目标。", + "Capture the next iteration choice explicitly before adding more files.": "在继续增加文件前，明确记录下一轮迭代选择。", + "Cleaner routing and fewer accidental activations.": "路由更清晰，误触发更少。", + "Stronger execution quality without bloating the entrypoint.": "在不膨胀入口文件的前提下提升执行质量。", + "A clearer path from exploratory package to maintained asset.": "更清晰地从探索性包体走向可维护资产。", + "A cleaner package shape with less accidental over-design.": "包体形态更清晰，也减少偶然过度设计。", + "Safer cross-environment reuse with less target drift.": "跨环境复用更安全，目标漂移更少。", + "A clearer path for the next author or reviewer.": "让下一位作者或评审者更容易接手。", + "提交有效 intake packet，并让 ledger 通过 artifact SHA-256 校验。": "提交有效 intake packet，并让 ledger 通过 artifact SHA-256 校验。", + "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", +} + +TEXT_EN = { + "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。": "Turn one-off experience into a reusable, evaluable, and portable skill package.", + "Skill 作者、复用团队和后续 reviewer。": "Skill authors, reuse teams, and later reviewers.", + "创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。": "After creation, open reports/skill-overview.html before expanding the package further.", + "触发面保持精简，并锚定在 frontmatter description。": "The trigger surface stays lean and anchored in the frontmatter description.", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。": "Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。": "Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。": "Output Eval Lab scorecard is generated to compare with-skill and baseline output quality.", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。": "Output Execution Runs is generated to distinguish recorded fixtures, command runs, and model-run evidence.", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。": "Output Review Adjudication is generated to record blind-review decisions, agreement rate, and pending cases.", + "已生成 Runtime Conformance Matrix，可审查目标端消费能力。": "Runtime Conformance Matrix is generated to review target-side consumption capability.", + "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。": "Security Trust Report is generated to review scripts, dependencies, secrets, and package-integrity risk.", + "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。": "Skill Atlas is generated to review route collisions, stale assets, and owner gaps across a skill library.", + "已生成 Registry Audit，可审查版本、owner、license、checksum 和目标兼容矩阵。": "Registry Audit is generated to review version, owner, license, checksum, and target compatibility metadata.", + "已生成 Install Simulation，可审查 zip 解压、入口加载、接口元数据和 adapter 可读性。": "Install Simulation is generated to review zip extraction, entrypoint loading, interface metadata, and adapter readability.", + "已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。": "Adoption Drift Report is generated to turn local usage feedback into next-iteration signals.", + "已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。": "Review Waivers ledger is generated to record reviewer approval, rationale, scope, and expiry for accepted warning risk.", + "已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。": "Review Annotations ledger is generated to attach reviewer notes to gates, files, and line numbers.", + "已生成 Review Studio 2.0，可在一页中查看 blocker、warning、证据路径和发布闸门。": "Review Studio 2.0 is generated to inspect blockers, warnings, evidence paths, and release gates on one page.", + "已打包 agents/interface.yaml，便于后续做跨平台适配。": "Portable interface metadata is packaged for later adapter-based export.", + "长指导被拆到 references 中，入口文件可以保持轻量。": "Extended guidance is separated into references so the entrypoint can stay compact.", + "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。": "Deterministic helper logic lives in scripts instead of hidden prompt text.", + "包内包含可随 Skill 迁移的质量门禁或触发检查。": "The package includes portable quality gates or trigger checks.", + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。": "Use this report to quickly understand the generated skill's role, principles, trigger boundary, and deliverables.", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。": "Clarify the recurring job, real input shape, and deliverable output before adding references, scripts, or evals.", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。": "If the request is still fuzzy, tighten the boundary through intent dialogue before expanding the package.", + "尚未生成盲评审定报告。": "The blind review adjudication report has not been generated yet.", + "尚未生成输出执行证据报告。": "The output execution evidence report has not been generated yet.", + "先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。": "Record the reviewer's A/B choice before opening the answer key and calculating agreement.", + "缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。": "When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.", + "recorded fixture 只能证明可复现样本，不等同于模型执行。": "A recorded fixture proves reproducible samples only; it is not model execution.", + "只有 provider runner 返回 model metadata 时才计入 model-executed。": "Only provider runners that return model metadata count as model-executed.", + "SKILL.md 已存在，是 Skill 的入口。": "SKILL.md exists and acts as the skill entrypoint.", + "README.md 已存在，便于人工阅读。": "README.md exists for human-readable usage.", + "agents/interface.yaml 已存在，便于跨平台适配。": "agents/interface.yaml exists for cross-platform adaptation.", + "manifest.json 已存在，生命周期信息可追踪。": "manifest.json exists so lifecycle metadata is traceable.", + "reports/ 已存在，生成证据可以随包体迁移。": "reports/ exists so generated evidence can travel with the package.", + "references/ 已存在，长指导可以从入口文件拆出。": "references/ exists so long guidance can stay out of the entrypoint.", + "scripts/ 已存在，确定性逻辑有位置承载。": "scripts/ exists to hold deterministic logic.", + "evals/ 已存在，触发或质量检查可以随包体迁移。": "evals/ exists so trigger or quality checks can travel with the package.", + "frontmatter description 已存在，具备基础路由面。": "The frontmatter description exists, giving the skill a basic routing surface.", + "description 有足够长度说明任务边界。": "The description is long enough to explain the task boundary.", + "description 已包含使用场景或排除边界信号。": "The description includes usage-scenario or exclusion-boundary signals.", + "description 证据不足，触发边界不稳定。": "Description evidence is insufficient, so the trigger boundary is unstable.", + "description 偏短，建议补充输入、输出或非目标。": "The description is short; add inputs, outputs, or non-goals.", + "description 缺少明确使用场景或排除边界。": "The description lacks clear usage scenarios or exclusion boundaries.", + "evals/ 已存在，可承载触发样例或质量检查。": "evals/ exists and can hold trigger examples or quality checks.", + "evals/ 证据不足，误触发检查仍偏弱。": "evals/ evidence is insufficient, so accidental-trigger checks remain weak.", + "intent-confidence 报告已生成，可辅助判断触发稳定性。": "The intent-confidence report exists and helps judge trigger stability.", + "intent-confidence 证据不足。": "intent-confidence evidence is insufficient.", + "入口文件保持克制，可维护性较好。": "The entrypoint stays restrained, which supports maintainability.", + "入口文件偏长，建议继续拆到 references/。": "The entrypoint is long; continue splitting durable guidance into references/.", + "references/ 已承载扩展指导。": "references/ carries extended guidance.", + "references/ 证据不足，长指导可能堆在入口。": "references/ evidence is insufficient; long guidance may still be crowded into the entrypoint.", + "scripts/ 已承载确定性逻辑。": "scripts/ carries deterministic logic.", + "scripts/ 证据不足，重复执行逻辑可能仍靠人工。": "scripts/ evidence is insufficient; repeated execution may still rely on manual work.", + "evals/ 已承载可迁移检查。": "evals/ carries portable checks.", + "agents/interface.yaml 已存在。": "agents/interface.yaml exists.", + "manifest.json 已存在。": "manifest.json exists.", + "目标平台或 adapter target 已声明。": "Target platforms or adapter targets are declared.", + "目标平台证据不足。": "Target-platform evidence is insufficient.", + "入口文件未发现明显私有绝对路径。": "No obvious private absolute paths were found in the entrypoint.", + "入口文件含私有绝对路径，迁移风险较高。": "The entrypoint contains private absolute paths, increasing portability risk.", + "分数越高代表上下文成本越低。": "A higher score means lower context cost.", + "上下文成本处于可控区间。": "Context cost is within a controlled range.", + "上下文成本偏高，建议压缩入口或拆分 references。": "Context cost is high; compress the entrypoint or split references further.", + "手动触发 + description 路由": "Manual activation plus description-based routing", + "跨平台": "Cross-platform", + "本地复用": "Local reuse", + "输入材料": "Input material", + "用户提供的工作流、提示词、文档、记录或散乱笔记": "User-provided workflows, prompts, documents, records, or rough notes.", + "期望沉淀的复用场景、排除项、约束和质量标准": "The reusable scenario, exclusions, constraints, and quality standards to capture.", + "Skill 包体": "Skill package", + "可路由的 SKILL.md": "A routeable SKILL.md.", + "agents/interface.yaml 元数据": "agents/interface.yaml metadata.", + "必要的 references、scripts、evals、reports 证据": "Necessary references, scripts, evals, and reports evidence.", + "可复用能力": "Reusable capability", + "入口层": "Entrypoint layer", + "参考层": "Reference layer", + "脚本层": "Script layer", + "评估层": "Evaluation layer", + "报告层": "Report layer", + "评分雷达": "Rating Radar", + "交付流程": "Delivery Flow", + "能力矩阵": "Capability Matrix", + "分层结构": "Layered Structure", + "风险热力": "Risk Heatmap", + "资产分布": "Asset Distribution", + "迭代时间": "Iteration Timeline", + "执行确定性": "Execution certainty", + "知识密度": "Knowledge density", + "发生概率": "Probability", + "影响程度": "Impact", + "评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。": "The radar chart compares completeness, trigger clarity, evidence, maintainability, and portability.", + "交付流程把用户输入、生成的包体和可复用能力放在一条线上。": "The delivery flow places user input, generated package, and reusable capability on one path.", + "能力矩阵说明这个 Skill 更偏知识密集还是执行确定。": "The capability matrix shows whether the skill leans toward knowledge density or execution certainty.", + "分层结构展示入口、参考、脚本、评估和报告如何各司其职。": "The layered structure shows how entrypoint, references, scripts, evals, and reports each carry a distinct role.", + "风险热力图用影响程度和发生概率标出当前治理重点。": "The risk heatmap marks governance priorities by impact and probability.", + "资产分布图展示当前包体的文件和目录重心。": "The asset distribution chart shows where files and directories are concentrated.", + "迭代时间线把下一步升级收束成少数可执行动作。": "The iteration timeline narrows the next upgrade into a few executable moves.", + "只需要一次性回答、没有复用价值的临时请求。": "One-off requests that do not need reusable skill behavior.", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。": "Requests to perform an adjacent task directly rather than create or use this skill.", + "缺少必要事实且用户不允许澄清的场景。": "Cases that lack required facts and do not allow clarification.", + "相邻任务需要先确认是否应转为独立 Skill。": "Adjacent tasks should first be checked for whether they need a separate skill.", + "不替代人工事实核查，也不静默扩大职责。": "Does not replace human fact checking or silently expand responsibility.", + "先改触发边界，再扩展工作流。": "Tighten trigger boundaries before expanding the workflow.", + "只把重复且稳定的步骤沉淀为脚本。": "Turn only repeated and stable steps into scripts.", + "每次升级后重新生成报告并检查分数原因。": "Regenerate the report after each upgrade and inspect score reasons.", + "先补证据和边界，再增加包体复杂度。": "Improve evidence and boundaries before adding package complexity.", + "补齐世界证据": "Close world-class evidence", + "提交有效 intake packet，并让 ledger 通过 artifact SHA-256 校验。": "Submit valid intake packets and let the ledger verify artifact SHA-256 digests.", + "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。": "Only after the ledger accepts all external and human evidence should the public world-class claim move to review.", + "缺少真实 provider 模型运行和 token metadata。": "Missing a real provider model run and token metadata.", + "盲评 pair 仍待真实 reviewer 决策。": "Blind-review pairs still need real reviewer decisions.", + "原生 runtime enforcement 仍待目标客户端或外部安装器证明。": "Native runtime enforcement still needs target-client or external-installer proof.", + "真实外部客户端 metadata-only 事件仍未导入。": "Real external-client metadata-only events have not been imported yet.", +} + +MODE_ZH = { + "scaffold": "脚手架", + "production": "生产", + "library": "库级", + "governed": "治理", + "manual": "手动", + "inline": "内联", + "agent-skills": "Agent Skills", +} + +PACKAGE_LABEL_ZH = { + "SKILL.md": "Skill 入口文件", + "README.md": "人类可读使用说明", + "agents/interface.yaml": "跨平台接口元数据", + "manifest.json": "生命周期与打包元数据", + "references": "扩展指导与复用资料", + "scripts": "确定性脚本或本地工具", + "evals": "触发与质量检查", + "reports": "生成的证据与总结报告", +} + +KIND_ZH = {"file": "文件", "folder": "目录"} + +LABEL_EN = { + "强项": "Strength", + "缺口": "Gap", + "保留并复用": "Keep", + "纳入下一轮修复": "Fix next", + "误触发风险": "Trigger risk", + "输出漂移风险": "Output drift risk", + "证据不足风险": "Evidence gap risk", + "包体膨胀风险": "Package bloat risk", + "跨平台迁移风险": "Portability risk", +} + +METRIC_LABEL_EN = { + "完整度": "Completeness", + "触发清晰": "Trigger clarity", + "证据充分": "Evidence depth", + "可维护性": "Maintainability", + "可迁移性": "Portability", + "上下文成本": "Context cost", +} + +WORLD_CLASS_LABEL_EN = { + "提供商留出": "provider holdout", + "人工盲评": "human adjudication", + "原生权限": "native permission", + "原生遥测": "native telemetry", +} + + +def contains_cjk(text: str) -> bool: + return any("\u4e00" <= char <= "\u9fff" for char in str(text)) + + +def zh_for(text: str) -> str: + value = str(text).strip() + if not value: + return "" + if value in TEXT_ZH: + return TEXT_ZH[value] + if value in TEXT_EN or contains_cjk(value): + return value + if value.startswith("Use this skill when the request matches:"): + return "当用户请求与该 Skill 的触发描述匹配时使用。" + if value.startswith("用户说出类似需求时："): + return "当用户提出与该 Skill 触发描述相近的请求时使用。" + if value.startswith("Use $") and " when you need to " in value: + skill, need = value.removeprefix("Use ").split(" when you need to ", 1) + return f"当你需要{zh_for(need).rstrip('。')}时使用 `{skill}`。" + if value.startswith("Read the strongest pattern from "): + repo = value.removeprefix("Read the strongest pattern from ").rstrip(".") + return f"阅读 `{repo}` 中最值得借鉴的模式。" + if value.startswith("Primary prompt task family:"): + return "主要提示任务类型已记录在 prompt quality profile 中。" + if value.startswith("Complexity:"): + return "复杂度判断已记录在 prompt quality profile 中。" + if value.startswith("Stability:"): + return "系统稳定性评分已记录在 system model 中。" + if value.startswith("Owned job:"): + return "负责的核心任务已在 system model 中说明。" + if value.startswith("Leverage:"): + return "关键杠杆点已在 system model 中说明。" + return "原始说明可切换到英文查看；默认中文报告保留结论与结构说明。" + + +def en_for(text: str) -> str: + value = str(text).strip() + if not value: + return "" + if value in TEXT_EN: + return TEXT_EN[value] + if value in METRIC_LABEL_EN: + return METRIC_LABEL_EN[value] + if value.startswith("创建完成后建议先打开 ") and value.endswith("，再继续扩展包体。"): + path = value.removeprefix("创建完成后建议先打开 ").removesuffix("，再继续扩展包体。") + return f"After creation, open {path} before expanding the package further." + if value.startswith("交付结果："): + return "Deliverables: " + value.removeprefix("交付结果：") + if value.startswith("能力类型："): + return "Capability type: " + value.removeprefix("能力类型：") + if value.startswith("成熟度："): + return "Maturity: " + value.removeprefix("成熟度：") + if value.startswith("触发强度："): + return "Trigger strength: " + en_for(value.removeprefix("触发强度：")) + if value.startswith("复用范围："): + return "Reuse scope: " + en_for(value.removeprefix("复用范围：")) + if value.startswith("评审进度："): + return "Review progress: " + value.removeprefix("评审进度：") + if value.startswith("待评审："): + return "Pending review: " + value.removeprefix("待评审：") + if value.startswith("一致率："): + tail = value.removeprefix("一致率：") + return "Agreement rate: " + ("not available yet" if tail == "暂无" else tail) + if value.startswith("非法决策："): + return "Invalid decisions: " + value.removeprefix("非法决策：") + if value.startswith("变体运行："): + return "Variant runs: " + value.removeprefix("变体运行：") + if value.startswith("模型执行："): + return "Model executions: " + value.removeprefix("模型执行：") + if value.startswith("记录样本："): + return "Recorded fixtures: " + value.removeprefix("记录样本：") + if value.startswith("Token 估算："): + return "Token estimates: " + value.removeprefix("Token 估算：") + match = re.match(r"^世界级证据仍有\s+(\d+)\s+项待补；公开完成态 claim 必须继续保持阻塞。$", value) + if match: + return f"World-class evidence still has {match.group(1)} pending item(s); public completion claims must stay blocked." + match = re.match(r"^补齐(.+?)证据：(.+)$", value) + if match: + label = WORLD_CLASS_LABEL_EN.get(match.group(1), match.group(1)) + return f"Close {label} evidence: {en_for(match.group(2))}" + match = re.match(r"^继续补齐剩余\s+(\d+)\s+项外部/人工证据，并保持 claim guard 为 pending 状态。$", value) + if match: + return f"Close the remaining {match.group(1)} external or human evidence item(s) and keep the claim guard pending." + match = re.match( + r"^按 ledger 总量继续补齐\s+(\d+)\s+项待补证据（外部\s+(\d+)\s+项、人工\s+(\d+)\s+项；未展开\s+(\d+)\s+项），并保持 claim guard 为 pending 状态。$", + value, + ) + if match: + return ( + f"Close all {match.group(1)} pending evidence item(s) in the ledger " + f"(external {match.group(2)}, human {match.group(3)}; {match.group(4)} not shown here) " + "and keep the claim guard pending." + ) + match = re.match(r"^已生成\s+(\d+)\s+/\s+(\d+)\s+类报告证据。$", value) + if match: + return f"Generated {match.group(1)} / {match.group(2)} evidence report types." + match = re.match(r"^(.+?)\s+已存在。$", value) + if match: + return f"{match.group(1)} exists." + match = re.match(r"^(.+?)\s+未发现或为空，完整度扣分。$", value) + if match: + return f"{match.group(1)} was not found or is empty, reducing completeness." + match = re.match(r"^(.+?)\s+证据不足。$", value) + if match: + return f"{match.group(1)} evidence is insufficient." + match = re.match(r"^SKILL\.md 约\s+(.+?)\s+个词/字。$", value) + if match: + return f"SKILL.md is about {match.group(1)} words/characters." + match = re.match(r"^入口约\s+(.+?)\s+个词/字，references 约\s+(.+?)\s+个词/字。$", value) + if match: + return f"Entrypoint is about {match.group(1)} words/characters; references are about {match.group(2)}." + match = re.match(r"^结构化 Skill 目录，共\s+(.+?)\s+类关键资产。$", value) + if match: + return f"Structured skill directory with {match.group(1)} key asset groups." + if value.startswith("证据不足：缺少 "): + return "Evidence gap: missing " + value.removeprefix("证据不足：缺少 ").rstrip("。") + "." + for metric_label in METRIC_LABEL_EN: + prefix = metric_label + "需要补强：" + if value.startswith(prefix): + return f"{METRIC_LABEL_EN[metric_label]} needs improvement: {en_for(value.removeprefix(prefix))}" + if value.startswith("Use this skill when the request matches:"): + return "Use this skill when the request matches the frontmatter description." + if value.startswith("用户说出类似需求时："): + return "Use this skill when the user asks for a matching scenario." + if value.startswith("当你需要") and "时使用" in value: + return "Use this skill when the request matches its stated scenario." + if contains_cjk(value): + return "Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording." + return value + + +def bi_span(zh: str, en: str | None = None) -> str: + english = en_for(en) if en is not None else en_for(zh) + return ( + f'{html.escape(str(zh))}' + f'{html.escape(str(english))}' + ) + + +def bi_item(text: str) -> str: + return bi_span(zh_for(text), en_for(text)) + + +def mode_zh(value: str) -> str: + return MODE_ZH.get(str(value), str(value)) + + +def readable_description_zh(description: str) -> str: + if contains_cjk(description): + return description + return "该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。" diff --git a/scripts/skill_report_layout.py b/scripts/skill_report_layout.py index 9f603208..cfcf03ca 100644 --- a/scripts/skill_report_layout.py +++ b/scripts/skill_report_layout.py @@ -2,6 +2,7 @@ """Static layout contract for skill overview report HTML.""" import html +from pathlib import Path from typing import Any @@ -9,6 +10,14 @@ SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to keep overview report layout and CSS out of data rendering." +ROOT = Path(__file__).resolve().parent.parent +ASSET_DIR = ROOT / "assets" + + +def read_layout_asset(filename: str) -> str: + return (ASSET_DIR / filename).read_text(encoding="utf-8").strip() + + def bi_span(zh: str, en: str) -> str: return ( f'{html.escape(str(zh))}' @@ -33,621 +42,8 @@ def render_language_switch() -> str: def skill_overview_css() -> str: - return """ - :root { - --paper: #ffffff; - --wash: #f8fafc; - --wash-strong: #f2f5f8; - --line: #e6e0d4; - --line-soft: #eee9df; - --brand: #1B365D; - --brand-soft: #EEF3F8; - --brand-mid: #315982; - --ink: #151515; - --text: #2f2d29; - --muted: #68625a; - --faint: #8b857b; - --success: #2f6f5e; - --warn: #8a5a19; - --serif: "TsangerJinKai02", "Source Han Serif SC", "Noto Serif CJK SC", "Songti SC", "STSong", Charter, Georgia, serif; - --mono: "JetBrains Mono", "SF Mono", ui-monospace, Menlo, Consolas, monospace; - --shadow-soft: 0 1px 2px rgba(27, 54, 93, 0.06), 0 16px 44px rgba(27, 54, 93, 0.08); - } - * { box-sizing: border-box; } - html { scroll-behavior: smooth; } - body { - margin: 0; - background: #ffffff; - color: var(--ink); - font-family: var(--serif); - line-height: 1.62; - letter-spacing: 0; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; - text-rendering: optimizeLegibility; - } - body[data-report-lang="zh-CN"] [data-lang="en"], - body[data-report-lang="en"] [data-lang="zh-CN"] { display: none !important; } - .skip-link { - position: fixed; - left: 18px; - top: 10px; - z-index: 40; - transform: translateY(-140%); - padding: 8px 12px; - border-radius: 8px; - background: var(--brand); - color: #fff; - text-decoration: none; - transition: transform 160ms cubic-bezier(0.16, 1, 0.3, 1); - } - .skip-link:focus-visible { transform: translateY(0); outline: 2px solid var(--brand-mid); outline-offset: 2px; } - .topbar { - position: sticky; - top: 0; - z-index: 20; - background: rgba(255, 255, 255, 0.96); - border-bottom: 1px solid var(--line); - } - .progress-track { - height: 2px; - background: transparent; - } - .progress-bar { - display: block; - width: 100%; - height: 100%; - background: var(--brand); - transition: transform 160ms cubic-bezier(0.16, 1, 0.3, 1); - transform-origin: left center; - transform: scaleX(0); - } - .topbar-inner { - max-width: 1240px; - margin: 0 auto; - padding: 9px 28px; - display: grid; - grid-template-columns: minmax(0, 1fr) auto; - gap: 18px; - align-items: center; - } - .nav-shell { - display: grid; - grid-template-columns: auto minmax(0, 1fr); - gap: 18px; - align-items: center; - min-width: 0; - } - .report-mark { - color: var(--brand); - font-family: var(--mono); - font-size: 11px; - line-height: 1; - text-transform: uppercase; - white-space: nowrap; - } - .report-nav { - display: flex; - gap: 4px; - overflow-x: auto; - scrollbar-width: none; - min-width: 0; - } - .report-nav::-webkit-scrollbar { display: none; } - .report-nav a { - flex: 0 0 auto; - min-width: 76px; - min-height: 40px; - display: inline-flex; - align-items: center; - justify-content: center; - padding: 0 10px; - color: var(--brand); - text-decoration: none; - text-align: center; - font-size: 13px; - border-radius: 8px; - white-space: nowrap; - transition-property: background-color, color, transform; - transition-duration: 160ms; - transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1); - } - @media (hover: hover) { - .report-nav a:hover { background: var(--brand-soft); } - } - .report-nav a:focus-visible { - outline: 2px solid var(--brand-mid); - outline-offset: 2px; - background: var(--brand-soft); - } - .report-nav a[aria-current="true"] { - background: var(--brand); - color: #ffffff; - } - .report-nav a[aria-current="true"] span { color: #ffffff; } - .language-switch { - display: inline-flex; - gap: 3px; - padding: 3px; - border: 1px solid var(--line); - border-radius: 8px; - background: var(--paper); - box-shadow: 0 1px 2px rgba(27, 54, 93, 0.05); - } - .language-switch button { - appearance: none; - min-width: 42px; - min-height: 34px; - border: 0; - border-radius: 6px; - padding: 0 10px; - background: transparent; - color: var(--muted); - font: inherit; - font-size: 13px; - cursor: pointer; - transition-property: background-color, color, transform; - transition-duration: 160ms; - transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1); - } - .language-switch button:active { transform: scale(0.96); } - .language-switch button:focus-visible { outline: 2px solid var(--brand-mid); outline-offset: 2px; } - .language-switch button[aria-pressed="true"] { - background: var(--brand-soft); - color: var(--brand); - } - .wrap { - max-width: 1240px; - margin: 0 auto; - padding: 58px 28px 92px; - } - .hero { - padding: 16px 0 34px; - } - .hero-grid { - display: grid; - grid-template-columns: minmax(0, 1fr) 360px; - gap: 44px; - align-items: end; - } - .eyebrow, .report-mark { - letter-spacing: 0; - } - .eyebrow { - margin: 0 0 12px; - color: var(--brand); - font-family: var(--mono); - font-size: 12px; - text-transform: uppercase; - } - .slug { - margin: 0 0 18px; - color: var(--faint); - font-family: var(--mono); - font-size: 13px; - overflow-wrap: anywhere; - } - h1, h2, h3 { - margin: 0; - font-weight: 500; - letter-spacing: 0; - text-wrap: balance; - } - h1 { - max-width: 8em; - color: var(--ink); - font-size: 4rem; - line-height: 1.02; - } - .lead { - max-width: 760px; - margin: 20px 0 0; - color: var(--text); - font-size: 1.08rem; - text-wrap: pretty; - } - .hero-meta, .badges { - display: flex; - flex-wrap: wrap; - gap: 8px; - margin-top: 20px; - } - .hero-meta span, .badges span, .tag { - display: inline-flex; - align-items: center; - min-height: 30px; - padding: 5px 10px; - border-radius: 6px; - background: var(--brand-soft); - color: var(--brand); - font-size: 13px; - font-variant-numeric: tabular-nums; - } - .hero-card { - position: relative; - padding: 22px 22px 24px; - border-radius: 8px; - background: linear-gradient(180deg, #ffffff 0%, #fbfaf7 100%); - box-shadow: var(--shadow-soft); - border: 1px solid var(--line-soft); - } - .hero-card::before { - content: ""; - position: absolute; - top: 18px; - right: 18px; - width: 38px; - height: 2px; - background: var(--brand); - } - .score-strip { - display: grid; - grid-template-columns: repeat(4, minmax(0, 1fr)); - gap: 10px; - margin-top: 32px; - padding-top: 20px; - border-top: 1px solid var(--line); - } - .score-chip { - padding: 14px 14px 12px; - border-radius: 8px; - background: var(--wash); - border: 1px solid var(--line-soft); - } - .score-chip span { - display: block; - color: var(--muted); - font-size: 12px; - } - .score-chip strong { - display: block; - margin: 5px 0 8px; - color: var(--brand); - font-family: var(--mono); - font-size: 1.65rem; - font-weight: 500; - line-height: 1; - font-variant-numeric: tabular-nums; - } - .score-chip i { - display: block; - height: 3px; - border-radius: 999px; - background: linear-gradient(90deg, var(--brand) var(--score), #dfe6ee var(--score)); - } - .score-chip small { - display: block; - margin-top: 8px; - color: var(--muted); - font-size: 12px; - line-height: 1.45; - } - section { - scroll-margin-top: 78px; - padding-top: 44px; - margin-top: 44px; - border-top: 1px solid var(--line); - } - section.hero { - scroll-margin-top: 0; - padding-top: 16px; - margin-top: 0; - border-top: 0; - } - .section-head { - display: grid; - grid-template-columns: minmax(0, 246px) minmax(0, 1fr); - gap: 36px; - align-items: start; - } - h2 { - color: var(--ink); - font-size: 1.5rem; - line-height: 1.18; - } - h2::before { - content: ""; - display: block; - width: 32px; - height: 2px; - margin-bottom: 12px; - background: var(--brand); - } - h3 { - color: var(--ink); - font-size: 1.02rem; - line-height: 1.3; - margin-bottom: 10px; - } - .section-head p { - margin: 12px 0 0; - color: var(--muted); - max-width: 43ch; - text-wrap: pretty; - } - .two-col, .metric-grid, .chart-grid { - display: grid; - grid-template-columns: repeat(2, minmax(0, 1fr)); - gap: 16px; - align-items: stretch; - } - .quality-panels { - margin-top: 16px; - } - .metrics-stack { - display: grid; - gap: 18px; - } - .metrics-lead { - display: grid; - grid-template-columns: minmax(420px, 1.15fr) minmax(300px, 0.85fr); - gap: 18px; - align-items: stretch; - } - .metrics-note { - display: grid; - align-content: start; - gap: 16px; - } - .metrics-note p { - margin: 0; - max-width: none; - color: var(--text); - } - .metric-summary-list { - margin: 0; - padding: 0; - list-style: none; - display: grid; - gap: 10px; - } - .metric-summary-list li { - display: grid; - grid-template-columns: auto minmax(0, 1fr) auto; - gap: 8px 10px; - align-items: baseline; - padding: 10px 0; - border-bottom: 1px solid var(--line-soft); - } - .metric-summary-list li:last-child { border-bottom: 0; } - .metric-summary-list b { - color: var(--ink); - font-weight: 500; - } - .metric-summary-list em { - color: var(--brand); - font-family: var(--mono); - font-style: normal; - font-variant-numeric: tabular-nums; - } - .metric-summary-list small { - grid-column: 2 / 4; - color: var(--muted); - line-height: 1.5; - } - .metric-status { - display: inline-flex; - align-items: center; - min-height: 24px; - padding: 2px 7px; - border-radius: 6px; - background: var(--brand-soft); - color: var(--brand); - font-size: 12px; - } - .metric-grid { - grid-template-columns: repeat(2, minmax(0, 1fr)); - margin-top: 2px; - } - .list, .compact-list, .step-list { - margin: 0; - padding-left: 1.15em; - display: grid; - gap: 8px; - } - .list li::marker, .compact-list li::marker, .step-list li::marker { color: var(--brand); } - .compact-list { - gap: 6px; - font-size: 13px; - color: var(--muted); - } - .panel, .metric-card, .roadmap-item { - background: #ffffff; - border: 1px solid var(--line-soft); - border-radius: 8px; - padding: 22px; - box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04); - } - .metric-card { - display: grid; - grid-template-columns: 92px minmax(0, 1fr); - gap: 18px; - align-content: start; - min-height: 0; - } - .metric-card-head { - display: grid; - align-content: start; - } - .metric-card strong { - display: block; - margin: 8px 0 10px; - color: var(--brand); - font-family: var(--mono); - font-size: 2rem; - font-weight: 500; - line-height: 1; - font-variant-numeric: tabular-nums; - } - .metric-label { - color: var(--muted); - font-size: 13px; - } - .metric-card-body { - min-width: 0; - } - .chart-figure { - margin: 0; - padding: 18px; - border: 1px solid var(--line-soft); - border-radius: 8px; - background: #ffffff; - box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04); - } - .chart-figure svg { - width: 100%; - height: auto; - display: block; - overflow: visible; - } - .chart-figure text { - fill: var(--ink); - font-family: var(--serif); - font-size: 13px; - } - .chart-title { - fill: var(--brand); - font-size: 16px; - font-weight: 500; - } - .chart-line { - fill: none; - stroke: var(--brand); - stroke-width: 2; - } - figcaption { - margin-top: 12px; - padding-top: 10px; - border-top: 1px solid var(--line-soft); - color: var(--muted); - font-size: 13px; - line-height: 1.5; - text-wrap: pretty; - } - table { - width: 100%; - border-collapse: collapse; - font-size: 14px; - } - th, td { - padding: 12px 10px; - text-align: left; - border-bottom: 1px solid var(--line-soft); - vertical-align: top; - } - th { - color: var(--brand); - font-weight: 500; - font-size: 12px; - } - td:first-child, th:first-child { width: 96px; } - .roadmap { - display: grid; - grid-template-columns: repeat(3, minmax(0, 1fr)); - gap: 16px; - } - .step { - display: inline-flex; - margin-bottom: 12px; - min-height: 28px; - align-items: center; - padding: 3px 8px; - border-radius: 6px; - background: var(--brand-soft); - color: var(--brand); - font-size: 12px; - } - .unlock { - margin-top: 14px; - color: var(--muted); - font-size: 13px; - } - @media (max-width: 980px) { - .topbar-inner { - grid-template-columns: minmax(0, 1fr) auto; - padding: 8px 16px; - } - .nav-shell { grid-template-columns: 1fr; gap: 6px; } - .report-mark { display: none; } - .report-nav a { min-width: 72px; } - .hero-grid, .section-head { grid-template-columns: 1fr; } - .hero-grid { gap: 24px; } - .score-strip { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .metrics-lead, .two-col, .metric-grid, .chart-grid, .roadmap { grid-template-columns: 1fr; } - .metric-card { grid-template-columns: 86px minmax(0, 1fr); } - h1 { font-size: 2.75rem; } - } - @media (max-width: 540px) { - .wrap { padding: 34px 16px 72px; } - .topbar-inner { gap: 10px; } - .report-nav { - display: flex; - gap: 2px; - } - .report-nav a { - min-width: 66px; - min-height: 38px; - padding: 0 8px; - font-size: 12px; - } - .language-switch button { min-width: 38px; min-height: 32px; padding: 0 8px; } - .hero { padding-top: 8px; } - h1 { max-width: 8em; font-size: 2.35rem; } - .lead { font-size: 1rem; } - .hero-meta span, .badges span { font-size: 12px; } - .score-strip { grid-template-columns: 1fr; } - section { margin-top: 34px; padding-top: 34px; } - .panel, .metric-card, .roadmap-item, .chart-figure, .hero-card { padding: 18px; } - .metric-card { grid-template-columns: 1fr; gap: 10px; } - .metric-card strong { margin-bottom: 2px; } - .metric-summary-list li { grid-template-columns: auto minmax(0, 1fr) auto; } - table { font-size: 13px; } - th, td { padding: 10px 7px; } - } - @media (prefers-reduced-motion: reduce) { - html { scroll-behavior: auto; } - *, *::before, *::after { transition-duration: 0.01ms !important; } - } - """.strip() + return read_layout_asset("skill-overview.css") def skill_overview_script() -> str: - return """ - (function () { - var buttons = Array.prototype.slice.call(document.querySelectorAll("[data-set-lang]")); - var navLinks = Array.prototype.slice.call(document.querySelectorAll(".report-nav a")); - var sections = navLinks - .map(function (link) { return document.querySelector(link.getAttribute("href")); }) - .filter(Boolean); - var progressBar = document.querySelector(".progress-bar"); - function setLanguage(lang) { - document.body.setAttribute("data-report-lang", lang); - document.documentElement.setAttribute("lang", lang); - buttons.forEach(function (button) { - button.setAttribute("aria-pressed", button.getAttribute("data-set-lang") === lang ? "true" : "false"); - }); - } - function updateProgress() { - var scrollTop = window.scrollY || document.documentElement.scrollTop; - var height = Math.max(1, document.documentElement.scrollHeight - window.innerHeight); - var pct = Math.min(100, Math.max(0, (scrollTop / height) * 100)); - if (progressBar) progressBar.style.transform = "scaleX(" + pct / 100 + ")"; - var active = sections[0]; - sections.forEach(function (section) { - if (section.getBoundingClientRect().top <= 110) active = section; - }); - navLinks.forEach(function (link) { - link.setAttribute("aria-current", link.getAttribute("href") === "#" + active.id ? "true" : "false"); - }); - } - buttons.forEach(function (button) { - button.addEventListener("click", function () { - setLanguage(button.getAttribute("data-set-lang")); - }); - }); - window.addEventListener("scroll", updateProgress, { passive: true }); - window.addEventListener("resize", updateProgress); - setLanguage("zh-CN"); - updateProgress(); - })(); - """.strip() + return read_layout_asset("skill-overview.js") diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index 5e9b86d0..a269f3dd 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -1,33 +1,26 @@ #!/usr/bin/env python3 -import json -import re from datetime import date from pathlib import Path +from skill_ir_paths import find_skill_ir from skill_report_metrics import calculate_scorecard - -try: - import yaml -except ImportError: # pragma: no cover - yaml = None +from skill_report_sources import ( + extract_title, + load_json, + load_yaml, + package_entries, + parse_frontmatter, + parse_sections, + summarize_logic, + summarize_usage, +) +from skill_report_sections import package_assets, quality_review, risk_governance +from skill_report_world_class import world_class_readiness, world_class_roadmap_item SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to build the v2 report data model." -KNOWN_ENTRIES = [ - ("SKILL.md", "Skill entrypoint"), - ("README.md", "Human-readable usage guide"), - ("agents/interface.yaml", "Neutral interface metadata"), - ("manifest.json", "Lifecycle and portability metadata"), - ("references", "Extended guidance and reusable notes"), - ("scripts", "Deterministic helpers or local tooling"), - ("evals", "Trigger and quality checks"), - ("reports", "Generated evidence and overview artifacts"), -] - -IGNORED_PACKAGE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"} - REPORT_NAV_V2 = [ {"label": "技能概述", "label_en": "Overview", "href": "overview"}, {"label": "总览指标", "label_en": "Metrics", "href": "metrics"}, @@ -41,131 +34,6 @@ REPORT_NAV_V2 = [ ] -def parse_frontmatter(text: str) -> tuple[dict, str]: - lines = text.splitlines() - if not lines or lines[0].strip() != "---": - return {}, text - try: - end_index = lines[1:].index("---") + 1 - except ValueError: - return {}, text - frontmatter_text = "\n".join(lines[1:end_index]) - body = "\n".join(lines[end_index + 1 :]).lstrip() - if yaml is not None: - data = yaml.safe_load(frontmatter_text) or {} - return data if isinstance(data, dict) else {}, body - data = {} - for line in frontmatter_text.splitlines(): - if ":" not in line: - continue - key, value = line.split(":", 1) - data[key.strip()] = value.strip().strip('"') - return data, body - - -def load_yaml(path: Path) -> dict: - if not path.exists(): - return {} - text = path.read_text(encoding="utf-8") - if yaml is not None: - payload = yaml.safe_load(text) or {} - return payload if isinstance(payload, dict) else {} - return {} - - -def load_json(path: Path) -> dict: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - return payload if isinstance(payload, dict) else {} - - -def extract_title(body: str, fallback: str) -> str: - for line in body.splitlines(): - if line.startswith("# "): - return line[2:].strip() - return fallback - - -def parse_sections(body: str) -> dict[str, str]: - sections: dict[str, list[str]] = {} - current = "_preamble" - sections[current] = [] - for line in body.splitlines(): - if line.startswith("## "): - current = line[3:].strip() - sections[current] = [] - continue - sections[current].append(line) - return {name: "\n".join(lines).strip() for name, lines in sections.items()} - - -def extract_list_items(text: str) -> list[str]: - items = [] - for line in text.splitlines(): - stripped = line.strip() - if not stripped: - continue - ordered = re.match(r"^\d+\.\s+(.*)$", stripped) - bullet = re.match(r"^[-*]\s+(.*)$", stripped) - match = ordered or bullet - if match: - items.append(match.group(1).strip()) - return items - - -def summarize_logic(sections: dict[str, str]) -> list[str]: - for key in ("Compact Workflow", "Workflow", "How It Works", "Logic", "Quick Start"): - if key in sections: - items = extract_list_items(sections[key]) - if items: - return items[:5] - return extract_list_items(sections.get("_preamble", ""))[:5] or [ - "Understand the request", - "Execute the main task", - "Validate the result", - ] - - -def summarize_usage(sections: dict[str, str], default_prompt: str, description: str) -> list[str]: - for key in ("How To Use", "Quick Start", "Usage", "Runbook"): - if key in sections: - items = extract_list_items(sections[key]) - if items: - return items[:5] - usage = [] - if default_prompt: - usage.append(default_prompt) - usage.append(f"Use this skill when the request matches: {description}") - return usage[:5] - - -def package_entries(skill_dir: Path) -> list[dict]: - items = [] - for rel_path, label in KNOWN_ENTRIES: - target = skill_dir / rel_path - if target.exists(): - kind = "folder" if target.is_dir() else "file" - if target.is_dir(): - count = len( - [ - path - for path in target.rglob("*") - if path.is_file() - and not path.is_symlink() - and not any(part in IGNORED_PACKAGE_PARTS for part in path.relative_to(target).parts) - and path.suffix not in {".pyc", ".pyo"} - ] - ) - else: - count = 1 - items.append({"path": rel_path, "label": label, "kind": kind, "file_count": count}) - return items - - def context_payload(intent: dict) -> dict: context = intent.get("context", {}) if isinstance(intent, dict) else {} return context if isinstance(context, dict) else {} @@ -307,9 +175,12 @@ def principle_nodes(system_model: dict) -> list[dict]: ] -def roadmap_items(iteration: dict) -> list[dict]: +def roadmap_items(iteration: dict, readiness: dict | None = None) -> list[dict]: directions = iteration.get("directions", []) if isinstance(iteration, dict) else [] items = [] + evidence_item = world_class_roadmap_item(readiness or {}) + if evidence_item: + items.append(evidence_item) for item in directions[:3]: items.append( { @@ -319,6 +190,8 @@ def roadmap_items(iteration: dict) -> list[dict]: "unlocks": item.get("unlocks", ""), } ) + if len(items) >= 3: + break if items: return items return [ @@ -331,42 +204,6 @@ def roadmap_items(iteration: dict) -> list[dict]: ] -def artifact_design_highlights(profile: dict) -> list[str]: - primary = profile.get("primary_artifact", {}) - highlights = [] - if primary.get("direction"): - highlights.append(primary["direction"]) - highlights.extend(profile.get("quality_gates", [])[:3]) - return highlights[:4] - - -def prompt_quality_highlights(profile: dict) -> list[str]: - highlights = [] - primary = profile.get("primary_task_family", {}) - complexity = profile.get("complexity", {}) - if primary.get("label"): - highlights.append(f"Primary prompt task family: {primary['label']}.") - if complexity.get("band"): - highlights.append(f"Complexity: {complexity['band']} — {complexity.get('reason', '')}") - for item in profile.get("quality_matrix", [])[:2]: - highlights.append(f"{item.get('label', 'Quality')}: {item.get('score', 'n/a')}/100.") - return highlights[:4] - - -def system_model_highlights(model: dict) -> list[str]: - highlights = [] - stability = model.get("stability", {}) - if stability: - highlights.append(f"Stability: {stability.get('band', 'unknown')} ({stability.get('score', 'n/a')}/100).") - boundary = model.get("boundary_map", {}) - if boundary.get("owned_job"): - highlights.append(f"Owned job: {boundary['owned_job']}") - for point in model.get("leverage_points", [])[:2]: - if point.get("point"): - highlights.append(f"Leverage: {point['point']} — {point.get('move', '')}") - return highlights[:4] - - def capability_profile(manifest: dict, interface_data: dict, prompt_quality: dict) -> dict: maturity = manifest.get("maturity_tier", "scaffold") task_family = prompt_quality.get("primary_task_family", {}).get("label", "Skill workflow") @@ -384,82 +221,6 @@ def capability_profile(manifest: dict, interface_data: dict, prompt_quality: dic } -def risk_governance(output_risk: dict, system_model: dict, scorecard: dict) -> dict: - risk_names = [ - ("误触发风险", "trigger_score"), - ("输出漂移风险", "evidence_score"), - ("证据不足风险", "evidence_score"), - ("包体膨胀风险", "maintainability_score"), - ("跨平台迁移风险", "portability_score"), - ] - risks = [] - for index, (name, metric_key) in enumerate(risk_names): - score = scorecard.get(metric_key, {}).get("score", 50) - probability = max(1, min(3, 4 - round(score / 34))) - impact = 3 if index in {0, 2, 4} else 2 - risks.append( - { - "name": name, - "impact": impact, - "probability": probability, - "signal": scorecard.get(metric_key, {}).get("reasons", ["证据不足"])[0], - "response": "先补证据和边界，再增加包体复杂度。", - } - ) - human_boundary = system_model.get("boundary_map", {}).get("human_judgment_boundary", []) - return { - "risks": risks, - "risk_families": output_risk.get("risk_families", []), - "human_judgment_boundary": human_boundary, - } - - -def quality_review( - strengths: list[str], - scorecard: dict, - artifact_design: dict, - prompt_quality: dict, - system_model: dict, -) -> dict: - gaps = [] - for key, payload in scorecard.items(): - if payload.get("score", 0) < 70: - gaps.append(f"{payload.get('label', key)}需要补强：{payload.get('reasons', ['证据不足'])[0]}") - return { - "strengths": strengths, - "gaps": gaps or ["当前关键证据较完整，优先保持轻量。"], - "recommendations": [ - "先改触发边界，再扩展工作流。", - "只把重复且稳定的步骤沉淀为脚本。", - "每次升级后重新生成报告并检查分数原因。", - ], - "artifact_design": { - "design_system": artifact_design.get("design_system", "content-led editorial"), - "highlights": artifact_design_highlights(artifact_design), - }, - "prompt_quality": { - "overall_quality_score": prompt_quality.get("overall_quality_score", "n/a"), - "highlights": prompt_quality_highlights(prompt_quality), - }, - "system_model": { - "stability": system_model.get("stability", {}), - "highlights": system_model_highlights(system_model), - }, - } - - -def package_assets(package_map: list[dict]) -> dict: - files = sum(item.get("file_count", 0) for item in package_map if item.get("kind") == "file") - folders = [item for item in package_map if item.get("kind") == "folder"] - distribution = [{"label": item["path"], "value": max(1, item.get("file_count", 1))} for item in package_map] - return { - "entries": package_map, - "file_count": files + sum(item.get("file_count", 0) for item in folders), - "folder_count": len(folders), - "distribution": distribution, - } - - def build_report_model(skill_dir: Path) -> dict: skill_dir = skill_dir.resolve() skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") @@ -475,7 +236,9 @@ def build_report_model(skill_dir: Path) -> dict: output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json") output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json") output_blind_review = load_json(skill_dir / "reports" / "output_blind_review_pack.json") + output_review_kit = load_json(skill_dir / "reports" / "output_review_kit.json") output_review_adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json") + benchmark_reproducibility = load_json(skill_dir / "reports" / "benchmark_reproducibility.json") conformance = load_json(skill_dir / "reports" / "conformance_matrix.json") runtime_permissions = load_json(skill_dir / "reports" / "runtime_permission_probes.json") trust_report = load_json(skill_dir / "reports" / "security_trust_report.json") @@ -487,15 +250,14 @@ def build_report_model(skill_dir: Path) -> dict: adoption_drift = load_json(skill_dir / "reports" / "adoption_drift_report.json") review_waivers = load_json(skill_dir / "reports" / "review_waivers.json") review_annotations = load_json(skill_dir / "reports" / "review_annotations.json") + world_class_evidence = load_json(skill_dir / "reports" / "world_class_evidence_plan.json") + world_class_evidence_ledger = load_json(skill_dir / "reports" / "world_class_evidence_ledger.json") compiled_targets = load_json(skill_dir / "reports" / "compiled_targets.json") - skill_ir = load_json(skill_dir / "reports" / "skill-ir.json") - if not skill_ir: - example_ir = skill_dir / "skill-ir" / "examples" / f"{frontmatter.get('name', skill_dir.name)}.json" - skill_ir = load_json(example_ir) reference_synthesis = load_json(skill_dir / "reports" / "reference-synthesis.json") iteration = load_json(skill_dir / "reports" / "iteration-directions.json") name = frontmatter.get("name", skill_dir.name) + skill_ir, skill_ir_path = find_skill_ir(skill_dir, name) description = frontmatter.get("description", "No description found.") title = extract_title(body, name.replace("-", " ").title()) display_name = interface_data.get("interface", {}).get("display_name", title) @@ -508,7 +270,8 @@ def build_report_model(skill_dir: Path) -> dict: trigger = trigger_contract(interface_data, description) io = io_contract(intent, package_map, description) principles = principle_nodes(system_model) - roadmap = roadmap_items(iteration) + readiness = world_class_readiness(world_class_evidence_ledger) + roadmap = roadmap_items(iteration, readiness) metadata = { "canonical_format": interface_data.get("compatibility", {}).get("canonical_format", "agent-skills"), "targets": interface_data.get("compatibility", {}).get("adapter_targets", []), @@ -519,7 +282,7 @@ def build_report_model(skill_dir: Path) -> dict: deliverables = [ "SKILL.md", "agents/interface.yaml", - "reports/skill-ir.json", + skill_ir_path or "reports/skill-ir.json", "reports/compiled_targets.md", "reports/output_quality_scorecard.md", "reports/conformance_matrix.md", @@ -533,6 +296,7 @@ def build_report_model(skill_dir: Path) -> dict: "reports/review_waivers.md", "reports/review_annotations.md", "reports/review-studio.html", + "reports/skill-interpretation.html", "reports/skill-overview.html", ] if (skill_dir / "reports" / "runtime_permission_probes.md").exists(): @@ -547,9 +311,25 @@ def build_report_model(skill_dir: Path) -> dict: if (skill_dir / "reports" / "output_blind_answer_key.json").exists(): insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 deliverables.insert(insert_after, "reports/output_blind_answer_key.json") + if (skill_dir / "reports" / "output_review_kit.md").exists(): + insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/output_review_kit.md") if (skill_dir / "reports" / "output_review_adjudication.md").exists(): - insert_after = deliverables.index("reports/output_blind_answer_key.json") + 1 if "reports/output_blind_answer_key.json" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + insert_after = deliverables.index("reports/output_review_kit.md") + 1 if "reports/output_review_kit.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 deliverables.insert(insert_after, "reports/output_review_adjudication.md") + if (skill_dir / "reports" / "benchmark_reproducibility.md").exists(): + insert_after = deliverables.index("reports/output_review_adjudication.md") + 1 if "reports/output_review_adjudication.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/benchmark_reproducibility.md") + if (skill_dir / "reports" / "world_class_evidence_plan.md").exists(): + insert_after = deliverables.index("reports/review_waivers.md") + 1 + deliverables.insert(insert_after, "reports/world_class_evidence_plan.md") + if (skill_dir / "reports" / "world_class_evidence_ledger.md").exists(): + insert_after = ( + deliverables.index("reports/world_class_evidence_plan.md") + 1 + if "reports/world_class_evidence_plan.md" in deliverables + else deliverables.index("reports/review_waivers.md") + 1 + ) + deliverables.insert(insert_after, "reports/world_class_evidence_ledger.md") skill_summary = { "name": name, @@ -599,6 +379,7 @@ def build_report_model(skill_dir: Path) -> dict: "contract_boundary": contract, "quality_review": q_review, "risk_governance": risk_governance(output_risk, system_model, scorecard), + "world_class_readiness": readiness, "package_assets": package_assets(package_map), "iteration_roadmap": {"items": roadmap}, "report_contract": report_contract, @@ -625,6 +406,7 @@ def build_report_model(skill_dir: Path) -> dict: "benchmark_highlights": [], "skill_ir": { "schema_version": skill_ir.get("schema_version", ""), + "source_path": skill_ir_path, "target_count": len(skill_ir.get("targets", [])), "trigger_samples": len(skill_ir.get("trigger_surface", {}).get("should_trigger", [])), "output_eval_cases": len(skill_ir.get("eval_plan", {}).get("output", [])), @@ -664,6 +446,12 @@ def build_report_model(skill_dir: Path) -> dict: "pair_count": output_blind_review.get("summary", {}).get("pair_count", 0), "answer_key_separate": output_blind_review.get("summary", {}).get("answer_key_separate", False), }, + "output_review_kit": { + "ok": output_review_kit.get("ok", False), + "summary": output_review_kit.get("summary", {}), + "artifacts": output_review_kit.get("artifacts", {}), + "failures": output_review_kit.get("failures", []), + }, "output_review_adjudication": { "ok": output_review_adjudication.get("ok", False), "summary": output_review_adjudication.get("summary", {}), @@ -671,6 +459,13 @@ def build_report_model(skill_dir: Path) -> dict: "reviewed_at": output_review_adjudication.get("reviewed_at", ""), "failures": output_review_adjudication.get("failures", []), }, + "benchmark_reproducibility": { + "ok": benchmark_reproducibility.get("ok", False), + "summary": benchmark_reproducibility.get("summary", {}), + "commit": benchmark_reproducibility.get("commit", ""), + "missing_artifacts": benchmark_reproducibility.get("missing_artifacts", []), + "limitations": benchmark_reproducibility.get("limitations", []), + }, "runtime_conformance": conformance.get("summary", {}), "runtime_permissions": { "ok": runtime_permissions.get("ok", False), @@ -738,6 +533,18 @@ def build_report_model(skill_dir: Path) -> dict: "annotations": review_annotations.get("annotations", [])[:8], "failures": review_annotations.get("failures", []), }, + "world_class_evidence_plan": { + "ok": world_class_evidence.get("ok", False), + "summary": world_class_evidence.get("summary", {}), + "tasks": world_class_evidence.get("tasks", [])[:8], + "source_audit": world_class_evidence.get("source_audit", {}), + }, + "world_class_evidence_ledger": { + "ok": world_class_evidence_ledger.get("ok", False), + "summary": world_class_evidence_ledger.get("summary", {}), + "entries": world_class_evidence_ledger.get("entries", [])[:8], + "source_plan": world_class_evidence_ledger.get("source_plan", {}), + }, "synthesis_highlights": synthesis, "artifact_design": q_review["artifact_design"], "prompt_quality": q_review["prompt_quality"], diff --git a/scripts/skill_report_sections.py b/scripts/skill_report_sections.py new file mode 100644 index 00000000..d78680cd --- /dev/null +++ b/scripts/skill_report_sections.py @@ -0,0 +1,114 @@ +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to assemble overview report quality, risk, and asset sections." + + +def artifact_design_highlights(profile: dict) -> list[str]: + primary = profile.get("primary_artifact", {}) + highlights = [] + if primary.get("direction"): + highlights.append(primary["direction"]) + highlights.extend(profile.get("quality_gates", [])[:3]) + return highlights[:4] + + +def prompt_quality_highlights(profile: dict) -> list[str]: + highlights = [] + primary = profile.get("primary_task_family", {}) + complexity = profile.get("complexity", {}) + if primary.get("label"): + highlights.append(f"Primary prompt task family: {primary['label']}.") + if complexity.get("band"): + highlights.append(f"Complexity: {complexity['band']} — {complexity.get('reason', '')}") + for item in profile.get("quality_matrix", [])[:2]: + highlights.append(f"{item.get('label', 'Quality')}: {item.get('score', 'n/a')}/100.") + return highlights[:4] + + +def system_model_highlights(model: dict) -> list[str]: + highlights = [] + stability = model.get("stability", {}) + if stability: + highlights.append(f"Stability: {stability.get('band', 'unknown')} ({stability.get('score', 'n/a')}/100).") + boundary = model.get("boundary_map", {}) + if boundary.get("owned_job"): + highlights.append(f"Owned job: {boundary['owned_job']}") + for point in model.get("leverage_points", [])[:2]: + if point.get("point"): + highlights.append(f"Leverage: {point['point']} — {point.get('move', '')}") + return highlights[:4] + + +def risk_governance(output_risk: dict, system_model: dict, scorecard: dict) -> dict: + risk_names = [ + ("误触发风险", "trigger_score"), + ("输出漂移风险", "evidence_score"), + ("证据不足风险", "evidence_score"), + ("包体膨胀风险", "maintainability_score"), + ("跨平台迁移风险", "portability_score"), + ] + risks = [] + for index, (name, metric_key) in enumerate(risk_names): + score = scorecard.get(metric_key, {}).get("score", 50) + probability = max(1, min(3, 4 - round(score / 34))) + impact = 3 if index in {0, 2, 4} else 2 + risks.append( + { + "name": name, + "impact": impact, + "probability": probability, + "signal": scorecard.get(metric_key, {}).get("reasons", ["证据不足"])[0], + "response": "先补证据和边界，再增加包体复杂度。", + } + ) + human_boundary = system_model.get("boundary_map", {}).get("human_judgment_boundary", []) + return { + "risks": risks, + "risk_families": output_risk.get("risk_families", []), + "human_judgment_boundary": human_boundary, + } + + +def quality_review( + strengths: list[str], + scorecard: dict, + artifact_design: dict, + prompt_quality: dict, + system_model: dict, +) -> dict: + gaps = [] + for key, payload in scorecard.items(): + if payload.get("score", 0) < 70: + gaps.append(f"{payload.get('label', key)}需要补强：{payload.get('reasons', ['证据不足'])[0]}") + return { + "strengths": strengths, + "gaps": gaps or ["当前关键证据较完整，优先保持轻量。"], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。", + ], + "artifact_design": { + "design_system": artifact_design.get("design_system", "content-led editorial"), + "highlights": artifact_design_highlights(artifact_design), + }, + "prompt_quality": { + "overall_quality_score": prompt_quality.get("overall_quality_score", "n/a"), + "highlights": prompt_quality_highlights(prompt_quality), + }, + "system_model": { + "stability": system_model.get("stability", {}), + "highlights": system_model_highlights(system_model), + }, + } + + +def package_assets(package_map: list[dict]) -> dict: + files = sum(item.get("file_count", 0) for item in package_map if item.get("kind") == "file") + folders = [item for item in package_map if item.get("kind") == "folder"] + distribution = [{"label": item["path"], "value": max(1, item.get("file_count", 1))} for item in package_map] + return { + "entries": package_map, + "file_count": files + sum(item.get("file_count", 0) for item in folders), + "folder_count": len(folders), + "distribution": distribution, + } diff --git a/scripts/skill_report_sources.py b/scripts/skill_report_sources.py new file mode 100644 index 00000000..961171d6 --- /dev/null +++ b/scripts/skill_report_sources.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Source loading and package scanning helpers for skill overview reports.""" + +import json +import re +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to load source files and scan overview package assets." + + +KNOWN_ENTRIES = [ + ("SKILL.md", "Skill entrypoint"), + ("README.md", "Human-readable usage guide"), + ("agents/interface.yaml", "Neutral interface metadata"), + ("manifest.json", "Lifecycle and portability metadata"), + ("references", "Extended guidance and reusable notes"), + ("scripts", "Deterministic helpers or local tooling"), + ("evals", "Trigger and quality checks"), + ("reports", "Generated evidence and overview artifacts"), +] + +IGNORED_PACKAGE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"} + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + data = yaml.safe_load(frontmatter_text) or {} + return data if isinstance(data, dict) else {}, body + data = {} + for line in frontmatter_text.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def load_yaml(path: Path) -> dict: + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8") + if yaml is not None: + payload = yaml.safe_load(text) or {} + return payload if isinstance(payload, dict) else {} + return {} + + +def load_json(path: Path) -> dict: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def extract_title(body: str, fallback: str) -> str: + for line in body.splitlines(): + if line.startswith("# "): + return line[2:].strip() + return fallback + + +def parse_sections(body: str) -> dict[str, str]: + sections: dict[str, list[str]] = {} + current = "_preamble" + sections[current] = [] + for line in body.splitlines(): + if line.startswith("## "): + current = line[3:].strip() + sections[current] = [] + continue + sections[current].append(line) + return {name: "\n".join(lines).strip() for name, lines in sections.items()} + + +def extract_list_items(text: str) -> list[str]: + items = [] + for line in text.splitlines(): + stripped = line.strip() + if not stripped: + continue + ordered = re.match(r"^\d+\.\s+(.*)$", stripped) + bullet = re.match(r"^[-*]\s+(.*)$", stripped) + match = ordered or bullet + if match: + items.append(match.group(1).strip()) + return items + + +def summarize_logic(sections: dict[str, str]) -> list[str]: + for key in ("Compact Workflow", "Workflow", "How It Works", "Logic", "Quick Start"): + if key in sections: + items = extract_list_items(sections[key]) + if items: + return items[:5] + return extract_list_items(sections.get("_preamble", ""))[:5] or [ + "Understand the request", + "Execute the main task", + "Validate the result", + ] + + +def summarize_usage(sections: dict[str, str], default_prompt: str, description: str) -> list[str]: + for key in ("How To Use", "Quick Start", "Usage", "Runbook"): + if key in sections: + items = extract_list_items(sections[key]) + if items: + return items[:5] + usage = [] + if default_prompt: + usage.append(default_prompt) + usage.append(f"Use this skill when the request matches: {description}") + return usage[:5] + + +def package_entries(skill_dir: Path) -> list[dict]: + items = [] + for rel_path, label in KNOWN_ENTRIES: + target = skill_dir / rel_path + if target.exists(): + kind = "folder" if target.is_dir() else "file" + if target.is_dir(): + count = len( + [ + path + for path in target.rglob("*") + if path.is_file() + and not path.is_symlink() + and not any(part in IGNORED_PACKAGE_PARTS for part in path.relative_to(target).parts) + and path.suffix not in {".pyc", ".pyo"} + ] + ) + else: + count = 1 + items.append({"path": rel_path, "label": label, "kind": kind, "file_count": count}) + return items diff --git a/scripts/skill_report_world_class.py b/scripts/skill_report_world_class.py new file mode 100644 index 00000000..fffcd96d --- /dev/null +++ b/scripts/skill_report_world_class.py @@ -0,0 +1,128 @@ +"""World-class evidence shaping for the static skill overview report.""" + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to summarize world-class evidence readiness and roadmap actions." + +WORLD_CLASS_ENTRY_COPY = { + "provider-holdout": { + "label_zh": "提供商留出", + "label_en": "Provider Holdout", + "summary_zh": "缺少真实 provider 模型运行和 token metadata。", + "summary_en": "Missing a real provider model run and token metadata.", + }, + "human-adjudication": { + "label_zh": "人工盲评", + "label_en": "Human Adjudication", + "summary_zh": "盲评 pair 仍待真实 reviewer 决策。", + "summary_en": "Blind-review pairs still need real reviewer decisions.", + }, + "native-permission-enforcement": { + "label_zh": "原生权限", + "label_en": "Native Permission", + "summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。", + "summary_en": "Native runtime enforcement still needs target-client or external-installer proof.", + }, + "native-client-telemetry": { + "label_zh": "原生遥测", + "label_en": "Native Telemetry", + "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", + "summary_en": "Real external-client metadata-only events have not been imported yet.", + }, +} + + +def world_class_readiness(ledger: dict) -> dict: + summary = ledger.get("summary", {}) if isinstance(ledger, dict) else {} + entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] + if not isinstance(entries, list): + entries = [] + source_check_count = int(summary.get("source_check_count", 0) or 0) + source_pass_count = int(summary.get("source_pass_count", 0) or 0) + pending_count = int(summary.get("pending_count", len(entries)) or 0) + accepted_count = int(summary.get("accepted_count", 0) or 0) + entry_count = int(summary.get("ledger_entry_count", len(entries)) or 0) + ready = bool(summary.get("ready_to_claim_world_class", False)) + decision = str(summary.get("decision", "not-generated" if entry_count == 0 else "evidence-pending")) + if entry_count == 0: + conclusion_zh = "未生成 world-class ledger；当前报告不会宣称世界级完成。" + conclusion_en = "No world-class ledger was generated; this report does not claim world-class completion." + elif ready: + conclusion_zh = "世界级证据已被 ledger 接受，可进入公开 claim 前的最终复核。" + conclusion_en = "World-class evidence is accepted by the ledger and can move to final claim review." + else: + conclusion_zh = f"世界级证据尚未完成：{pending_count} 项待补，{accepted_count} 项已接受。" + conclusion_en = f"World-class evidence is not complete: {pending_count} pending, {accepted_count} accepted." + + items = [] + for entry in entries[:4]: + if not isinstance(entry, dict): + continue + key = str(entry.get("key", "")) + copy = WORLD_CLASS_ENTRY_COPY.get(key, {}) + category = str(entry.get("category", "external")) + checklist = entry.get("source_checklist", []) + blocked_checks = [ + str(check.get("label", "")) + for check in checklist + if isinstance(check, dict) and check.get("status") == "blocked" and str(check.get("label", "")).strip() + ][:3] + items.append( + { + "key": key, + "label_zh": copy.get("label_zh", str(entry.get("label", key))), + "label_en": copy.get("label_en", str(entry.get("label", key))), + "category": category, + "category_zh": "人工证据" if category == "human" else "外部证据", + "category_en": "Human evidence" if category == "human" else "External evidence", + "status": str(entry.get("status", "pending")), + "summary_zh": copy.get("summary_zh", str(entry.get("current", ""))), + "summary_en": copy.get("summary_en", str(entry.get("current", ""))), + "blocked_checks": blocked_checks, + } + ) + return { + "ready": ready, + "decision": decision, + "entry_count": entry_count, + "pending_count": pending_count, + "accepted_count": accepted_count, + "external_pending_count": int(summary.get("external_pending_count", 0) or 0), + "human_pending_count": int(summary.get("human_pending_count", 0) or 0), + "source_check_count": source_check_count, + "source_pass_count": source_pass_count, + "conclusion_zh": conclusion_zh, + "conclusion_en": conclusion_en, + "entries": items, + } + + +def world_class_roadmap_item(readiness: dict) -> dict | None: + if not isinstance(readiness, dict): + return None + pending_count = int(readiness.get("pending_count", 0) or 0) + if readiness.get("ready") or pending_count <= 0: + return None + external_pending_count = int(readiness.get("external_pending_count", 0) or 0) + human_pending_count = int(readiness.get("human_pending_count", 0) or 0) + actions = [] + entries = [entry for entry in readiness.get("entries", []) if isinstance(entry, dict)] + for entry in entries[:2]: + label = str(entry.get("label_zh") or entry.get("key") or "证据项") + summary = str(entry.get("summary_zh") or "仍待补充真实证据。") + actions.append(f"补齐{label}证据：{summary}") + remaining = pending_count - len(actions) + if remaining > 0: + actions.append( + f"按 ledger 总量继续补齐 {pending_count} 项待补证据" + f"（外部 {external_pending_count} 项、人工 {human_pending_count} 项；" + f"未展开 {remaining} 项），并保持 claim guard 为 pending 状态。" + ) + else: + actions.append("提交有效 intake packet，并让 ledger 通过 artifact SHA-256 校验。") + return { + "title": "补齐世界证据", + "why": f"世界级证据仍有 {pending_count} 项待补；公开完成态 claim 必须继续保持阻塞。", + "actions": actions[:3], + "unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。", + "source": "world_class_evidence_ledger", + } diff --git a/scripts/skillops_opportunity.py b/scripts/skillops_opportunity.py new file mode 100644 index 00000000..f8657023 --- /dev/null +++ b/scripts/skillops_opportunity.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import hashlib +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_daily_skillops_report.py to score SkillOps opportunities without writing files." + +ACTION_POLICY: dict[str, dict[str, str]] = { + "language_default": { + "action_type": "patch_existing_skill", + "opportunity_type": "report-experience", + "reason": "Repeated language preferences can improve existing report templates after approval.", + }, + "report_ui": { + "action_type": "patch_existing_skill", + "opportunity_type": "artifact-quality", + "reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.", + }, + "approval_safety": { + "action_type": "agents_update", + "opportunity_type": "governance", + "reason": "Repeated privacy and approval signals should tighten durable operating guidance.", + }, + "delivery_format": { + "action_type": "patch_existing_skill", + "opportunity_type": "artifact-discoverability", + "reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.", + }, + "evidence_testing": { + "action_type": "add_eval", + "opportunity_type": "quality-gate", + "reason": "Repeated verification and evidence requests should become focused checks.", + }, +} + +RISK_PENALTY = { + "low": 0, + "medium": 8, + "high": 18, +} + + +def _stable_id(pattern_id: str, target_files: list[str], title: str) -> str: + digest = hashlib.sha1(f"{pattern_id}:{title}:{','.join(target_files)}".encode("utf-8")).hexdigest() + return f"skillops-{digest[:10]}" + + +def _as_int(value: Any, default: int = 0) -> int: + if isinstance(value, bool): + return default + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _risk_level(proposal: dict[str, Any]) -> str: + risk = str(proposal.get("risk_level") or "medium").lower() + return risk if risk in RISK_PENALTY else "medium" + + +def _target_files(proposal: dict[str, Any]) -> list[str]: + files = proposal.get("target_files") + if not isinstance(files, list): + return [] + return [str(item) for item in files if str(item).strip()] + + +def _verification_commands(proposal: dict[str, Any]) -> list[str]: + commands = proposal.get("verification_commands") + if not isinstance(commands, list): + return [] + return [str(item) for item in commands if str(item).strip()] + + +def _score(pattern: dict[str, Any], proposal: dict[str, Any]) -> tuple[int, list[str]]: + support = _as_int(pattern.get("support_count"), _as_int(proposal.get("support_count"), 0)) + confidence = pattern.get("confidence", 0.5) + try: + confidence_score = int(float(confidence) * 25) + except (TypeError, ValueError): + confidence_score = 12 + target_files = _target_files(proposal) + verification_commands = _verification_commands(proposal) + risk = _risk_level(proposal) + reasons = [ + f"support_count={support}", + f"confidence={confidence}", + f"risk={risk}", + ] + score = 28 + score += min(30, support * 10) + score += max(0, min(25, confidence_score)) + if target_files: + score += 8 + reasons.append("target_files_present") + else: + reasons.append("target_files_missing") + if verification_commands: + score += 7 + reasons.append("verification_present") + else: + reasons.append("verification_missing") + score -= RISK_PENALTY[risk] + if proposal.get("requires_approval") is True: + reasons.append("approval_required") + return max(0, min(100, score)), reasons + + +def _decision(score: int, proposal: dict[str, Any]) -> str: + if score >= 85: + return "ready_for_approval_review" + if score >= 70: + return "proposal_review" + if score >= 50: + return "observe_more_evidence" + if proposal.get("requires_approval") is True: + return "report_only" + return "no_action" + + +def _priority(score: int, risk: str) -> str: + if score >= 85 and risk == "low": + return "high" + if score >= 70: + return "medium" + return "low" + + +def build_opportunities(patterns: list[dict[str, Any]], proposals: list[dict[str, Any]]) -> list[dict[str, Any]]: + pattern_by_id = {str(item.get("pattern_id", "")): item for item in patterns if isinstance(item, dict)} + opportunities: list[dict[str, Any]] = [] + for proposal in proposals: + if not isinstance(proposal, dict): + continue + pattern_id = str(proposal.get("pattern_id") or "generic") + pattern = pattern_by_id.get(pattern_id, {}) + policy = ACTION_POLICY.get( + pattern_id, + { + "action_type": "report_only", + "opportunity_type": "manual-review", + "reason": "Unknown pattern family requires manual review before any durable change.", + }, + ) + target_files = _target_files(proposal) + verification_commands = _verification_commands(proposal) + risk = _risk_level(proposal) + score, score_reasons = _score(pattern, proposal) + evidence_refs = proposal.get("evidence_refs") + if not isinstance(evidence_refs, list): + evidence_refs = [] + opportunities.append( + { + "opportunity_id": _stable_id(pattern_id, target_files, str(proposal.get("title") or "")), + "proposal_id": proposal.get("proposal_id", ""), + "pattern_id": pattern_id, + "title": proposal.get("title", ""), + "opportunity_type": policy["opportunity_type"], + "action_type": policy["action_type"], + "decision": _decision(score, proposal), + "priority": _priority(score, risk), + "score": score, + "score_reasons": score_reasons, + "policy_reason": policy["reason"], + "risk_level": risk, + "requires_approval": proposal.get("requires_approval") is True, + "write_allowed_without_approval": False, + "evidence_count": len(evidence_refs) or _as_int(pattern.get("support_count"), 0), + "target_files": target_files, + "verification_commands": verification_commands, + } + ) + opportunities.sort(key=lambda item: (-int(item["score"]), str(item["opportunity_id"]))) + return opportunities + + +def summarize_opportunities(opportunities: list[dict[str, Any]]) -> dict[str, Any]: + by_action: dict[str, int] = {} + by_decision: dict[str, int] = {} + for item in opportunities: + by_action[str(item.get("action_type", ""))] = by_action.get(str(item.get("action_type", "")), 0) + 1 + by_decision[str(item.get("decision", ""))] = by_decision.get(str(item.get("decision", "")), 0) + 1 + top_score = int(opportunities[0]["score"]) if opportunities else 0 + return { + "opportunity_count": len(opportunities), + "top_score": top_score, + "ready_for_approval_review_count": by_decision.get("ready_for_approval_review", 0), + "proposal_review_count": by_decision.get("proposal_review", 0), + "observe_more_evidence_count": by_decision.get("observe_more_evidence", 0), + "report_only_count": by_decision.get("report_only", 0), + "action_type_counts": by_action, + "decision_counts": by_decision, + } + + +def decision_policy_contract() -> dict[str, Any]: + return { + "schema_version": "1.0", + "contract": "skillops-opportunity-scoring", + "score_range": [0, 100], + "score_bands": { + "85-100": "ready_for_approval_review", + "70-84": "proposal_review", + "50-69": "observe_more_evidence", + "0-49": "report_only_or_no_action", + }, + "action_types": sorted({item["action_type"] for item in ACTION_POLICY.values()} | {"report_only"}), + "writes_source_files": False, + "auto_patch_enabled": False, + "approval_required_for_writes": True, + } diff --git a/scripts/summarize_user_signals.py b/scripts/summarize_user_signals.py new file mode 100644 index 00000000..3ce15e4a --- /dev/null +++ b/scripts/summarize_user_signals.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Scans an explicit local source file and summarizes redacted repeated user preference signals." + +TEXT_FIELDS = ("text", "message", "content", "excerpt", "prompt", "note", "body") +HISTORY_FILENAMES = {".zsh_history", ".bash_history", ".fish_history", "History"} +SECRET_PATTERNS = [ + re.compile(r"sk-[A-Za-z0-9_-]{12,}"), + re.compile(r"AKIA[0-9A-Z]{12,}"), + re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b\s*[:=]\s*[^\s,;]+"), +] +EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") +LOCAL_PATH_RE = re.compile(r"/Users/[^\s'\"<>]+") + +PATTERN_RULES = [ + { + "pattern_id": "language_default", + "label": "Default language preference", + "signal_type": "report-language", + "keywords": ["中文", "简体", "默认中文", "英文", "双语", "language", "bilingual", "chinese", "english"], + "recommended_action": "Keep generated reports Chinese-first with an English switch where user-facing.", + }, + { + "pattern_id": "report_ui", + "label": "Report UI and visualization preference", + "signal_type": "artifact-design", + "keywords": ["报告", "html", "图表", "排版", "ui", "kami", "白底", "模块", "导航", "report", "chart", "layout"], + "recommended_action": "Prioritize white-background Kami-style reports with readable charts and stable navigation.", + }, + { + "pattern_id": "approval_safety", + "label": "Approval and privacy boundary", + "signal_type": "governance", + "keywords": ["审批", "授权", "不要扫描", "隐私", "私人", "日志", "明确路径", "回滚", "提案", "批准", "approval", "privacy", "private", "proposal", "rollback"], + "recommended_action": "Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path.", + }, + { + "pattern_id": "delivery_format", + "label": "Delivery format preference", + "signal_type": "artifact-format", + "keywords": ["markdown", "pdf", "word", "docx", "html", "地址", "路径", "打开", "输出", "交付"], + "recommended_action": "Surface stable artifact paths and formats in CLI output and generated summaries.", + }, + { + "pattern_id": "evidence_testing", + "label": "Evidence and testing preference", + "signal_type": "quality-gate", + "keywords": ["测试", "验证", "ci", "证据", "覆盖", "github", "push", "evidence", "review"], + "recommended_action": "Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade.", + }, +] + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def display_path(path: Path, skill_dir: Path) -> str: + try: + return str(path.resolve().relative_to(skill_dir.resolve())) + except ValueError: + return f"[external-explicit-source]/{path.name}" + + +def resolve_output(skill_dir: Path, value: str) -> Path: + path = Path(value) + return path if path.is_absolute() else skill_dir / path + + +def source_fingerprint(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def redact_text(text: str) -> str: + redacted = text + for pattern in SECRET_PATTERNS: + redacted = pattern.sub("[REDACTED_SECRET]", redacted) + redacted = EMAIL_RE.sub("[REDACTED_EMAIL]", redacted) + redacted = LOCAL_PATH_RE.sub("[LOCAL_PATH]", redacted) + redacted = re.sub(r"\s+", " ", redacted).strip() + if len(redacted) > 240: + return redacted[:237].rstrip() + "..." + return redacted + + +def extract_text(raw: Any) -> str: + if isinstance(raw, str): + return raw + if not isinstance(raw, dict): + return "" + for field in TEXT_FIELDS: + value = raw.get(field) + if isinstance(value, str) and value.strip(): + return value + messages = raw.get("messages") + if isinstance(messages, list): + parts = [] + for item in messages: + if isinstance(item, dict): + content = item.get("content") + if isinstance(content, str): + parts.append(content) + elif isinstance(item, str): + parts.append(item) + return "\n".join(parts) + return "" + + +def load_records(source: Path) -> tuple[list[dict[str, str]], list[str]]: + records: list[dict[str, str]] = [] + failures: list[str] = [] + text = source.read_text(encoding="utf-8", errors="replace") + if source.suffix.lower() == ".jsonl": + for index, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + try: + raw = json.loads(line) + except json.JSONDecodeError as exc: + failures.append(f"line {index}: invalid JSONL source: {exc.msg}") + continue + extracted = extract_text(raw) + if not extracted.strip(): + failures.append(f"line {index}: no supported text field found") + continue + records.append({"record_id": f"line-{index}", "excerpt": redact_text(extracted)}) + else: + for index, line in enumerate(text.splitlines(), start=1): + if line.strip(): + records.append({"record_id": f"line-{index}", "excerpt": redact_text(line)}) + return records, failures + + +def classify_patterns(records: list[dict[str, str]], min_support: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + patterns: list[dict[str, Any]] = [] + discarded: list[dict[str, Any]] = [] + for rule in PATTERN_RULES: + matches = [] + for record in records: + lowered = record["excerpt"].lower() + if any(keyword.lower() in lowered for keyword in rule["keywords"]): + matches.append(record) + if not matches: + continue + item = { + "pattern_id": rule["pattern_id"], + "label": rule["label"], + "signal_type": rule["signal_type"], + "support_count": len(matches), + "confidence": min(0.95, round(0.55 + (0.12 * len(matches)), 2)), + "reason": f"{len(matches)} redacted records matched repeated {rule['signal_type']} signals.", + "recommended_action": rule["recommended_action"], + "evidence": matches[:3], + } + if len(matches) >= min_support: + patterns.append(item) + else: + discarded.append({**item, "discard_reason": f"support_count below min_support {min_support}"}) + return patterns, discarded + + +def build_report( + skill_dir: Path, + source: Path, + min_support: int, + generated_at: str, + allow_history_source: bool, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + source = source.resolve() + failures: list[str] = [] + records: list[dict[str, str]] = [] + fingerprint = "" + if not source.exists(): + failures.append(f"Explicit source does not exist: {display_path(source, skill_dir)}") + elif not source.is_file(): + failures.append(f"Explicit source must be a file: {display_path(source, skill_dir)}") + elif source.name in HISTORY_FILENAMES and not allow_history_source: + failures.append(f"Refusing private history source by default: {source.name}") + else: + fingerprint = source_fingerprint(source) + records, load_failures = load_records(source) + failures.extend(load_failures) + patterns, discarded = classify_patterns(records, min_support) if not failures else ([], []) + return { + "ok": not failures, + "schema_version": "1.0", + "generated_at": generated_at, + "skill_dir": display_path(skill_dir, skill_dir), + "source": { + "label": source.name, + "path": display_path(source, skill_dir), + "fingerprint_sha256": fingerprint, + "explicit_source": True, + "record_count": len(records), + }, + "privacy_contract": { + "local_only": True, + "explicit_source_required": True, + "implicit_private_log_scan": False, + "raw_content_stored": False, + "redacted_excerpts_only": True, + "redacted_excerpt_limit": 240, + "writes_repository_files": False, + }, + "summary": { + "record_count": len(records), + "pattern_count": len(patterns), + "discarded_signal_count": len(discarded), + "min_support": min_support, + "failure_count": len(failures), + }, + "patterns": patterns, + "discarded_signals": discarded, + "failures": failures, + "artifacts": { + "json": "reports/user_patterns.json", + "markdown": "reports/user_patterns.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + lines = [ + "# User Pattern Summary", + "", + f"- Generated at: `{report['generated_at']}`", + f"- Local only: `{str(report['privacy_contract']['local_only']).lower()}`", + f"- Explicit source: `{report['source']['path']}`", + f"- Records: `{report['summary']['record_count']}`", + f"- Patterns: `{report['summary']['pattern_count']}`", + f"- Discarded signals: `{report['summary']['discarded_signal_count']}`", + "", + "## Privacy Contract", + "", + "- No implicit private log scan.", + "- No unredacted raw content stored.", + "- Scan and proposal stages do not write source files.", + "", + "## Patterns", + "", + ] + if not report["patterns"]: + lines.append("- No repeated pattern met the support threshold.") + for pattern in report["patterns"]: + lines.extend( + [ + f"### {pattern['label']}", + "", + f"- Pattern: `{pattern['pattern_id']}`", + f"- Support: `{pattern['support_count']}`", + f"- Confidence: `{pattern['confidence']}`", + f"- Reason: {pattern['reason']}", + f"- Recommended action: {pattern['recommended_action']}", + "- Redacted evidence:", + ] + ) + for item in pattern["evidence"]: + lines.append(f" - `{item['record_id']}`: {item['excerpt']}") + lines.append("") + if report["discarded_signals"]: + lines.extend(["## Discarded Signals", ""]) + for item in report["discarded_signals"]: + lines.append(f"- `{item['pattern_id']}`: {item['discard_reason']}") + if report["failures"]: + lines.extend(["", "## Failures", ""]) + lines.extend(f"- {failure}" for failure in report["failures"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Summarize repeated user preference signals from one explicit local source file.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--source", required=True) + parser.add_argument("--output-json", default="reports/user_patterns.json") + parser.add_argument("--output-md", default="reports/user_patterns.md") + parser.add_argument("--min-support", type=int, default=2) + parser.add_argument("--generated-at", default=utc_now()) + parser.add_argument("--allow-history-source", action="store_true") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report( + skill_dir, + Path(args.source), + min_support=max(2, args.min_support), + generated_at=args.generated_at, + allow_history_source=args.allow_history_source, + ) + if report["ok"]: + output_json = resolve_output(skill_dir, args.output_json) + output_md = resolve_output(skill_dir, args.output_md) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + report["artifacts"] = { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + } + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/sync_local_install.py b/scripts/sync_local_install.py index 62577629..d18eff4b 100644 --- a/scripts/sync_local_install.py +++ b/scripts/sync_local_install.py @@ -3,14 +3,19 @@ import argparse import json import shutil import subprocess +import tempfile +from datetime import date from pathlib import Path +from simulate_install import simulate_install + ROOT = Path(__file__).resolve().parent.parent SKILL_NAME = "yao-meta-skill" SENTINEL_NAME = ".yao-local-install.json" DEFAULT_INSTALL_DIR = Path.home() / ".agents" / "skills.disabled" / SKILL_NAME ACTIVE_INSTALL_DIR = Path.home() / ".agents" / "skills" / SKILL_NAME +DEFAULT_PACKAGE_DIR = ROOT / "dist" ALLOW_UNTRACKED_PREFIXES = { ".github", "agents", @@ -133,6 +138,31 @@ def write_sentinel(root: Path, install_dir: Path, dry_run: bool) -> None: (install_dir / SENTINEL_NAME).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") +def install_preflight(root: Path, package_dir: Path, generated_at: str) -> dict: + package_dir = package_dir.resolve() + with tempfile.TemporaryDirectory(prefix="yao-local-install-preflight-") as temp_dir: + report = simulate_install(root, package_dir, Path(temp_dir), generated_at) + summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {} + permission_failures = int(summary.get("installer_permission_failure_count", 0) or 0) + if not report.get("ok") or permission_failures: + failure_preview = "; ".join(str(item) for item in report.get("failures", [])[:3]) + if permission_failures and not failure_preview: + failure_preview = f"{permission_failures} installer permission failures" + raise ValueError(f"Install preflight failed for {package_dir}: {failure_preview or 'unknown failure'}") + return { + "ok": True, + "package_dir": str(package_dir), + "archive_extracted": bool(summary.get("archive_extracted")), + "adapter_count": int(summary.get("adapter_count", 0) or 0), + "installer_permission_enforced_count": int(summary.get("installer_permission_enforced_count", 0) or 0), + "installer_permission_failure_count": permission_failures, + "permission_target_count": int(summary.get("permission_target_count", 0) or 0), + "permission_capability_count": int(summary.get("permission_capability_count", 0) or 0), + "failure_count": int(summary.get("failure_count", 0) or 0), + "warning_count": int(summary.get("warning_count", 0) or 0), + } + + def remove_stale_files(install_dir: Path, desired_files: set[Path], dry_run: bool) -> list[str]: removed = [] if not install_dir.exists(): @@ -176,10 +206,21 @@ def copy_files(root: Path, install_dir: Path, files: list[Path], dry_run: bool) return copied, skipped -def sync_local_install(root: Path, install_dir: Path, dry_run: bool = False) -> dict: +def sync_local_install( + root: Path, + install_dir: Path, + dry_run: bool = False, + package_dir: Path | None = None, + generated_at: str | None = None, + skip_install_preflight: bool = False, +) -> dict: root = root.resolve() install_dir = install_dir.resolve() validate_install_dir(root, install_dir) + preflight = {"ok": True, "skipped": True} + if not skip_install_preflight: + resolved_package_dir = (package_dir or DEFAULT_PACKAGE_DIR).resolve() + preflight = install_preflight(root, resolved_package_dir, generated_at or str(date.today())) files, skipped_untracked = candidate_files(root) desired_files = set(files) desired_files.add(Path(SENTINEL_NAME)) @@ -197,6 +238,7 @@ def sync_local_install(root: Path, install_dir: Path, dry_run: bool = False) -> "removed_count": len(removed), "skipped_source_count": len(skipped_sources), "skipped_untracked_count": len(skipped_untracked), + "install_preflight": preflight, "copied_samples": copied[:10], "removed_samples": removed[:10], "skipped_source_samples": skipped_sources[:10], @@ -208,13 +250,23 @@ def main() -> None: parser = argparse.ArgumentParser(description="Sync the current yao-meta-skill source into a managed local skill mirror.") parser.add_argument("--root", default=str(ROOT)) parser.add_argument("--install-dir", default=str(DEFAULT_INSTALL_DIR)) + parser.add_argument("--package-dir", default=str(DEFAULT_PACKAGE_DIR)) + parser.add_argument("--generated-at", default=str(date.today())) + parser.add_argument("--skip-install-preflight", action="store_true") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() + root = Path(args.root).resolve() + package_dir = Path(args.package_dir) + if not package_dir.is_absolute(): + package_dir = root / package_dir try: result = sync_local_install( - Path(args.root), + root, resolve_install_dir(args.install_dir), dry_run=args.dry_run, + package_dir=package_dir, + generated_at=args.generated_at, + skip_install_preflight=args.skip_install_preflight, ) except (OSError, subprocess.CalledProcessError, ValueError) as exc: result = {"ok": False, "failures": [str(exc)]} diff --git a/scripts/telemetry_native_host.py b/scripts/telemetry_native_host.py new file mode 100644 index 00000000..f3520559 --- /dev/null +++ b/scripts/telemetry_native_host.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import shlex +import stat +import struct +import sys +from pathlib import Path +from typing import Any, BinaryIO + +from emit_telemetry_event import append_event, default_spool_path +from render_adoption_drift_report import display_path, normalize_event, skill_defaults + + +SCHEMA_VERSION = "1.0" +DEFAULT_HOST_NAME = "com.yao.meta_skill.telemetry" +DEFAULT_DESCRIPTION = "Yao metadata-only telemetry native messaging host" +SKILL_DIR_ENV = "YAO_TELEMETRY_SKILL_DIR" +EVENTS_ENV = "YAO_TELEMETRY_EVENTS" +MAX_MESSAGE_BYTES = 1024 * 1024 + + +def read_native_message(stream: BinaryIO) -> dict[str, Any] | None: + header = stream.read(4) + if not header: + return None + if len(header) != 4: + raise ValueError("native message header must be exactly 4 bytes") + size = struct.unpack(" MAX_MESSAGE_BYTES: + raise ValueError(f"native message is too large: {size} bytes") + payload = stream.read(size) + if len(payload) != size: + raise ValueError("native message payload ended before declared length") + message = json.loads(payload.decode("utf-8")) + if not isinstance(message, dict): + raise ValueError("native message must be a JSON object") + return message + + +def write_native_message(stream: BinaryIO, payload: dict[str, Any]) -> None: + data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + stream.write(struct.pack(" dict[str, Any]: + raw = dict(raw_message) + raw.setdefault("source", "external") + raw.setdefault("command", "native-messaging-host") + event, failures = normalize_event(raw, skill_defaults(skill_dir), "native-host-message") + if event and not failures and not dry_run: + append_event(output_jsonl, event) + return { + "ok": not failures, + "schema_version": SCHEMA_VERSION, + "mode": "native-messaging-host", + "skill_dir": display_path(skill_dir), + "output_jsonl": display_path(output_jsonl), + "dry_run": dry_run, + "emitted": bool(event and not failures and not dry_run), + "event": event or {}, + "failures": failures, + } + + +def write_launcher(path: Path, host_script: Path, skill_dir: Path, output_jsonl: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + script = "\n".join( + [ + "#!/bin/sh", + "set -eu", + ( + "exec python3 " + f"{shlex.quote(str(host_script.resolve()))} " + f"{shlex.quote(str(skill_dir.resolve()))} " + "--stdio " + f"--output-jsonl {shlex.quote(str(output_jsonl.resolve()))}" + ), + "", + ] + ) + path.write_text(script, encoding="utf-8") + current = path.stat().st_mode + path.chmod(current | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def write_manifest( + manifest_path: Path, + host_name: str, + description: str, + host_path: Path, + allowed_origins: list[str], +) -> dict[str, Any]: + failures: list[str] = [] + if not allowed_origins: + failures.append("At least one --allowed-origin is required for a Chrome native messaging manifest.") + for origin in allowed_origins: + if not origin.startswith("chrome-extension://") or not origin.endswith("/"): + failures.append(f"Unsupported Chrome extension origin: {origin}") + if not host_path.exists(): + failures.append(f"Native host path does not exist: {host_path}") + elif not os.access(host_path, os.X_OK): + failures.append(f"Native host path must be executable: {host_path}") + manifest = { + "name": host_name, + "description": description, + "path": str(host_path.resolve()), + "type": "stdio", + "allowed_origins": allowed_origins, + } + if not failures: + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return { + "ok": not failures, + "schema_version": SCHEMA_VERSION, + "manifest": manifest, + "manifest_path": display_path(manifest_path), + "failures": failures, + } + + +def run_stdio(skill_dir: Path, output_jsonl: Path, dry_run: bool) -> int: + while True: + raw = read_native_message(sys.stdin.buffer) + if raw is None: + return 0 + response = emit_native_message(skill_dir, output_jsonl, raw, dry_run=dry_run) + write_native_message(sys.stdout.buffer, response) + + +def env_default(name: str, fallback: str) -> str: + return os.environ.get(name) or fallback + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run a metadata-only telemetry native messaging host.") + parser.add_argument("skill_dir", nargs="?", default=env_default(SKILL_DIR_ENV, ".")) + parser.add_argument("--output-jsonl", default=os.environ.get(EVENTS_ENV)) + parser.add_argument("--message-json", help="Emit one JSON object directly, useful for host smoke tests.") + parser.add_argument("--stdio", action="store_true", help="Use Chrome/Browser Native Messaging length-prefixed stdio.") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--write-manifest") + parser.add_argument("--write-launcher") + parser.add_argument("--host-name", default=DEFAULT_HOST_NAME) + parser.add_argument("--description", default=DEFAULT_DESCRIPTION) + parser.add_argument("--allowed-origin", action="append", default=[]) + parser.add_argument("--host-path") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + output_jsonl = Path(args.output_jsonl).resolve() if args.output_jsonl else default_spool_path(skill_dir).resolve() + host_script = Path(__file__).resolve() + + launcher_path = Path(args.write_launcher).resolve() if args.write_launcher else None + if launcher_path: + write_launcher(launcher_path, host_script, skill_dir, output_jsonl) + + if args.write_manifest: + host_path = Path(args.host_path).resolve() if args.host_path else launcher_path or host_script + report = write_manifest( + Path(args.write_manifest).resolve(), + args.host_name, + args.description, + host_path, + args.allowed_origin, + ) + if launcher_path: + report["launcher_path"] = display_path(launcher_path) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + return + + if args.stdio: + try: + raise SystemExit(run_stdio(skill_dir, output_jsonl, args.dry_run)) + except Exception as exc: + write_native_message( + sys.stdout.buffer, + { + "ok": False, + "schema_version": SCHEMA_VERSION, + "mode": "native-messaging-host", + "emitted": False, + "failures": [str(exc)], + }, + ) + raise SystemExit(2) + + if args.message_json: + try: + message = json.loads(args.message_json) + except json.JSONDecodeError as exc: + print(json.dumps({"ok": False, "failures": [f"Invalid --message-json: {exc.msg}"]}, ensure_ascii=False, indent=2)) + raise SystemExit(2) + if not isinstance(message, dict): + print(json.dumps({"ok": False, "failures": ["--message-json must decode to a JSON object"]}, ensure_ascii=False, indent=2)) + raise SystemExit(2) + report = emit_native_message(skill_dir, output_jsonl, message, dry_run=args.dry_run) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + return + + print( + json.dumps( + { + "ok": False, + "schema_version": SCHEMA_VERSION, + "failures": ["Use --message-json, --stdio, or --write-manifest."], + }, + ensure_ascii=False, + indent=2, + ) + ) + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/trust_check.py b/scripts/trust_check.py index 5feb1544..8d03125c 100644 --- a/scripts/trust_check.py +++ b/scripts/trust_check.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import argparse -import ast import hashlib import json import subprocess @@ -9,25 +8,25 @@ import re from datetime import date from pathlib import Path from typing import Any -from urllib.parse import urlparse try: import yaml except ImportError: # pragma: no cover yaml = None +from trust_check_scripts import INTERNAL_SCRIPT_INTERFACE, script_inventory + ROOT = Path(__file__).resolve().parent.parent -SCAN_DIRS = ["agents", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"] +SCAN_DIRS = ["agents", "assets", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"] ROOT_FILES = ["SKILL.md", "README.md", "manifest.json", "requirements-ci.txt", "Makefile"] -TEXT_SUFFIXES = {".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"} +TEXT_SUFFIXES = {".css", ".js", ".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"} PACKAGE_HASH_SCOPE = "source-contract-without-generated-reports" -INTERNAL_SCRIPT_INTERFACE = "internal-module" NETWORK_POLICY_REL_PATH = "security/network_policy.json" PERMISSION_POLICY_REL_PATH = "security/permission_policy.json" HELP_SMOKE_TIMEOUT_SECONDS = 5.0 PERMISSION_CAPABILITIES = ("network", "file_write", "subprocess", "interactive") -PERMISSION_TARGETS = ("openai", "claude", "generic") +PERMISSION_TARGETS = ("openai", "claude", "generic", "vscode") SECRET_PATTERNS = [ ("private_key", re.compile(r"-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")), ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")), @@ -94,137 +93,6 @@ def scan_secrets(skill_dir: Path, files: list[Path]) -> list[dict[str, Any]]: return findings -def script_inventory(skill_dir: Path) -> list[dict[str, Any]]: - scripts_dir = skill_dir / "scripts" - if not scripts_dir.exists(): - return [] - inventory = [] - for path in sorted(scripts_dir.glob("*.py")): - text = path.read_text(encoding="utf-8", errors="replace") - flags = script_flags(text) - interface = script_interface(text) - urls = extract_url_literals(text) - inventory.append( - { - "path": relpath(skill_dir, path), - "interface": interface["name"], - "interface_declared": interface["declared"], - "interface_reason": interface["reason"], - "has_argparse": "argparse" in text, - "has_main_guard": 'if __name__ == "__main__"' in text, - "uses_input": flags["uses_input"], - "uses_network": flags["uses_network"], - "uses_file_write": flags["uses_file_write"], - "uses_subprocess": flags["uses_subprocess"], - "network_urls": urls, - "network_hosts": sorted({urlparse(url).hostname or "" for url in urls if urlparse(url).hostname}), - } - ) - return inventory - - -def string_assignment(tree: ast.Module, variable_name: str) -> str: - for node in tree.body: - value_node = None - if isinstance(node, ast.Assign): - if any(isinstance(target, ast.Name) and target.id == variable_name for target in node.targets): - value_node = node.value - elif isinstance(node, ast.AnnAssign): - target = node.target - if isinstance(target, ast.Name) and target.id == variable_name: - value_node = node.value - if isinstance(value_node, ast.Constant) and isinstance(value_node.value, str): - return value_node.value - return "" - - -def script_interface(text: str) -> dict[str, Any]: - try: - tree = ast.parse(text) - except SyntaxError: - match = re.search(r"SCRIPT_INTERFACE\s*=\s*['\"]([^'\"]+)['\"]", text) - reason_match = re.search(r"SCRIPT_INTERFACE_REASON\s*=\s*['\"]([^'\"]+)['\"]", text) - name = match.group(1) if match else "cli" - reason = reason_match.group(1) if reason_match else "" - return {"name": name, "declared": bool(match), "reason": reason} - - name = string_assignment(tree, "SCRIPT_INTERFACE") - reason = string_assignment(tree, "SCRIPT_INTERFACE_REASON") - if name: - return {"name": name, "declared": True, "reason": reason} - return {"name": "cli", "declared": False, "reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules."} - - -def extract_url_literals(text: str) -> list[str]: - values: list[str] = [] - try: - tree = ast.parse(text) - for node in ast.walk(tree): - if isinstance(node, ast.Constant) and isinstance(node.value, str): - values.append(node.value) - except SyntaxError: - values = re.findall(r"['\"]([^'\"]+)['\"]", text) - - urls = [] - seen = set() - for value in values: - for match in re.finditer(r"https?://[^\s'\"<>]+", value): - url = match.group(0).rstrip(").,]") - if url not in seen: - seen.add(url) - urls.append(url) - return urls - - -def call_name(node: ast.AST) -> str: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - base = call_name(node.value) - return f"{base}.{node.attr}" if base else node.attr - return "" - - -def script_flags(text: str) -> dict[str, bool]: - try: - tree = ast.parse(text) - except SyntaxError: - return { - "uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)), - "uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)), - "uses_file_write": bool( - re.search(r"\.(?:write_text|write_bytes|mkdir|unlink)\s*\(", text) - or re.search(r"\b(?:open)\s*\([^)]*['\"][wa+x]", text) - or "shutil.rmtree" in text - ), - "uses_subprocess": "subprocess." in text, - } - call_nodes = [node for node in ast.walk(tree) if isinstance(node, ast.Call)] - calls = [call_name(node.func) for node in call_nodes] - return { - "uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls), - "uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls), - "uses_file_write": any(is_file_write_call(node, call_name(node.func)) for node in call_nodes), - "uses_subprocess": any(name.startswith("subprocess.") for name in calls), - } - - -def is_file_write_call(node: ast.Call, name: str) -> bool: - if name.endswith((".write_text", ".write_bytes", ".mkdir", ".unlink", ".rmdir")): - return True - if name in {"shutil.copy", "shutil.copy2", "shutil.copytree", "shutil.move", "shutil.rmtree"}: - return True - if name == "zipfile.ZipFile": - return True - if name in {"open", "Path.open"} or name.endswith(".open"): - args = list(node.args) - keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg} - mode_node = args[1] if len(args) > 1 else keywords.get("mode") - if isinstance(mode_node, ast.Constant) and isinstance(mode_node.value, str): - return any(flag in mode_node.value for flag in ("w", "a", "x", "+")) - return False - - def dependency_status(skill_dir: Path) -> dict[str, Any]: candidates = ["requirements-ci.txt", "requirements.txt", "pyproject.toml", "package-lock.json", "uv.lock", "poetry.lock"] present = [name for name in candidates if (skill_dir / name).exists()] diff --git a/scripts/trust_check_scripts.py b/scripts/trust_check_scripts.py new file mode 100644 index 00000000..44cdd7f5 --- /dev/null +++ b/scripts/trust_check_scripts.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +import ast +import re +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Static script inventory helpers imported by trust_check.py." +INTERNAL_SCRIPT_INTERFACE = "internal-module" + + +def relpath(skill_dir: Path, path: Path) -> str: + return str(path.relative_to(skill_dir)) + + +def script_inventory(skill_dir: Path) -> list[dict[str, Any]]: + scripts_dir = skill_dir / "scripts" + if not scripts_dir.exists(): + return [] + inventory = [] + for path in sorted(scripts_dir.glob("*.py")): + text = path.read_text(encoding="utf-8", errors="replace") + flags = script_flags(text) + interface = script_interface(text) + urls = extract_url_literals(text) + inventory.append( + { + "path": relpath(skill_dir, path), + "interface": interface["name"], + "interface_declared": interface["declared"], + "interface_reason": interface["reason"], + "has_argparse": "argparse" in text, + "has_main_guard": 'if __name__ == "__main__"' in text, + "uses_input": flags["uses_input"], + "uses_network": flags["uses_network"], + "uses_file_write": flags["uses_file_write"], + "uses_subprocess": flags["uses_subprocess"], + "network_urls": urls, + "network_hosts": sorted({urlparse(url).hostname or "" for url in urls if urlparse(url).hostname}), + } + ) + return inventory + + +def string_assignment(tree: ast.Module, variable_name: str) -> str: + for node in tree.body: + value_node = None + if isinstance(node, ast.Assign): + if any(isinstance(target, ast.Name) and target.id == variable_name for target in node.targets): + value_node = node.value + elif isinstance(node, ast.AnnAssign): + target = node.target + if isinstance(target, ast.Name) and target.id == variable_name: + value_node = node.value + if isinstance(value_node, ast.Constant) and isinstance(value_node.value, str): + return value_node.value + return "" + + +def script_interface(text: str) -> dict[str, Any]: + try: + tree = ast.parse(text) + except SyntaxError: + match = re.search(r"SCRIPT_INTERFACE\s*=\s*['\"]([^'\"]+)['\"]", text) + reason_match = re.search(r"SCRIPT_INTERFACE_REASON\s*=\s*['\"]([^'\"]+)['\"]", text) + name = match.group(1) if match else "cli" + reason = reason_match.group(1) if reason_match else "" + return {"name": name, "declared": bool(match), "reason": reason} + + name = string_assignment(tree, "SCRIPT_INTERFACE") + reason = string_assignment(tree, "SCRIPT_INTERFACE_REASON") + if name: + return {"name": name, "declared": True, "reason": reason} + return {"name": "cli", "declared": False, "reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules."} + + +def extract_url_literals(text: str) -> list[str]: + values: list[str] = [] + try: + tree = ast.parse(text) + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + values.append(node.value) + except SyntaxError: + values = re.findall(r"['\"]([^'\"]+)['\"]", text) + + urls = [] + seen = set() + for value in values: + for match in re.finditer(r"https?://[^\s'\"<>]+", value): + url = match.group(0).rstrip(").,]") + if url not in seen: + seen.add(url) + urls.append(url) + return urls + + +def call_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + base = call_name(node.value) + return f"{base}.{node.attr}" if base else node.attr + return "" + + +def script_flags(text: str) -> dict[str, bool]: + try: + tree = ast.parse(text) + except SyntaxError: + return { + "uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)), + "uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)), + "uses_file_write": bool( + re.search(r"\.(?:write_text|write_bytes|mkdir|unlink)\s*\(", text) + or re.search(r"\b(?:open)\s*\([^)]*['\"][wa+x]", text) + or "shutil.rmtree" in text + ), + "uses_subprocess": "subprocess." in text, + } + call_nodes = [node for node in ast.walk(tree) if isinstance(node, ast.Call)] + calls = [call_name(node.func) for node in call_nodes] + return { + "uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls), + "uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls), + "uses_file_write": any(is_file_write_call(node, call_name(node.func)) for node in call_nodes), + "uses_subprocess": any(name.startswith("subprocess.") for name in calls), + } + + +def is_file_write_call(node: ast.Call, name: str) -> bool: + if name.endswith((".write_text", ".write_bytes", ".mkdir", ".unlink", ".rmdir")): + return True + if name in {"shutil.copy", "shutil.copy2", "shutil.copytree", "shutil.move", "shutil.rmtree"}: + return True + if name == "zipfile.ZipFile": + return True + if name in {"open", "Path.open"} or name.endswith(".open"): + args = list(node.args) + keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg} + mode_node = args[1] if len(args) > 1 else keywords.get("mode") + if isinstance(mode_node, ast.Constant) and isinstance(mode_node.value, str): + return any(flag in mode_node.value for flag in ("w", "a", "x", "+")) + return False diff --git a/scripts/verify_package.py b/scripts/verify_package.py index 4b277278..c84f078d 100644 --- a/scripts/verify_package.py +++ b/scripts/verify_package.py @@ -54,7 +54,7 @@ def generated_zip_entries(names: list[str]) -> list[str]: generated = [] for name in names: parts = PurePosixPath(name).parts - if "dist" in parts or (len(parts) > 2 and parts[1] == "tests" and any(part.startswith("tmp") for part in parts[2:])): + if ".previews" in parts or "dist" in parts or (len(parts) > 2 and parts[1] == "tests" and any(part.startswith("tmp") for part in parts[2:])): generated.append(name) return generated @@ -75,6 +75,10 @@ def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str, failures.append(detail) +def package_name(manifest: dict[str, Any], skill_dir: Path) -> str: + return str(manifest.get("name") or skill_dir.name) + + def verify_package( skill_dir: Path, package_dir: Path, @@ -91,6 +95,7 @@ def verify_package( manifest_path = package_dir / "manifest.json" manifest = load_json(manifest_path) + package_root = package_name(manifest, skill_dir) add_check(checks, failures, "package-manifest", bool(manifest), f"Package manifest exists: {display_path(manifest_path)}") targets = required_targets(expectations, package_dir) @@ -108,15 +113,16 @@ def verify_package( field in adapter, f"{target} adapter includes field: {field}", ) - for target, key in ( - ("openai", "openai_required_files"), - ("claude", "claude_required_files"), - ("generic", "generic_required_files"), - ): - for rel in expectations.get(key, []): + required_files_by_target = { + key[: -len("_required_files")]: value + for key, value in expectations.items() + if key.endswith("_required_files") + } + for target, required_files in required_files_by_target.items(): + for rel in required_files: add_check(checks, failures, f"{target}-file-{rel}", (package_dir / rel).exists(), f"Package contains {rel}") - archive_path = package_dir / f"{skill_dir.name}.zip" + archive_path = package_dir / f"{package_root}.zip" archive_sha = "" archive_entries: list[str] = [] if archive_path.exists(): @@ -128,15 +134,15 @@ def verify_package( else: unsafe_entries = unsafe_zip_entries(archive_entries) required_entries = [ - f"{skill_dir.name}/SKILL.md", - f"{skill_dir.name}/manifest.json", - f"{skill_dir.name}/agents/interface.yaml", + f"{package_root}/SKILL.md", + f"{package_root}/manifest.json", + f"{package_root}/agents/interface.yaml", ] add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries") for entry in required_entries: add_check(checks, failures, f"archive-entry-{entry}", entry in archive_entries, f"Archive contains {entry}") generated_entries = generated_zip_entries(archive_entries) - add_check(checks, failures, "archive-excludes-generated", not generated_entries, "Archive excludes generated dist/ and tests/tmp* contents") + add_check(checks, failures, "archive-excludes-generated", not generated_entries, "Archive excludes generated dist/, .previews/, and tests/tmp* contents") elif require_zip: add_check(checks, failures, "archive-present", False, f"Missing required package archive: {display_path(archive_path)}") else: diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py new file mode 100644 index 00000000..c15cae8e --- /dev/null +++ b/scripts/world_class_evidence_contract.py @@ -0,0 +1,553 @@ +#!/usr/bin/env python3 +import hashlib +import json +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from output_review_privacy import BLOCKED_DECISION_FIELDS +from world_class_human_evidence import validate_human_adjudication_report +from world_class_native_permission_evidence import validate_native_permission_report +from world_class_native_telemetry_evidence import validate_native_telemetry_report +from world_class_provider_evidence import validate_provider_execution_report + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks." + +REQUIRED_TOP_LEVEL = [ + "schema_version", + "evidence_key", + "template_only", + "category", + "source_type", + "submitted_by", + "submitted_at", + "summary", + "artifact_refs", + "provenance", + "privacy", + "anti_overclaim", + "attestation", +] +REQUIRED_PRIVACY_FALSE = [ + "raw_user_content_included", + "raw_provider_prompt_included", + "credentials_included", + "secrets_included", +] +REQUIRED_ANTI_OVERCLAIM_FALSE = [ + "planned_work_counts_as_evidence", + "metadata_fallback_counts_as_native_enforcement", + "pending_review_counts_as_human_decision", + "local_command_runner_counts_as_provider_model", +] +REQUIRED_ATTESTATION_TRUE = [ + "real_external_or_human_evidence", + "reviewer_or_operator_identity_present", + "artifact_refs_reviewed", + "privacy_contract_satisfied", + "ledger_reviewer_approved", +] +EXPECTED_SOURCE_TYPES = { + "provider-holdout": "provider-output-eval", + "human-adjudication": "blind-ab-review", + "native-permission-enforcement": "runtime-permission-guard", + "native-client-telemetry": "native-client-telemetry", +} +SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") +SUBMITTED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$") +DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"} +REQUIRED_REAL_ARTIFACT_PATHS = { + "provider-holdout": {"reports/output_execution_runs.json"}, + "human-adjudication": { + "reports/output_review_adjudication.json", + "reports/output_review_decisions.json", + }, + "native-permission-enforcement": { + "reports/runtime_permission_probes.json", + "reports/install_simulation.json", + }, + "native-client-telemetry": { + "reports/adoption_drift_report.json", + "reports/telemetry_hook_recipes.json", + }, +} +EXPECTED_REAL_ARTIFACT_KINDS = { + "provider-holdout": { + "reports/output_execution_runs.json": "aggregate-report", + }, + "human-adjudication": { + "reports/output_review_adjudication.json": "adjudication-report", + "reports/output_review_decisions.json": "review-decisions", + }, + "native-permission-enforcement": { + "reports/runtime_permission_probes.json": "runtime-probe-report", + "reports/install_simulation.json": "installer-enforcement-report", + }, + "native-client-telemetry": { + "reports/adoption_drift_report.json": "adoption-drift-report", + "reports/telemetry_hook_recipes.json": "hook-recipes", + }, +} +PLACEHOLDER_FRAGMENTS = ( + "YYYY-MM-DD", + "name or team handle", + "operator with provider credentials", + "human reviewer", + "target client or installer integrator", + "Browser/Chrome/IDE/provider client integrator", + "Browser/Chrome/IDE/provider client", + "openai|claude|generic|vscode|other", + "client or installer component", + "/local/path/not/committed", +) +FORBIDDEN_REAL_SUBMISSION_FIELDS = BLOCKED_DECISION_FIELDS + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]: + if not path.exists(): + return {}, "missing" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {}, "invalid-json" + if not isinstance(payload, dict): + return {}, "invalid-json" + return payload, "present" + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def has_placeholder_text(value: Any) -> bool: + text = str(value or "").strip() + if not text: + return False + lowered = text.lower() + return any(fragment.lower() in lowered for fragment in PLACEHOLDER_FRAGMENTS) + + +def require_real_text(errors: list[str], value: Any, field: str) -> None: + text = str(value or "").strip() + add_error(errors, bool(text), f"{field} is required") + add_error(errors, not has_placeholder_text(text), f"{field} must not use template placeholder text") + + +def parse_audit_timestamp(value: Any) -> datetime | None: + text = str(value or "").strip() + if not SUBMITTED_AT_RE.match(text): + return None + if "T" in text: + return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc) + return datetime.fromisoformat(text).replace(tzinfo=timezone.utc) + + +def forbidden_submission_field_paths(value: Any, prefix: str = "$") -> list[str]: + found: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + key_text = str(key) + child_path = f"{prefix}.{key_text}" + if key_text.strip().lower() in FORBIDDEN_REAL_SUBMISSION_FIELDS: + found.append(child_path) + found.extend(forbidden_submission_field_paths(child, child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + found.extend(forbidden_submission_field_paths(child, f"{prefix}[{index}]")) + return found + + +def validate_real_submission_file_identity( + payload: dict[str, Any], + errors: list[str], + *, + path: Path, + template_expected: bool, +) -> None: + if template_expected: + return + evidence_key = str(payload.get("evidence_key", "")).strip() + expected_name = f"{evidence_key}.json" + add_error( + errors, + path.name == expected_name, + f"real submission filename must be {expected_name}", + ) + + +def validate_real_submission_privacy_fields( + payload: dict[str, Any], + errors: list[str], + *, + template_expected: bool, +) -> None: + if template_expected: + return + blocked_paths = forbidden_submission_field_paths(payload) + add_error( + errors, + not blocked_paths, + "real submission must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: " + + ", ".join(blocked_paths[:8]), + ) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]: + text = raw_path.strip() + if any(token in text for token in ("<", ">", "*", "?")): + return None, "artifact_refs path must be concrete, not a placeholder or glob" + candidate = Path(text) + if candidate.is_absolute(): + return None, "artifact_refs path must be relative to the skill directory" + if ".." in candidate.parts: + return None, "artifact_refs path must not escape the skill directory" + resolved = (root / candidate).resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError: + return None, "artifact_refs path must stay inside the skill directory" + return resolved, None + + +def validate_artifact_refs( + payload: dict[str, Any], + errors: list[str], + *, + root: Path, + template_expected: bool, +) -> dict[str, int]: + refs = payload.get("artifact_refs") + add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference") + evidence_key = str(payload.get("evidence_key", "")) + required_paths = REQUIRED_REAL_ARTIFACT_PATHS.get(evidence_key, set()) + expected_kinds = EXPECTED_REAL_ARTIFACT_KINDS.get(evidence_key, {}) + observed_paths: set[str] = set() + seen_artifact_paths: set[str] = set() + stats = { + "artifact_ref_count": len(refs) if isinstance(refs, list) else 0, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + "required_artifact_count": len(required_paths) if not template_expected else 0, + "required_artifact_verified_count": 0, + } + if not isinstance(refs, list): + return stats + for index, ref in enumerate(refs): + if not isinstance(ref, dict): + errors.append(f"artifact_refs[{index}] must be an object") + continue + path_text = str(ref.get("path", "")).strip() + add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required") + add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required") + add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content") + if template_expected or not path_text: + continue + candidate = Path(path_text) + if not candidate.is_absolute() and ".." not in candidate.parts and not any( + token in path_text for token in ("<", ">", "*", "?") + ): + observed_paths.add(candidate.as_posix()) + resolved, path_error = resolve_artifact_path(path_text, root) + if path_error: + errors.append(f"artifact_refs[{index}].path {path_error}") + continue + rel = rel_path(resolved, root) + add_error( + errors, + rel not in seen_artifact_paths, + f"artifact_refs[{index}].path must not duplicate another artifact reference", + ) + seen_artifact_paths.add(rel) + if rel in DISALLOWED_REAL_ARTIFACTS: + errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs") + expected_kind = expected_kinds.get(rel) + if expected_kind: + observed_kind = str(ref.get("kind", "")).strip() + add_error( + errors, + observed_kind == expected_kind, + f"artifact_refs[{index}].kind must be {expected_kind} for {rel}", + ) + if not resolved.exists() or not resolved.is_file(): + errors.append(f"artifact_refs[{index}].path does not exist as a local file") + continue + stats["artifact_existing_count"] += 1 + declared = str(ref.get("sha256", "")).strip() + if not declared: + errors.append(f"artifact_refs[{index}].sha256 is required for a real submission") + continue + if not SHA256_RE.match(declared): + errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest") + continue + actual = sha256_file(resolved) + if actual.lower() != declared.lower(): + errors.append(f"artifact_refs[{index}].sha256 does not match local artifact") + continue + stats["artifact_sha256_verified_count"] += 1 + if rel in required_paths: + stats["required_artifact_verified_count"] += 1 + if not template_expected and required_paths: + missing_required = sorted(required_paths - observed_paths) + for path in missing_required: + errors.append(f"artifact_refs must include required evidence artifact {path}") + if not missing_required and stats["required_artifact_verified_count"] < len(required_paths): + errors.append("all required evidence artifacts must have verified sha256 digests") + return stats + + +def artifact_ref_path_map(payload: dict[str, Any], root: Path) -> dict[str, Path]: + refs = payload.get("artifact_refs", []) + if not isinstance(refs, list): + return {} + paths: dict[str, Path] = {} + for ref in refs: + if not isinstance(ref, dict): + continue + path_text = str(ref.get("path", "")).strip() + if not path_text: + continue + resolved, path_error = resolve_artifact_path(path_text, root) + if path_error or resolved is None: + continue + paths[rel_path(resolved, root)] = resolved + return paths + + +def validate_provider_holdout_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: + paths = artifact_ref_path_map(payload, root) + execution = load_json(paths.get("reports/output_execution_runs.json", root / "__missing__")) + if not execution: + return + provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} + validate_provider_execution_report(execution, provenance, errors) + + +def validate_human_adjudication_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: + paths = artifact_ref_path_map(payload, root) + adjudication = load_json(paths.get("reports/output_review_adjudication.json", root / "__missing__")) + decisions = load_json(paths.get("reports/output_review_decisions.json", root / "__missing__")) + if not adjudication or not decisions: + return + provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} + validate_human_adjudication_report(adjudication, decisions, provenance, errors) + + +def validate_native_permission_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: + paths = artifact_ref_path_map(payload, root) + probes = load_json(paths.get("reports/runtime_permission_probes.json", root / "__missing__")) + install = load_json(paths.get("reports/install_simulation.json", root / "__missing__")) + if probes: + validate_native_permission_report(probes, install, errors) + + +def validate_native_client_telemetry_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: + paths = artifact_ref_path_map(payload, root) + adoption = load_json(paths.get("reports/adoption_drift_report.json", root / "__missing__")) + recipes = load_json(paths.get("reports/telemetry_hook_recipes.json", root / "__missing__")) + validate_native_telemetry_report(adoption, recipes, errors) + + +def validate_real_artifact_payloads(payload: dict[str, Any], errors: list[str], root: Path, template_expected: bool) -> None: + if template_expected: + return + evidence_key = str(payload.get("evidence_key", "")) + if evidence_key == "provider-holdout": + validate_provider_holdout_artifacts(payload, errors, root) + elif evidence_key == "human-adjudication": + validate_human_adjudication_artifacts(payload, errors, root) + elif evidence_key == "native-permission-enforcement": + validate_native_permission_artifacts(payload, errors, root) + elif evidence_key == "native-client-telemetry": + validate_native_client_telemetry_artifacts(payload, errors, root) + + +def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None: + key = str(payload.get("evidence_key", "")) + template_expected = payload.get("template_only") is True + provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} + if key == "provider-holdout": + add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required") + add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required") + add_error( + errors, + provenance.get("credential_material_committed") is False, + "provider-holdout must attest credential_material_committed is false", + ) + elif key == "human-adjudication": + add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required") + add_error(errors, bool(str(provenance.get("reviewed_at", "")).strip()), "human-adjudication provenance.reviewed_at is required") + if not template_expected: + require_real_text(errors, provenance.get("reviewer", ""), "human-adjudication provenance.reviewer") + add_error( + errors, + bool(SUBMITTED_AT_RE.match(str(provenance.get("reviewed_at", "")).strip())), + "human-adjudication provenance.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) + add_error( + errors, + provenance.get("answer_key_opened_after_decisions") is True, + "human-adjudication must attest answer_key_opened_after_decisions is true", + ) + add_error( + errors, + provenance.get("blind_review_completed_before_answer_key") is True, + "human-adjudication must attest blind_review_completed_before_answer_key is true", + ) + add_error( + errors, + bool(str(provenance.get("blind_pack_sha256", "")).strip()), + "human-adjudication provenance.blind_pack_sha256 is required", + ) + add_error( + errors, + provenance.get("reviewer_reason_required") is True, + "human-adjudication must attest reviewer_reason_required is true", + ) + elif key == "native-permission-enforcement": + add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required") + if not template_expected: + require_real_text(errors, provenance.get("target", ""), "native-permission-enforcement provenance.target") + add_error( + errors, + bool(str(provenance.get("guard_location", "")).strip()), + "native-permission-enforcement provenance.guard_location is required", + ) + if not template_expected: + require_real_text( + errors, + provenance.get("guard_location", ""), + "native-permission-enforcement provenance.guard_location", + ) + add_error( + errors, + str(provenance.get("guard_scope", "")).strip() + in {"target-client-native", "external-installer-runtime-guard"}, + "native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard", + ) + add_error( + errors, + provenance.get("guard_blocks_undeclared_capability") is True, + "native-permission-enforcement must attest guard_blocks_undeclared_capability is true", + ) + add_error( + errors, + provenance.get("metadata_fallback_retained_for_other_targets") is True, + "native-permission-enforcement must retain metadata fallback for non-native targets", + ) + elif key == "native-client-telemetry": + add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required") + if not template_expected: + require_real_text(errors, provenance.get("client", ""), "native-client-telemetry provenance.client") + if str(provenance.get("native_host_manifest", "")).strip(): + require_real_text( + errors, + provenance.get("native_host_manifest", ""), + "native-client-telemetry provenance.native_host_manifest", + ) + add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external") + add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only") + + +def validate_payload( + payload: dict[str, Any], + entry: dict[str, Any], + *, + path: Path, + root: Path, + template_expected: bool, +) -> dict[str, Any]: + errors: list[str] = [] + for key in REQUIRED_TOP_LEVEL: + add_error(errors, key in payload, f"missing {key}") + evidence_key = str(entry.get("key", "")) + add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0") + add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}") + add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}") + add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}") + add_error( + errors, + payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key), + f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}", + ) + add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required") + add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required") + add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required") + if not template_expected: + validate_real_submission_file_identity(payload, errors, path=path, template_expected=template_expected) + validate_real_submission_privacy_fields(payload, errors, template_expected=template_expected) + require_real_text(errors, payload.get("submitted_by", ""), "submitted_by") + add_error( + errors, + bool(SUBMITTED_AT_RE.match(str(payload.get("submitted_at", "")).strip())), + "submitted_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) + artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected) + privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {} + for key in REQUIRED_PRIVACY_FALSE: + add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false") + anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {} + for key in REQUIRED_ANTI_OVERCLAIM_FALSE: + add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false") + validate_evidence_specific(payload, errors) + attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {} + if not template_expected: + for key in REQUIRED_ATTESTATION_TRUE: + add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission") + ledger_reviewer = str(attestation.get("ledger_reviewer", "")).strip() + submitted_by = str(payload.get("submitted_by", "")).strip() + require_real_text(errors, ledger_reviewer, "attestation.ledger_reviewer") + add_error( + errors, + bool(SUBMITTED_AT_RE.match(str(attestation.get("ledger_reviewed_at", "")).strip())), + "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) + add_error( + errors, + bool(ledger_reviewer and submitted_by and ledger_reviewer.casefold() != submitted_by.casefold()), + "attestation.ledger_reviewer must be different from submitted_by", + ) + submitted_at = parse_audit_timestamp(payload.get("submitted_at")) + ledger_reviewed_at = parse_audit_timestamp(attestation.get("ledger_reviewed_at")) + add_error( + errors, + bool(submitted_at and ledger_reviewed_at and ledger_reviewed_at >= submitted_at), + "attestation.ledger_reviewed_at must be at or after submitted_at", + ) + validate_real_artifact_payloads(payload, errors, root, template_expected) + return { + "path": rel_path(path, root), + "evidence_key": evidence_key, + "status": "pass" if not errors else "fail", + "template_only": template_expected, + "artifact_integrity": artifact_integrity, + "errors": errors, + } diff --git a/scripts/world_class_human_evidence.py b/scripts/world_class_human_evidence.py new file mode 100644 index 00000000..c0be8f87 --- /dev/null +++ b/scripts/world_class_human_evidence.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +import re +from typing import Any + +from output_review_privacy import forbidden_decision_field_paths, forbidden_raw_content_field_paths + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows." + +REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$") + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def require_real_text(errors: list[str], value: Any, field: str) -> None: + add_error(errors, bool(str(value or "").strip()), f"{field} is required") + + +def object_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def case_ids(items: list[dict[str, Any]]) -> list[str]: + return [str(item.get("case_id", "")).strip() for item in items] + + +def duplicate_case_ids(ids: list[str]) -> list[str]: + seen: set[str] = set() + duplicates: list[str] = [] + for case_id in ids: + if case_id in seen and case_id not in duplicates: + duplicates.append(case_id) + seen.add(case_id) + return duplicates + + +def confidence_valid(value: Any) -> bool: + if value is None or value == "": + return True + if isinstance(value, bool): + return False + try: + parsed = float(value) + except (TypeError, ValueError): + return False + return 0 <= parsed <= 1 + + +def validate_adjudication_raw_content(adjudication: dict[str, Any], errors: list[str]) -> None: + blocked = forbidden_raw_content_field_paths(adjudication, "output_review_adjudication") + add_error( + errors, + not blocked, + "human-adjudication adjudication report must not include raw content, credential, secret, token, prompt, output, transcript, or message fields: " + + ", ".join(blocked[:8]), + ) + + +def validate_decision_rows( + decisions: list[dict[str, Any]], + expected_case_ids: set[str], + errors: list[str], +) -> None: + decision_ids = case_ids(decisions) + add_error(errors, not any(not case_id for case_id in decision_ids), "human-adjudication decisions must include case_id for every row") + duplicates = duplicate_case_ids([case_id for case_id in decision_ids if case_id]) + add_error(errors, not duplicates, "human-adjudication decisions must not contain duplicate case_id values") + add_error( + errors, + set(decision_ids) == expected_case_ids, + "human-adjudication decisions case_id set must match adjudication pairs", + ) + for index, item in enumerate(decisions, start=1): + blocked = forbidden_decision_field_paths(item, f"decisions[{index}]") + add_error( + errors, + not blocked, + "human-adjudication decisions must not include raw content or answer-key fields: " + + ", ".join(blocked[:8]), + ) + winner = str(item.get("winner_variant", "")).strip().upper() + add_error(errors, winner in {"A", "B"}, "human-adjudication decisions must include A/B winner_variant for every case") + add_error(errors, confidence_valid(item.get("confidence")), "human-adjudication decisions confidence must be between 0 and 1") + add_error(errors, bool(str(item.get("reason", "")).strip()), "human-adjudication decisions must include reviewer reason for every case") + + +def validate_review_integrity( + adjudication: dict[str, Any], + decisions: dict[str, Any], + expected_case_ids: set[str], + errors: list[str], +) -> None: + adjudication_integrity = adjudication.get("review_integrity", {}) + decisions_integrity = decisions.get("review_integrity", {}) + add_error( + errors, + isinstance(adjudication_integrity, dict) and bool(adjudication_integrity.get("blind_pack_sha256")), + "human-adjudication adjudication review_integrity.blind_pack_sha256 is required", + ) + add_error( + errors, + isinstance(decisions_integrity, dict) and bool(decisions_integrity.get("blind_pack_sha256")), + "human-adjudication decisions review_integrity.blind_pack_sha256 is required", + ) + if isinstance(adjudication_integrity, dict) and isinstance(decisions_integrity, dict): + add_error( + errors, + bool(adjudication_integrity.get("blind_pack_sha256")) + and adjudication_integrity.get("blind_pack_sha256") == decisions_integrity.get("blind_pack_sha256"), + "human-adjudication review_integrity.blind_pack_sha256 must match between decisions and adjudication", + ) + decision_case_ids = {str(item) for item in decisions_integrity.get("case_ids", []) if str(item).strip()} + adjudication_case_ids = {str(item) for item in adjudication_integrity.get("case_ids", []) if str(item).strip()} + add_error( + errors, + decision_case_ids == expected_case_ids and adjudication_case_ids == expected_case_ids, + "human-adjudication review_integrity.case_ids must match adjudicated case ids", + ) + + +def validate_reviewer_attestation( + adjudication: dict[str, Any], + decisions: dict[str, Any], + errors: list[str], +) -> None: + summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {} + attestation = decisions.get("reviewer_attestation", {}) + add_error(errors, isinstance(attestation, dict), "human-adjudication decisions.reviewer_attestation must be an object") + if not isinstance(attestation, dict): + attestation = {} + for key in [ + "blind_review_completed_before_answer_key", + "answer_key_not_opened_before_decisions", + "raw_content_excluded", + "reviewer_reason_required", + ]: + add_error(errors, attestation.get(key) is True, f"human-adjudication decisions.reviewer_attestation.{key} must be true") + add_error( + errors, + summary.get("blind_review_attested") is True, + "human-adjudication adjudication summary.blind_review_attested must be true", + ) + add_error( + errors, + summary.get("raw_content_excluded_attested") is True, + "human-adjudication adjudication summary.raw_content_excluded_attested must be true", + ) + add_error( + errors, + summary.get("reviewer_reason_required_attested") is True, + "human-adjudication adjudication summary.reviewer_reason_required_attested must be true", + ) + + +def validate_adjudicated_pairs( + pairs: list[dict[str, Any]], + expected_count: int, + errors: list[str], +) -> set[str]: + pair_ids = case_ids(pairs) + add_error(errors, len(pairs) == expected_count, "human-adjudication adjudication pairs length must equal summary.pair_count") + add_error(errors, not any(not case_id for case_id in pair_ids), "human-adjudication adjudication pairs must include case_id") + duplicates = duplicate_case_ids([case_id for case_id in pair_ids if case_id]) + add_error(errors, not duplicates, "human-adjudication adjudication pairs must not contain duplicate case_id values") + for item in pairs: + case_id = str(item.get("case_id", "")).strip() + status = str(item.get("status", "")).strip() + add_error( + errors, + status in {"match", "disagree"}, + f"human-adjudication adjudication pair {case_id or ''} must be match or disagree", + ) + add_error( + errors, + item.get("expected_revealed") is True, + f"human-adjudication adjudication pair {case_id or ''} must reveal expected winner only after valid decision", + ) + add_error( + errors, + str(item.get("reviewer_winner_variant", "")).strip().upper() in {"A", "B"}, + f"human-adjudication adjudication pair {case_id or ''} must include reviewer_winner_variant", + ) + return {case_id for case_id in pair_ids if case_id} + + +def validate_human_adjudication_report( + adjudication: dict[str, Any], + decisions: dict[str, Any], + provenance: dict[str, Any], + errors: list[str], +) -> None: + summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {} + pair_count = real_int(summary.get("pair_count")) + judgment_count = real_int(summary.get("judgment_count")) + pending_count = real_int(summary.get("pending_count")) + invalid_decision_count = real_int(summary.get("invalid_decision_count")) + answer_revealed_count = real_int(summary.get("answer_revealed_count")) + pending_answer_hidden_count = real_int(summary.get("pending_answer_hidden_count")) + checklist_ready_count = real_int(summary.get("reviewer_checklist_ready_count")) + checklist_count = real_int(summary.get("reviewer_checklist_count")) + + validate_adjudication_raw_content(adjudication, errors) + add_error(errors, adjudication.get("ok") is True, "human-adjudication adjudication report ok must be true") + add_error(errors, bool(pair_count and pair_count > 0), "human-adjudication adjudication summary.pair_count must be >0") + add_error( + errors, + bool(pair_count and judgment_count == pair_count), + "human-adjudication adjudication summary.judgment_count must equal summary.pair_count", + ) + add_error(errors, pending_count == 0, "human-adjudication adjudication summary.pending_count must be 0") + add_error(errors, invalid_decision_count == 0, "human-adjudication adjudication summary.invalid_decision_count must be 0") + add_error( + errors, + bool(pair_count and answer_revealed_count == pair_count), + "human-adjudication adjudication summary.answer_revealed_count must equal summary.pair_count", + ) + add_error(errors, pending_answer_hidden_count == 0, "human-adjudication adjudication summary.pending_answer_hidden_count must be 0") + add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false") + add_error( + errors, + summary.get("reviewer_metadata_present") is True, + "human-adjudication adjudication summary.reviewer_metadata_present must be true", + ) + add_error( + errors, + summary.get("reason_required") is True, + "human-adjudication adjudication summary.reason_required must be true", + ) + add_error( + errors, + summary.get("ready_for_human_evidence") is True, + "human-adjudication adjudication summary.ready_for_human_evidence must be true", + ) + if checklist_count is not None or checklist_ready_count is not None: + add_error( + errors, + bool(pair_count and checklist_count == pair_count and checklist_ready_count == pair_count), + "human-adjudication reviewer checklist must be ready for every pair", + ) + add_error(errors, not adjudication.get("failures"), "human-adjudication adjudication failures must be empty") + + decision_rows = object_list(decisions.get("decisions", [])) + add_error(errors, decisions.get("schema_version") == "1.0", "human-adjudication decisions.schema_version must be 1.0") + reviewer = str(decisions.get("reviewer", "")).strip() + reviewed_at = str(decisions.get("reviewed_at", "")).strip() + require_real_text(errors, reviewer, "human-adjudication decisions.reviewer") + add_error( + errors, + bool(REVIEWED_AT_RE.match(reviewed_at)), + "human-adjudication decisions.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", + ) + add_error( + errors, + bool(pair_count and len(decision_rows) == pair_count), + "human-adjudication decisions count must equal adjudication summary.pair_count", + ) + + expected_case_ids = validate_adjudicated_pairs(object_list(adjudication.get("pairs", [])), int(pair_count or 0), errors) + validate_decision_rows(decision_rows, expected_case_ids, errors) + validate_review_integrity(adjudication, decisions, expected_case_ids, errors) + validate_reviewer_attestation(adjudication, decisions, errors) + + provenance_reviewer = str(provenance.get("reviewer", "")).strip() + provenance_reviewed_at = str(provenance.get("reviewed_at", "")).strip() + add_error( + errors, + bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer), + "human-adjudication provenance.reviewer must match decisions.reviewer", + ) + add_error( + errors, + bool(reviewed_at and provenance_reviewed_at and provenance_reviewed_at == reviewed_at), + "human-adjudication provenance.reviewed_at must match decisions.reviewed_at", + ) + add_error( + errors, + provenance.get("answer_key_opened_after_decisions") is True, + "human-adjudication provenance.answer_key_opened_after_decisions must be true", + ) + add_error( + errors, + provenance.get("blind_review_completed_before_answer_key") is True, + "human-adjudication provenance.blind_review_completed_before_answer_key must be true", + ) + add_error( + errors, + provenance.get("reviewer_reason_required") is True, + "human-adjudication provenance.reviewer_reason_required must be true", + ) + integrity = adjudication.get("review_integrity", {}) if isinstance(adjudication.get("review_integrity", {}), dict) else {} + add_error( + errors, + bool(provenance.get("blind_pack_sha256")) + and provenance.get("blind_pack_sha256") == integrity.get("blind_pack_sha256"), + "human-adjudication provenance.blind_pack_sha256 must match adjudication review_integrity.blind_pack_sha256", + ) + decision_fields = provenance.get("decision_fields", []) + required_decision_fields = {"case_id", "winner_variant", "reason"} + add_error( + errors, + isinstance(decision_fields, list) and required_decision_fields <= {str(item) for item in decision_fields}, + "human-adjudication provenance.decision_fields must include case_id, winner_variant, and reason", + ) + add_error(errors, adjudication.get("reviewer") == reviewer, "human-adjudication adjudication reviewer must match decisions.reviewer") + add_error( + errors, + adjudication.get("reviewed_at") == reviewed_at, + "human-adjudication adjudication reviewed_at must match decisions.reviewed_at", + ) diff --git a/scripts/world_class_native_permission_evidence.py b/scripts/world_class_native_permission_evidence.py new file mode 100644 index 00000000..f6ca6e36 --- /dev/null +++ b/scripts/world_class_native_permission_evidence.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +from typing import Any + +from output_review_privacy import forbidden_decision_field_paths + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows." + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def object_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def sorted_strings(value: Any) -> list[str]: + return sorted(str(item) for item in value) if isinstance(value, list) else [] + + +def validate_no_raw_fields(payload: dict[str, Any], label: str, errors: list[str]) -> None: + blocked = forbidden_decision_field_paths(payload, label) + add_error( + errors, + not blocked, + f"native-permission-enforcement {label} must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: " + + ", ".join(blocked[:8]), + ) + + +def validate_target_rows(probes: dict[str, Any], summary: dict[str, Any], errors: list[str]) -> None: + targets = object_list(probes.get("targets", [])) + target_count = real_int(summary.get("target_count")) + native_count = real_int(summary.get("native_enforcement_count")) + fail_count = real_int(summary.get("fail_count")) + expected_capabilities = sorted_strings(probes.get("expected_capabilities")) + + add_error(errors, bool(targets), "native-permission-enforcement runtime probe targets must not be empty") + if target_count is not None: + add_error(errors, len(targets) == target_count, "native-permission-enforcement runtime probe targets length must equal summary.target_count") + add_error(errors, fail_count == 0, "native-permission-enforcement runtime probe summary.fail_count must be 0") + + observed_native = 0 + for index, target in enumerate(targets, start=1): + target_name = str(target.get("target", "")).strip() or str(index) + add_error(errors, target.get("status") == "pass", f"native-permission-enforcement target {target_name} status must be pass") + add_error(errors, not target.get("failures"), f"native-permission-enforcement target {target_name} failures must be empty") + checks = object_list(target.get("checks", [])) + add_error(errors, bool(checks), f"native-permission-enforcement target {target_name} checks must not be empty") + add_error( + errors, + all(item.get("passed") is True for item in checks), + f"native-permission-enforcement target {target_name} checks must all pass", + ) + native = target.get("native_enforcement") is True + if native: + observed_native += 1 + add_error( + errors, + target.get("assurance") == "native-enforced", + f"native-permission-enforcement target {target_name} assurance must be native-enforced", + ) + else: + add_error( + errors, + target.get("metadata_fallback_explicit") is True, + f"native-permission-enforcement fallback target {target_name} must keep metadata_fallback_explicit true", + ) + add_error( + errors, + bool(target.get("residual_risks")), + f"native-permission-enforcement fallback target {target_name} must retain residual risks", + ) + if expected_capabilities: + add_error( + errors, + sorted_strings(target.get("declared_capabilities")) == expected_capabilities, + f"native-permission-enforcement target {target_name} declared_capabilities must match expected_capabilities", + ) + installer = target.get("installer_enforcement", {}) if isinstance(target.get("installer_enforcement"), dict) else {} + add_error( + errors, + installer.get("enforced") is True, + f"native-permission-enforcement target {target_name} installer_enforcement.enforced must be true", + ) + add_error( + errors, + not installer.get("missing_capabilities"), + f"native-permission-enforcement target {target_name} installer_enforcement missing_capabilities must be empty", + ) + add_error( + errors, + not installer.get("failure_details"), + f"native-permission-enforcement target {target_name} installer_enforcement failure_details must be empty", + ) + + add_error(errors, bool(native_count and observed_native == native_count), "native-permission-enforcement native target rows must match summary.native_enforcement_count") + add_error(errors, observed_native > 0, "native-permission-enforcement must include at least one native-enforced target row") + + +def validate_install_simulation_report(install: dict[str, Any], errors: list[str]) -> None: + summary = install.get("summary", {}) if isinstance(install.get("summary", {}), dict) else {} + validate_no_raw_fields(install, "install_simulation", errors) + add_error(errors, install.get("ok") is True, "native-permission-enforcement install simulation report ok must be true") + add_error( + errors, + bool(real_int(summary.get("installer_permission_enforced_count")) and summary["installer_permission_enforced_count"] > 0), + "native-permission-enforcement install simulation summary.installer_permission_enforced_count must be >0", + ) + add_error( + errors, + summary.get("installer_permission_failure_count") == 0, + "native-permission-enforcement install simulation summary.installer_permission_failure_count must be 0", + ) + add_error(errors, summary.get("failure_count") == 0, "native-permission-enforcement install simulation summary.failure_count must be 0") + permission_checks = [ + item + for item in object_list(install.get("checks", [])) + if str(item.get("id", "")).startswith("permission-") and str(item.get("id", "")).endswith(("-approved", "-target-enforcement")) + ] + if permission_checks: + add_error( + errors, + all(item.get("status") == "pass" for item in permission_checks), + "native-permission-enforcement install simulation permission checks must all pass", + ) + + +def validate_native_permission_report(probes: dict[str, Any], install: dict[str, Any], errors: list[str]) -> None: + summary = probes.get("summary", {}) if isinstance(probes.get("summary", {}), dict) else {} + validate_no_raw_fields(probes, "runtime_permission_probes", errors) + add_error(errors, probes.get("ok") is True, "native-permission-enforcement runtime probe report ok must be true") + add_error( + errors, + bool(real_int(summary.get("native_enforcement_count")) and summary["native_enforcement_count"] > 0), + "native-permission-enforcement runtime probe summary.native_enforcement_count must be >0", + ) + add_error( + errors, + summary.get("failure_count") == 0, + "native-permission-enforcement runtime probe summary.failure_count must be 0", + ) + add_error( + errors, + summary.get("installer_enforcement_ready") is True, + "native-permission-enforcement runtime probe summary.installer_enforcement_ready must be true", + ) + validate_target_rows(probes, summary, errors) + if install: + validate_install_simulation_report(install, errors) diff --git a/scripts/world_class_native_telemetry_evidence.py b/scripts/world_class_native_telemetry_evidence.py new file mode 100644 index 00000000..6455ff6e --- /dev/null +++ b/scripts/world_class_native_telemetry_evidence.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +from typing import Any + +from output_review_privacy import forbidden_decision_field_paths + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate native client telemetry evidence from metadata event rows." + +ADOPTION_EVENTS = {"skill_activation", "skill_output"} + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def object_list(value: Any) -> list[dict[str, Any]]: + return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] + + +def validate_no_raw_fields(payload: dict[str, Any], label: str, errors: list[str]) -> None: + blocked = forbidden_decision_field_paths(payload, label) + add_error( + errors, + not blocked, + f"native-client-telemetry {label} must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: " + + ", ".join(blocked[:8]), + ) + + +def source_types(payload: dict[str, Any]) -> dict[str, Any]: + summary = payload.get("summary", {}) if isinstance(payload.get("summary"), dict) else {} + value = summary.get("source_types", {}) + return value if isinstance(value, dict) else {} + + +def validate_adoption_report(adoption: dict[str, Any], errors: list[str]) -> None: + summary = adoption.get("summary", {}) if isinstance(adoption.get("summary"), dict) else {} + privacy = adoption.get("privacy_contract", {}) if isinstance(adoption.get("privacy_contract"), dict) else {} + events = object_list(adoption.get("recent_events", [])) + adoption_rows = object_list(adoption.get("adoption_by_skill", [])) + external_summary_count = real_int(source_types(adoption).get("external")) + adoption_sample_count = real_int(summary.get("adoption_sample_count")) + + validate_no_raw_fields(adoption, "adoption_drift_report", errors) + add_error(errors, adoption.get("ok") is True, "native-client-telemetry adoption drift report ok must be true") + add_error( + errors, + bool(external_summary_count and external_summary_count > 0), + "native-client-telemetry adoption drift summary.source_types.external must be >0", + ) + add_error( + errors, + bool(adoption_sample_count and adoption_sample_count > 0), + "native-client-telemetry adoption drift summary.adoption_sample_count must be >0", + ) + add_error( + errors, + privacy.get("raw_content_allowed") is False, + "native-client-telemetry adoption drift privacy_contract.raw_content_allowed must be false", + ) + add_error( + errors, + privacy.get("raw_event_log_packaged") is False, + "native-client-telemetry adoption drift privacy_contract.raw_event_log_packaged must be false", + ) + add_error(errors, bool(events), "native-client-telemetry adoption drift recent_events must not be empty") + external_events = [item for item in events if item.get("source") == "external"] + external_adoption_events = [item for item in external_events if item.get("event") in ADOPTION_EVENTS] + add_error( + errors, + bool(external_summary_count and len(external_events) >= external_summary_count), + "native-client-telemetry external event rows must cover summary.source_types.external", + ) + add_error( + errors, + bool(external_adoption_events), + "native-client-telemetry must include at least one external adoption event row", + ) + add_error( + errors, + bool( + adoption_sample_count + and len([item for item in events if item.get("event") in ADOPTION_EVENTS]) >= adoption_sample_count + ), + "native-client-telemetry adoption event rows must cover summary.adoption_sample_count", + ) + add_error( + errors, + any((count := real_int(row.get("adoption_events"))) is not None and count > 0 for row in adoption_rows), + "native-client-telemetry adoption_by_skill must include at least one adoption event", + ) + + +def validate_recipe_report(recipes: dict[str, Any], errors: list[str]) -> None: + summary = recipes.get("summary", {}) if isinstance(recipes.get("summary"), dict) else {} + privacy = recipes.get("privacy_contract", {}) if isinstance(recipes.get("privacy_contract"), dict) else {} + recipe_rows = object_list(recipes.get("recipes", [])) + recipe_count = real_int(summary.get("recipe_count")) + metadata_count = real_int(summary.get("metadata_only_recipe_count")) + + validate_no_raw_fields(recipes, "telemetry_hook_recipes", errors) + add_error(errors, recipes.get("ok") is True, "native-client-telemetry hook recipes report ok must be true") + add_error( + errors, + bool(metadata_count and metadata_count > 0), + "native-client-telemetry hook recipes summary.metadata_only_recipe_count must be >0", + ) + add_error( + errors, + privacy.get("raw_content_allowed") is False, + "native-client-telemetry hook recipes privacy_contract.raw_content_allowed must be false", + ) + add_error(errors, bool(recipe_rows), "native-client-telemetry hook recipes rows must not be empty") + if recipe_count is not None: + add_error(errors, len(recipe_rows) == recipe_count, "native-client-telemetry hook recipes length must equal summary.recipe_count") + if metadata_count is not None: + add_error( + errors, + sum(1 for item in recipe_rows if item.get("metadata_only") is True) == metadata_count, + "native-client-telemetry metadata_only recipe rows must match summary.metadata_only_recipe_count", + ) + add_error( + errors, + all(item.get("source") == "external" for item in recipe_rows), + "native-client-telemetry hook recipe rows must use source external", + ) + add_error( + errors, + all(item.get("native_auto_capture") is False for item in recipe_rows), + "native-client-telemetry hook recipe rows must not claim native_auto_capture", + ) + + +def validate_native_telemetry_report(adoption: dict[str, Any], recipes: dict[str, Any], errors: list[str]) -> None: + if adoption: + validate_adoption_report(adoption, errors) + if recipes: + validate_recipe_report(recipes, errors) diff --git a/scripts/world_class_operator_runbook_coordination.py b/scripts/world_class_operator_runbook_coordination.py new file mode 100644 index 00000000..87d3eac0 --- /dev/null +++ b/scripts/world_class_operator_runbook_coordination.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +import json +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Builds world-class operator coordination steps and release gates for render_world_class_operator_runbook.py." + + +def load_json(path: Path) -> dict[str, Any]: + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {} + + +COORDINATION_GUIDANCE: dict[str, dict[str, str]] = { + "provider-holdout": { + "phase": "provider-holdout", + "owner": "assistant + operator with provider credentials", + "user_action": "Provide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.", + "assistant_action": "Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.", + "external_dependency": "Valid provider credentials and a live provider endpoint.", + "command": "python3 scripts/yao.py output-exec . --provider-runner --provider-model --timeout-seconds 60", + "pass_condition": "reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.", + "artifact": "reports/output_execution_runs.json", + "privacy_boundary": "Commit aggregate metadata only; do not commit API keys, raw prompts, raw outputs, or provider request payloads.", + }, + "human-adjudication": { + "phase": "human-review", + "owner": "human reviewer + assistant", + "user_action": "Open the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved.", + "assistant_action": "Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet.", + "external_dependency": "A real human reviewer who can make blind A/B judgments.", + "command": "python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .", + "pass_condition": "reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true.", + "artifact": "reports/output_review_adjudication.json", + "privacy_boundary": "Store reviewer choices, reasons, hashes, and attestations; do not store raw prompts or answer-key exposure before decisions.", + }, + "native-permission-enforcement": { + "phase": "native-permission", + "owner": "target client or installer integrator + assistant", + "user_action": "Select a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback.", + "assistant_action": "Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet.", + "external_dependency": "A real target runtime, extension, client, or installer guard that can block undeclared capabilities.", + "command": "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "pass_condition": "reports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0.", + "artifact": "reports/runtime_permission_probes.json", + "privacy_boundary": "Do not mark metadata fallback as native enforcement; keep residual risks visible for fallback targets.", + }, + "native-client-telemetry": { + "phase": "native-telemetry", + "owner": "real client integrator + assistant", + "user_action": "Install the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event.", + "assistant_action": "Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet.", + "external_dependency": "A real client process that emits metadata-only telemetry through the native host.", + "command": "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source external", + "pass_condition": "reports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0.", + "artifact": "reports/adoption_drift_report.json", + "privacy_boundary": "Telemetry must remain metadata-only; do not store raw prompts, outputs, transcripts, notes, or messages.", + }, +} + + +def build_coordination_plan(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + pending_keys = [str(item.get("evidence_key", "")) for item in items if item.get("ledger_status") != "accepted"] + plan: list[dict[str, Any]] = [ + { + "step_id": "prepare-evidence-session", + "phase": "prepare", + "evidence_key": "", + "owner": "assistant + user", + "requires_user_input": True, + "user_action": "Confirm provider access, reviewer availability, target client path, and telemetry client path before collection starts.", + "assistant_action": "Run preflight and prepare submission drafts without accepting them as evidence.", + "external_dependency": "User-selected provider, reviewer, and real client surfaces.", + "command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "pass_condition": "Preflight lists the same pending evidence keys and no credential values are printed.", + "artifact": "reports/world_class_evidence_preflight.json", + "privacy_boundary": "Drafts and preflight are planning artifacts only; they do not count as evidence.", + "counts_as_completion": False, + } + ] + for item in items: + key = str(item.get("evidence_key", "")) + if key not in pending_keys: + continue + guidance = COORDINATION_GUIDANCE.get(key, {}) + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + commands = item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {} + plan.append( + { + "step_id": f"collect-{key}", + "phase": guidance.get("phase", "collect-source"), + "evidence_key": key, + "owner": guidance.get("owner", item.get("owner", "release reviewer")), + "requires_user_input": True, + "user_action": guidance.get("user_action", item.get("blocking_reason", "")), + "assistant_action": guidance.get("assistant_action", "Validate source evidence and prepare the submission packet."), + "external_dependency": guidance.get("external_dependency", "Real external or human evidence source."), + "command": guidance.get("command", commands.get("prepare_submission", "")), + "pass_condition": guidance.get("pass_condition", "; ".join(must_collect.get("success_checks", []))), + "artifact": guidance.get("artifact", item.get("submission_path", "")), + "privacy_boundary": guidance.get("privacy_boundary", "; ".join(must_collect.get("privacy_contract", []))), + "counts_as_completion": False, + } + ) + plan.append( + { + "step_id": "review-and-release-gate", + "phase": "release-gate", + "evidence_key": "", + "owner": "assistant + ledger reviewer", + "requires_user_input": True, + "user_action": "Approve only validated evidence packets and confirm the release wording after the claim guard passes.", + "assistant_action": "Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish.", + "external_dependency": "Accepted evidence packets for every pending world-class key.", + "command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-test", + "pass_condition": "Ledger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes.", + "artifact": "reports/world_class_evidence_ledger.json", + "privacy_boundary": "Release evidence may include aggregate reports and hashes only; raw content and credentials remain out of git.", + "counts_as_completion": False, + } + ) + return plan + + +def build_release_gate(skill_dir: Path, ledger: dict[str, Any]) -> dict[str, Any]: + reports_dir = skill_dir / "reports" + ledger_summary = ledger.get("summary", {}) if isinstance(ledger.get("summary", {}), dict) else {} + claim_guard = load_json(reports_dir / "world_class_claim_guard.json") + benchmark = load_json(reports_dir / "benchmark_reproducibility.json") + review_studio = load_json(reports_dir / "review-studio.json") + evidence_consistency = load_json(reports_dir / "evidence_consistency.json") + claim_summary = claim_guard.get("summary", {}) if isinstance(claim_guard.get("summary", {}), dict) else {} + benchmark_summary = benchmark.get("summary", {}) if isinstance(benchmark.get("summary", {}), dict) else {} + studio_summary = review_studio.get("summary", {}) if isinstance(review_studio.get("summary", {}), dict) else {} + consistency_summary = ( + evidence_consistency.get("summary", {}) + if isinstance(evidence_consistency.get("summary", {}), dict) + else {} + ) + checks = [ + { + "key": "world_class_ledger_ready", + "label": "World-class ledger ready", + "passed": ledger_summary.get("ready_to_claim_world_class") is True, + "current": ledger_summary.get("decision", "missing"), + "expected": "ready_to_claim_world_class == true", + "artifact": "reports/world_class_evidence_ledger.json", + }, + { + "key": "claim_guard_clean", + "label": "Claim guard clean", + "passed": claim_summary.get("violation_count") == 0 + and claim_summary.get("ledger_ready_to_claim_world_class") is True, + "current": ( + f"violations {claim_summary.get('violation_count', 'n/a')}; " + f"ledger ready {claim_summary.get('ledger_ready_to_claim_world_class', 'n/a')}" + ), + "expected": "violation_count == 0 and ledger_ready_to_claim_world_class == true", + "artifact": "reports/world_class_claim_guard.json", + }, + { + "key": "benchmark_public_claim_ready", + "label": "Benchmark public claim ready", + "passed": benchmark_summary.get("public_claim_ready") is True, + "current": f"public_claim_ready {benchmark_summary.get('public_claim_ready', 'n/a')}", + "expected": "public_claim_ready == true", + "artifact": "reports/benchmark_reproducibility.json", + }, + { + "key": "review_studio_clean", + "label": "Review Studio clean", + "passed": int(studio_summary.get("blocker_count", 1) or 0) == 0 + and int(studio_summary.get("warning_count", 1) or 0) == 0, + "current": ( + f"blockers {studio_summary.get('blocker_count', 'n/a')}; " + f"warnings {studio_summary.get('warning_count', 'n/a')}" + ), + "expected": "blocker_count == 0 and warning_count == 0", + "artifact": "reports/review-studio.json", + }, + { + "key": "evidence_consistency_clean", + "label": "Evidence consistency clean", + "passed": consistency_summary.get("decision") == "consistent" + and int(consistency_summary.get("fail_count", 1) or 0) == 0, + "current": consistency_summary.get("decision", "missing"), + "expected": "decision == consistent and fail_count == 0", + "artifact": "reports/evidence_consistency.json", + }, + ] + blocked = [item for item in checks if not item["passed"]] + return { + "ready": not blocked, + "decision": "ready-for-public-claim" if not blocked else "blocked-until-evidence-accepted", + "check_count": len(checks), + "blocked_count": len(blocked), + "checks": checks, + "blocked_checks": blocked, + "final_manual_check": "Run make ci-test in a clean worktree and verify GitHub Actions before converting the PR out of Draft.", + "counts_as_completion": False, + } diff --git a/scripts/world_class_phase_queue.py b/scripts/world_class_phase_queue.py new file mode 100644 index 00000000..ef88cc80 --- /dev/null +++ b/scripts/world_class_phase_queue.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Build phase-ordered operator queues for world-class evidence collection.""" + +from collections import defaultdict +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by prepare_world_class_submission_kit.py to group repair rows into an operator execution queue." + + +PHASE_LABELS = { + "select-evidence": "Select evidence", + "prepare-draft": "Prepare draft", + "unblock-access": "Unblock access", + "attach-artifacts": "Attach artifacts", + "collect-source": "Collect source", + "repair": "Repair", +} + + +def _phase_sort_key(row: dict[str, Any]) -> tuple[int, str, str]: + return ( + int(row.get("priority", 90) or 90), + str(row.get("evidence_key", "")), + str(row.get("target", "")), + ) + + +def build_phase_queue(repair_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Group repair rows by execution phase without changing evidence acceptance.""" + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in sorted(repair_rows, key=_phase_sort_key): + phase = str(row.get("phase", "repair") or "repair") + grouped[phase].append(row) + + queue: list[dict[str, Any]] = [] + for phase_rows in sorted(grouped.values(), key=lambda rows: _phase_sort_key(rows[0])): + first = phase_rows[0] + phase = str(first.get("phase", "repair") or "repair") + blocked_rows = [row for row in phase_rows if row.get("status") != "ready"] + owners = sorted({str(row.get("owner", "")).strip() for row in phase_rows if str(row.get("owner", "")).strip()}) + evidence_keys = sorted( + {str(row.get("evidence_key", "")).strip() for row in phase_rows if str(row.get("evidence_key", "")).strip()} + ) + queue.append( + { + "phase": phase, + "label": PHASE_LABELS.get(phase, phase.replace("-", " ").title()), + "priority": int(first.get("priority", 90) or 90), + "status": "blocked" if blocked_rows else "ready", + "blocked_count": len(blocked_rows), + "row_count": len(phase_rows), + "owners": owners, + "evidence_keys": evidence_keys, + "next_action_id": first.get("action_id", ""), + "next_action": first.get("next_action", ""), + "verification_command": first.get("verification_command", ""), + "counts_as_completion": False, + "rows": phase_rows, + } + ) + return queue + + +def summarize_phase_queue(queue: list[dict[str, Any]]) -> dict[str, Any]: + next_item = next((item for item in queue if item.get("status") != "ready"), queue[0] if queue else {}) + return { + "phase_queue_count": len(queue), + "phase_queue_blocked_count": sum(1 for item in queue if item.get("status") != "ready"), + "phase_queue_row_count": sum(int(item.get("row_count", 0) or 0) for item in queue), + "phase_queue_next_phase": next_item.get("phase", ""), + "phase_queue_next_action_id": next_item.get("next_action_id", ""), + "phase_queue_next_command": next_item.get("verification_command", ""), + "phase_queue_counts_as_completion": False, + } diff --git a/scripts/world_class_preflight_layout.py b/scripts/world_class_preflight_layout.py new file mode 100644 index 00000000..c17fbf40 --- /dev/null +++ b/scripts/world_class_preflight_layout.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""HTML layout helpers for the world-class evidence preflight report.""" + +from typing import Any + +from html_rendering import html_text + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly." + + +def html_list(values: list[Any], empty: str) -> str: + if not values: + return f"
  • {html_text(empty)}
    • " + return "".join(f"
  • {html_text(value)}
    • " for value in values) + + +def render_html_commands(commands: dict[str, str]) -> str: + return "".join( + f"
  • {html_text(label.replace('_', ' '))}{html_text(command)}
    • " + for label, command in commands.items() + ) + + +def render_html_prechecks(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No prechecks listed.

    " + return "".join( + """ +
    +
    + {kind} + {label} +
    +
    +
    Current
    {actual}
    +
    Status
    {status}
    +
    Action
    {action}
    +
    +
    + """.format( + status=html_text(row.get("status", "")), + kind=html_text(row.get("kind", "")), + label=html_text(row.get("label", "")), + actual=html_text(row.get("actual", "")), + action=html_text(row.get("next_action", "")), + ) + for row in rows + ) + + +def render_html_source_checks(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No source checks listed.

    " + return "".join( + """ +
    +
    + {field} + {label} +
    +
    +
    Current
    {actual}
    +
    Expected
    {expected}
    +
    Status
    {status}
    +
    Action
    {action}
    +
    +
    + """.format( + status=html_text(row.get("status", "")), + field=html_text(row.get("field", "")), + label=html_text(row.get("label", "")), + actual=html_text(row.get("actual", "")), + expected=html_text(row.get("expected", "")), + action=html_text(row.get("next_action", "")), + ) + for row in rows + ) + + +def render_html_repair_rows(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No repair rows listed.

    " + return "".join( + """ +
    +
    + {repair_type} + {target} +
    +
    +
    Priority
    {priority}
    +
    Phase
    {phase}
    +
    Owner
    {owner}
    +
    Status
    {status}
    +
    Reason
    {reason}
    +
    Action
    {action}
    +
    Verify
    {verify}
    +
    Evidence
    does not count as completion
    +
    +
    + """.format( + status=html_text(row.get("status", "")), + repair_type=html_text(row.get("repair_type", "")), + target=html_text(row.get("target", "")), + priority=html_text(row.get("priority", "")), + phase=html_text(row.get("phase", "")), + owner=html_text(row.get("owner", "")), + reason=html_text(row.get("blocking_reason", "")), + action=html_text(row.get("next_action", "")), + verify=html_text(row.get("verification_command", "")), + ) + for row in rows + ) + + +def render_html_phase_queue(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No phase queue rows listed.

    " + return "".join( + """ +
    +
    + #{priority} · {status} +

    {label}

    +
    +
    +
    Phase
    {phase}
    +
    Rows
    {blocked} / {total} blocked
    +
    Owners
    {owners}
    +
    Evidence
    {evidence}
    +
    Next
    {action}
    +
    Verify
    {verify}
    +
    Counts
    does not count as completion
    +
    +
    + """.format( + status=html_text(row.get("status", "")), + priority=html_text(row.get("priority", "")), + label=html_text(row.get("label", "")), + phase=html_text(row.get("phase", "")), + blocked=html_text(row.get("blocked_count", 0)), + total=html_text(row.get("row_count", 0)), + owners=html_text(", ".join(str(owner) for owner in row.get("owners", [])) or "n/a"), + evidence=html_text(", ".join(str(key) for key in row.get("evidence_keys", [])) or "n/a"), + action=html_text(row.get("next_action", "")), + verify=html_text(row.get("verification_command", "")), + ) + for row in rows + ) + + +def render_html_artifact_roles(contract: dict[str, Any]) -> str: + cards = [] + for role in contract.get("roles", []): + role_name = str(role.get("role", "")) + if role_name == "submission-ref": + ready = f"{contract.get('submission_ref_ready_count', 0)}/{contract.get('submission_ref_total_count', 0)} ready" + else: + ready = ( + f"{contract.get('supporting_evidence_ready_count', 0)}/" + f"{contract.get('supporting_evidence_total_count', 0)} ready" + ) + cards.append( + """ +
    + {label} +

    {role}

    + {ready} +

    {description}

    + copy to artifact_refs: {copy} +
    + """.format( + label=html_text(role.get("label", "")), + role=html_text(role_name), + ready=html_text(ready), + description=html_text(role.get("description", "")), + copy=html_text(str(role.get("copy_to_artifact_refs") is True).lower()), + ) + ) + return "".join(cards) + + +def render_html_item(item: dict[str, Any]) -> str: + role_contract = item.get("submission_kit", {}).get("artifact_role_contract", {}) + return f""" +
    +
    + {html_text(item.get('category', ''))} · {html_text(item.get('status', ''))} +

    {html_text(item.get('label', item.get('evidence_key', '')))}

    +
    +
    +
    Evidence
    {html_text(item.get('evidence_key', ''))}
    +
    Ledger
    {html_text(item.get('ledger_status', ''))}
    +
    Intake
    {html_text(item.get('intake_readiness', ''))}
    +
    Review
    {html_text(item.get('review_state', ''))}
    +
    Draft
    {html_text(item.get('submission_path', ''))}
    +
    +
    +

    Next Action

    +

    {html_text(item.get('next_action', ''))}

    + {html_text(item.get('commands', {}).get('prepare_submission', ''))} + {html_text(item.get('commands', {}).get('prepare_prefilled_submission', ''))} +
    +
    +

    Artifact Roles

    +
    {render_html_artifact_roles(role_contract)}
    +
    +
    +

    Prechecks

    +
    {render_html_prechecks(item.get('prechecks', []))}
    +
    +
    +

    Source Checks

    +
    {render_html_source_checks(item.get('source_checklist', []))}
    +
    +
    +

    Repair Rows

    +
    {render_html_repair_rows(item.get('repair_checklist', []))}
    +
    +
    +

    Runbook

    +
      {html_list(item.get('runbook', []), 'No runbook steps listed.')}
    +
    +
    + """ + + +def render_html(report: dict[str, Any]) -> str: + summary = report["summary"] + stats = [ + ("Decision", summary["decision"]), + ("Pending", summary["pending_count"]), + ("Ready", summary["collection_ready_count"]), + ("Blocked", summary["collection_blocked_count"]), + ("Source", f"{summary['source_pass_count']}/{summary['source_check_count']}"), + ("Repairs", f"{summary.get('repair_blocked_count', 0)}/{summary.get('repair_checklist_count', 0)}"), + ("Phases", f"{summary.get('phase_queue_blocked_count', 0)}/{summary.get('phase_queue_count', 0)}"), + ] + stat_html = "".join( + f"
    {html_text(label)}{html_text(value)}
    " + for label, value in stats + ) + role_contract = report["submissions"]["artifact_role_contract"] + item_cards = "".join(render_html_item(item) for item in report.get("items", [])) + phase_queue_html = render_html_phase_queue(report.get("phase_queue", [])) + html = f""" + + + + + World-Class Evidence Preflight + + + + +
    +
    + Evidence Collection +

    World-Class Evidence Preflight

    +

    This operator view shows which external and human evidence is still blocked, which commands prepare editable submission drafts, and why preflight never counts as accepted evidence.

    +
    {stat_html}
    +
    +
    +
    +

    Submission Kit

    +

    Generate drafts only after real provider, human-review, native-permission, or native-client work exists. Drafts remain non-evidence until valid aggregate artifact refs and SHA-256 digests are supplied. Artifact prefill is convenience data only.

    +
      +
    • submissions directory: {html_text(report['submissions']['directory'])}
      • +
    • drafts count as evidence: {html_text(str(report['submissions']['drafts_count_as_evidence']).lower())}
      • +
    • artifact prefill counts as evidence: {html_text(str(report['submissions']['artifact_prefill_counts_as_evidence']).lower())}
      • +
    • preflight accepts evidence: {html_text(str(report['summary']['preflight_counts_as_evidence']).lower())}
      • +
    +
    {render_html_artifact_roles(role_contract)}
    +

    submission-ref rows are the paths expected in artifact_refs; supporting-evidence rows stay available for audit context.

    +
    + +
    +
    +

    Phase Queue

    +

    Repair rows are grouped into execution phases so operators can clear the first blocker before moving to later source collection. Phase queue rows remain procedural and never count as accepted evidence.

    +
    {phase_queue_html}
    +

    Evidence Queue

    +
    {item_cards}
    +
    +
    +

    Safety Boundary

    +
    • Environment variables are displayed only as set or not-set; secret values are never printed.
    • Human-required and external-required states are operator work, not accepted evidence.
    • The world-class ledger remains the only source of truth for ready_to_claim_world_class.
    +
    +
    + + +""" + return "\n".join(line.rstrip() for line in html.splitlines()) + "\n" diff --git a/scripts/world_class_preflight_markdown.py b/scripts/world_class_preflight_markdown.py new file mode 100644 index 00000000..da074e01 --- /dev/null +++ b/scripts/world_class_preflight_markdown.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Markdown renderer for the world-class evidence preflight report.""" + +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_world_class_preflight.py to keep Markdown layout separate from preflight data assembly." + + +def md_cell(value: Any) -> str: + return str(value).replace("|", "\\|") + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + role_contract = report["submissions"]["artifact_role_contract"] + lines = [ + "# World-Class Evidence Preflight", + "", + f"Generated at: `{report['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- preflight counts as evidence: `{str(summary['preflight_counts_as_evidence']).lower()}`", + f"- credential value exposed: `{str(summary['credential_value_exposed']).lower()}`", + f"- collection ready: `{summary['collection_ready_count']}`", + f"- collection blocked: `{summary['collection_blocked_count']}`", + f"- source checks: `{summary['source_pass_count']}` pass / `{summary['source_check_count']}` total", + f"- repair rows: `{summary['repair_blocked_count']}` blocked / `{summary['repair_checklist_count']}` total", + f"- phase queue: `{summary['phase_queue_blocked_count']}` blocked / `{summary['phase_queue_count']}` phases", + f"- phase queue rows: `{summary['phase_queue_row_count']}`", + f"- next repair action: `{summary.get('next_repair_action_id', '')}`", + f"- next repair owner: `{summary.get('next_repair_owner', '')}`", + f"- next phase: `{summary.get('phase_queue_next_phase', '')}`", + f"- next phase action: `{summary.get('phase_queue_next_action_id', '')}`", + "", + "This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.", + "", + "## Submission Kit Handoff", + "", + f"- submissions directory: `{report['submissions']['directory']}`", + f"- prepare drafts: `{report['submissions']['commands']['prepare_submission']}`", + f"- prepare drafts with artifact SHA prefill: `{report['submissions']['commands']['prepare_prefilled_submission']}`", + f"- validate intake: `{report['submissions']['commands']['validate_intake']}`", + f"- review queue: `{report['submissions']['commands']['submission_review']}`", + f"- refresh ledger: `{report['submissions']['commands']['refresh_ledger']}`", + f"- guard claims: `{report['submissions']['commands']['guard_claim']}`", + f"- drafts count as evidence: `{str(report['submissions']['drafts_count_as_evidence']).lower()}`", + f"- artifact prefill counts as evidence: `{str(report['submissions']['artifact_prefill_counts_as_evidence']).lower()}`", + f"- submission refs ready: `{role_contract['submission_ref_ready_count']}` / `{role_contract['submission_ref_total_count']}`", + f"- supporting evidence ready: `{role_contract['supporting_evidence_ready_count']}` / `{role_contract['supporting_evidence_total_count']}`", + "", + "Generate the submission kit after the real provider, human, native-permission, or native-client work exists. The generated JSON drafts remain `template_only: true` until an operator edits them with real aggregate artifact references and matching SHA-256 digests. The prefill command only inserts local artifact SHA-256 digests; it does not make a draft count as evidence.", + "", + "| Role | Copy to artifact_refs | Ready | Meaning |", + "| --- | --- | --- | --- |", + ] + for role in role_contract.get("roles", []): + if role.get("role") == "submission-ref": + ready = f"{role_contract['submission_ref_ready_count']} / {role_contract['submission_ref_total_count']}" + else: + ready = ( + f"{role_contract['supporting_evidence_ready_count']} / " + f"{role_contract['supporting_evidence_total_count']}" + ) + lines.append( + f"| `{role['role']}` | `{str(role['copy_to_artifact_refs']).lower()}` | `{ready}` | " + f"{md_cell(role['description'])} |" + ) + lines.extend( + [ + "", + "`submission-ref` rows are the only checklist rows expected in `artifact_refs`; `supporting-evidence` rows stay available for audit context and reviewer traceability.", + "", + "## Phase Queue", + "", + "Phase queue rows group the same repair checklist into operator execution phases. They are procedural guidance only and do not count as completion evidence.", + "", + "| Priority | Phase | Status | Rows | Owners | Evidence | Verify | Next action |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + ) + for row in report.get("phase_queue", []): + owners = ", ".join(str(owner) for owner in row.get("owners", [])) + evidence_keys = ", ".join(str(key) for key in row.get("evidence_keys", [])) + rows = f"{row.get('blocked_count', 0)} / {row.get('row_count', 0)} blocked" + lines.append( + f"| `{row.get('priority', '')}` | `{row.get('phase', '')}` | `{row.get('status', '')}` | " + f"{md_cell(rows)} | {md_cell(owners)} | {md_cell(evidence_keys)} | " + f"`{md_cell(row.get('verification_command', ''))}` | {md_cell(row.get('next_action', ''))} |" + ) + lines.extend( + [ + "", + "## Evidence Items", + "", + "| Evidence | Status | Intake | Review | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + for item in report["items"]: + next_action = str(item.get("next_action", "")).replace("|", "\\|") + lines.append( + f"| `{item['evidence_key']}` | `{item['status']}` | `{item['intake_readiness']}` | `{item['review_state']}` | {next_action} |" + ) + lines.extend( + [ + "", + "## Repair Checklist", + "", + "Repair rows convert preflight and source blockers into a prioritized operator queue. They are guidance only and do not count as completion evidence.", + "", + "| Priority | Phase | Owner | Evidence | Type | Target | Status | Verify | Next action |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + ) + for row in report.get("repair_checklist", []): + lines.append( + f"| `{row.get('priority', '')}` | `{row.get('phase', '')}` | {md_cell(row.get('owner', ''))} | " + f"`{row['evidence_key']}` | `{row['repair_type']}` | `{row['target']}` | " + f"`{row['status']}` | `{md_cell(row.get('verification_command', ''))}` | {md_cell(row['next_action'])} |" + ) + for item in report["items"]: + lines.extend( + [ + "", + f"## {item['label']}", + "", + f"- status: `{item['status']}`", + f"- ledger: `{item['ledger_status']}`", + f"- submission: `{item.get('submission_path') or 'missing'}`", + f"- prepare draft: `{item['commands']['prepare_submission']}`", + f"- prepare draft with artifact SHA prefill: `{item['commands']['prepare_prefilled_submission']}`", + f"- submission refs ready: `{item['submission_kit']['artifact_role_contract']['submission_ref_ready_count']}` / `{item['submission_kit']['artifact_role_contract']['submission_ref_total_count']}`", + f"- supporting evidence ready: `{item['submission_kit']['artifact_role_contract']['supporting_evidence_ready_count']}` / `{item['submission_kit']['artifact_role_contract']['supporting_evidence_total_count']}`", + "", + "### Prechecks", + "", + "| Check | Kind | Current | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in item.get("prechecks", []): + action = md_cell(row.get("next_action", "")) + lines.append( + f"| {row['label']} | `{row['kind']}` | `{row['actual']}` | `{row['status']}` | {action} |" + ) + lines.extend( + [ + "", + "### Source Checks", + "", + "| Check | Current | Expected | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + for row in item.get("source_checklist", []): + action = md_cell(row.get("next_action", "")) + lines.append( + f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` | {action} |" + ) + lines.extend( + [ + "", + "## Boundary", + "", + "- Environment variables are reported only as `set` or `not-set`; values are never printed.", + "- Human-required and external-required states are operator actions, not accepted evidence.", + "- The world-class ledger remains the source of truth for `ready_to_claim_world_class`.", + ] + ) + return "\n".join(lines).rstrip() + "\n" diff --git a/scripts/world_class_provider_evidence.py b/scripts/world_class_provider_evidence.py new file mode 100644 index 00000000..271e35ed --- /dev/null +++ b/scripts/world_class_provider_evidence.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +import re +from typing import Any + +from output_review_privacy import forbidden_decision_field_paths + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows." + +SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def summary(payload: dict[str, Any]) -> dict[str, Any]: + value = payload.get("summary", {}) + return value if isinstance(value, dict) else {} + + +def normalized(value: Any) -> str: + return str(value or "").strip().casefold() + + +def run_rows(execution: dict[str, Any]) -> list[dict[str, Any]]: + runs = execution.get("runs", []) + return [item for item in runs if isinstance(item, dict)] if isinstance(runs, list) else [] + + +def validate_no_raw_fields(execution: dict[str, Any], errors: list[str]) -> None: + blocked = forbidden_decision_field_paths(execution, "output_execution_runs") + add_error( + errors, + not blocked, + "provider-holdout output execution report must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: " + + ", ".join(blocked[:8]), + ) + + +def positive_duration(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 + + +def observed_usage(run: dict[str, Any]) -> bool: + usage = run.get("usage", {}) + if not isinstance(usage, dict) or usage.get("estimated") is not False: + return False + token_values = [ + real_int(usage.get("input_tokens")), + real_int(usage.get("output_tokens")), + real_int(usage.get("total_tokens")), + ] + return all(value is not None and value > 0 for value in token_values) + + +def provider_model_runs(execution: dict[str, Any], provenance: dict[str, Any]) -> list[dict[str, Any]]: + expected_provider = normalized(provenance.get("provider")) + expected_model = normalized(provenance.get("model")) + matches: list[dict[str, Any]] = [] + for run in run_rows(execution): + if run.get("status") != "pass": + continue + if run.get("model_executed") is not True or run.get("execution_mode") != "model": + continue + if normalized(run.get("provider")) != expected_provider: + continue + if normalized(run.get("model")) != expected_model: + continue + if not observed_usage(run): + continue + if not positive_duration(run.get("duration_ms")): + continue + if not SHA256_RE.match(str(run.get("output_sha256", "")).strip()): + continue + matches.append(run) + return matches + + +def validate_provider_execution_report( + execution: dict[str, Any], + provenance: dict[str, Any], + errors: list[str], +) -> None: + execution_summary = summary(execution) + runs = run_rows(execution) + validate_no_raw_fields(execution, errors) + add_error(errors, execution.get("ok") is True, "provider-holdout output execution report ok must be true") + add_error(errors, bool(runs), "provider-holdout output execution report must include run rows") + + computed_model_runs = sum(1 for run in runs if run.get("model_executed") is True) + computed_timing = sum(1 for run in runs if run.get("duration_ms") is not None) + computed_token_observed = sum(1 for run in runs if isinstance(run.get("usage"), dict) and run["usage"].get("estimated") is False) + computed_failures = sum(1 for run in runs if run.get("status") != "pass") + add_error( + errors, + real_int(execution_summary.get("model_executed_count")) == computed_model_runs, + "provider-holdout output execution summary.model_executed_count must match model-executed run rows", + ) + add_error( + errors, + real_int(execution_summary.get("timing_observed_count")) == computed_timing, + "provider-holdout output execution summary.timing_observed_count must match timed run rows", + ) + add_error( + errors, + real_int(execution_summary.get("token_observed_count")) == computed_token_observed, + "provider-holdout output execution summary.token_observed_count must match non-estimated usage rows", + ) + add_error( + errors, + real_int(execution_summary.get("failure_count")) == computed_failures, + "provider-holdout output execution summary.failure_count must match failed run rows", + ) + add_error( + errors, + bool(real_int(execution_summary.get("model_executed_count")) and execution_summary["model_executed_count"] > 0), + "provider-holdout output execution summary.model_executed_count must be >0", + ) + add_error( + errors, + bool(real_int(execution_summary.get("timing_observed_count")) and execution_summary["timing_observed_count"] > 0), + "provider-holdout output execution summary.timing_observed_count must be >0", + ) + add_error( + errors, + bool(real_int(execution_summary.get("token_observed_count")) and execution_summary["token_observed_count"] > 0), + "provider-holdout output execution summary.token_observed_count must be >0", + ) + add_error( + errors, + execution_summary.get("failure_count") == 0, + "provider-holdout output execution summary.failure_count must be 0", + ) + add_error( + errors, + bool(provider_model_runs(execution, provenance)), + "provider-holdout output execution runs must include a passing model run with matching provider, model, timing, non-estimated usage, and output hash", + ) diff --git a/scripts/world_class_repair_checklist.py b/scripts/world_class_repair_checklist.py new file mode 100644 index 00000000..9e8d9134 --- /dev/null +++ b/scripts/world_class_repair_checklist.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""Build machine-readable repair checklists for world-class submission kits.""" + +from collections import defaultdict +import re +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Shared by submission kit generation to turn readiness blockers into actionable repair rows." + + +OWNER_BY_EVIDENCE = { + "provider-holdout": "operator with provider credentials", + "human-adjudication": "human reviewer", + "native-permission-enforcement": "target client or installer integrator", + "native-client-telemetry": "Browser/Chrome/IDE/provider client integrator", +} + +REPAIR_PHASES = { + "unknown-key": ("select-evidence", 5), + "draft": ("prepare-draft", 10), + "precheck": ("unblock-access", 20), + "artifact": ("attach-artifacts", 30), + "source-check": ("collect-source", 40), +} + + +def _by_key(rows: list[dict[str, Any]], key_name: str = "evidence_key") -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + key = str(row.get(key_name, "")).strip() + if key: + grouped[key].append(row) + return dict(grouped) + + +def _slug(value: str) -> str: + slug = re.sub(r"[^a-zA-Z0-9_.-]+", "-", value.strip()).strip("-") + return slug or "repair" + + +def repair_verification_command(evidence_key: str, repair_type: str) -> str: + preflight = "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" + if repair_type == "draft": + return ( + "python3 scripts/yao.py world-class-submission-kit . " + f"--evidence-key {evidence_key} --output-dir evidence/world_class/submissions" + ) + if repair_type == "artifact": + return ( + "python3 scripts/yao.py world-class-submission-kit . " + f"--evidence-key {evidence_key} --output-dir evidence/world_class/submissions --prefill-artifacts" + ) + if repair_type == "unknown-key": + return "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + if evidence_key == "provider-holdout" and repair_type == "source-check": + return ( + "python3 scripts/yao.py output-exec --provider-runner " + "--provider-model --timeout-seconds 60 && " + + preflight + ) + if evidence_key == "human-adjudication" and repair_type == "source-check": + return "python3 scripts/yao.py output-review && " + preflight + if evidence_key == "native-permission-enforcement" and repair_type == "source-check": + return "python3 scripts/yao.py runtime-permissions . --package-dir dist && " + preflight + if evidence_key == "native-client-telemetry" and repair_type == "source-check": + return ( + "python3 scripts/yao.py telemetry-import . " + "--input-jsonl .yao/telemetry_spool/external_events.jsonl && " + + preflight + ) + return preflight + + +def _repair_row( + *, + evidence_key: str, + repair_type: str, + target: str, + status: str, + blocking_reason: str, + next_action: str, + source: str, +) -> dict[str, Any]: + phase, priority = REPAIR_PHASES.get(repair_type, ("repair", 90)) + return { + "action_id": _slug(f"{evidence_key}:{repair_type}:{target}"), + "evidence_key": evidence_key, + "repair_type": repair_type, + "target": target, + "phase": phase, + "priority": priority, + "owner": OWNER_BY_EVIDENCE.get(evidence_key, "release reviewer"), + "status": status, + "blocking_reason": blocking_reason, + "next_action": next_action, + "verification_command": repair_verification_command(evidence_key, repair_type), + "source": source, + "counts_as_completion": False, + } + + +def sort_repair_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return sorted( + rows, + key=lambda row: ( + int(row.get("priority", 90) or 90), + str(row.get("evidence_key", "")), + str(row.get("target", "")), + ), + ) + + +def build_repair_checklist( + evidence_items: list[dict[str, Any]], + files: list[dict[str, Any]], + artifact_checklist: list[dict[str, Any]], + source_checklist: list[dict[str, Any]], + unknown_keys: list[str], +) -> list[dict[str, Any]]: + """Return concrete repair rows for every draft, artifact, and source blocker.""" + files_by_key = _by_key(files) + artifacts_by_key = _by_key(artifact_checklist) + sources_by_key = _by_key(source_checklist) + rows: list[dict[str, Any]] = [] + known_keys = [str(item.get("evidence_key", "")).strip() for item in evidence_items] + + for key in known_keys: + if not key: + continue + draft_rows = files_by_key.get(key, []) + if not draft_rows: + rows.append( + _repair_row( + evidence_key=key, + repair_type="draft", + target=key, + status="blocked", + blocking_reason="Submission draft was not generated.", + next_action="Run the submission kit command again for this evidence key.", + source="files", + ) + ) + for draft in draft_rows: + draft_status = str(draft.get("status", "")).strip() + if draft_status not in {"written", "exists"}: + target = str(draft.get("output_path") or draft.get("template_path") or key) + errors = draft.get("errors", []) + error_text = "; ".join(str(error) for error in errors) if isinstance(errors, list) else "" + rows.append( + _repair_row( + evidence_key=key, + repair_type="draft", + target=target, + status="blocked", + blocking_reason=error_text or f"Draft status is {draft_status or 'unknown'}.", + next_action="Fix template generation before asking for ledger review.", + source="files", + ) + ) + + for artifact in artifacts_by_key.get(key, []): + if artifact.get("artifact_ref_ready"): + continue + target = str(artifact.get("path") or artifact.get("source_pattern") or key) + artifact_status = str(artifact.get("status", "missing")) + role = str(artifact.get("artifact_role", "supporting-evidence")) + if artifact.get("concrete_reference_required"): + action = "Replace the glob with concrete files, then reference the generated SHA-256 digests." + elif artifact.get("submission_ref_required"): + action = "Create the required submission artifact or update artifact_refs to a concrete existing aggregate file." + else: + action = "Add the supporting artifact if it is needed for reviewer audit." + rows.append( + _repair_row( + evidence_key=key, + repair_type="artifact", + target=target, + status="blocked", + blocking_reason=f"{role} artifact is {artifact_status}.", + next_action=action, + source="artifact_checklist", + ) + ) + + for source in sources_by_key.get(key, []): + if source.get("status") == "pass": + continue + field = str(source.get("field") or source.get("label") or key) + rows.append( + _repair_row( + evidence_key=key, + repair_type="source-check", + target=field, + status="blocked", + blocking_reason=f"Current value {source.get('actual')!r} does not satisfy {source.get('expected')!r}.", + next_action=str(source.get("next_action") or "Collect the required source evidence."), + source="source_checklist", + ) + ) + + for key in unknown_keys: + rows.append( + _repair_row( + evidence_key=key, + repair_type="unknown-key", + target=key, + status="blocked", + blocking_reason="Requested evidence key is not present in the operator checklist.", + next_action="Use one of the evidence keys listed by world-class-intake.", + source="unknown_evidence_keys", + ) + ) + + return sort_repair_rows(rows) + + +def build_preflight_repair_checklist(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Return repair rows from preflight precheck and source-check blockers.""" + rows: list[dict[str, Any]] = [] + for item in items: + key = str(item.get("evidence_key", "")).strip() + if not key: + continue + for precheck in item.get("prechecks", []): + if not isinstance(precheck, dict): + continue + if precheck.get("required") is not True or precheck.get("status") == "pass": + continue + target = str(precheck.get("key") or precheck.get("label") or key) + rows.append( + _repair_row( + evidence_key=key, + repair_type="precheck", + target=target, + status="blocked", + blocking_reason=( + f"Required {precheck.get('kind', 'precheck')} precheck is " + f"{precheck.get('status', 'unknown')}." + ), + next_action=str(precheck.get("next_action") or "Complete the required preflight action."), + source="prechecks", + ) + ) + for source in item.get("source_checklist", []): + if not isinstance(source, dict): + continue + if source.get("status") == "pass": + continue + target = str(source.get("field") or source.get("label") or key) + rows.append( + _repair_row( + evidence_key=key, + repair_type="source-check", + target=target, + status="blocked", + blocking_reason=( + f"Current value {source.get('actual')!r} does not satisfy " + f"{source.get('expected')!r}." + ), + next_action=str(source.get("next_action") or "Collect the required source evidence."), + source="source_checklist", + ) + ) + return sort_repair_rows(rows) + + +def summarize_repair_checklist(rows: list[dict[str, Any]]) -> dict[str, Any]: + blocked_count = sum(1 for row in rows if row.get("status") != "ready") + phase_counts: dict[str, int] = {} + for row in rows: + phase = str(row.get("phase", "repair")) + phase_counts[phase] = phase_counts.get(phase, 0) + 1 + next_row = sort_repair_rows(rows)[0] if rows else {} + return { + "repair_checklist_count": len(rows), + "repair_blocked_count": blocked_count, + "repair_ready_count": len(rows) - blocked_count, + "repair_phase_counts": phase_counts, + "next_repair_action_id": next_row.get("action_id", ""), + "next_repair_phase": next_row.get("phase", ""), + "next_repair_owner": next_row.get("owner", ""), + "next_repair_command": next_row.get("verification_command", ""), + "repair_counts_as_completion": False, + } diff --git a/scripts/world_class_source_checks.py b/scripts/world_class_source_checks.py new file mode 100644 index 00000000..a05580b2 --- /dev/null +++ b/scripts/world_class_source_checks.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Shared source-evidence checks for world-class evidence workflows.""" + +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to keep source-evidence readiness checks consistent." + + +SOURCE_CHECK_SPECS = { + "provider-holdout": [ + ("Provider model run", "model_executed_count", ">0", "Run provider-backed output-exec with real credentials."), + ("Timing observed", "timing_observed_count", ">0", "Provider execution should record timing metadata."), + ("Token usage observed", "token_observed_count", ">0", "Provider execution should return non-estimated token usage."), + ], + "human-adjudication": [ + ("Review pairs exist", "pair_count", ">0", "Generate the blind A/B review pack."), + ("No pending decisions", "pending_count", "==0", "Record a reviewer choice and reason for every pair."), + ("Judgments complete", "judgment_count", "==pair_count", "Every pair needs one valid human judgment."), + ("No invalid decisions", "invalid_decision_count", "==0", "Fix malformed winner/confidence entries."), + ("Reviewer metadata", "reviewer_metadata_present", "true", "Record reviewer and reviewed_at before adjudication can count."), + ("Reason required", "reason_required", "true", "Keep reason mandatory for every imported or direct reviewer decision."), + ("Blind review attested", "blind_review_attested", "true", "Set reviewer_attestation only after choices are completed before opening the answer key."), + ("Raw content attested", "raw_content_excluded_attested", "true", "Attest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content."), + ("Raw content blocked", "raw_content_allowed", "false", "Adjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages."), + ("Human evidence ready", "ready_for_human_evidence", "true", "Complete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints."), + ], + "native-permission-enforcement": [ + ("Native enforcement", "native_enforcement_count", ">0", "Collect real target-client or external runtime guard proof."), + ("Probe failures", "failure_count", "==0", "Runtime permission probes must stay clean."), + ("Installer support", "installer_enforcement_ready", "true", "Installer enforcement is supporting evidence, not native proof."), + ], + "native-client-telemetry": [ + ("External events", "external_source_events", ">0", "Import at least one metadata-only event from a real client."), + ("Adoption sample", "adoption_sample_count", ">0", "Telemetry must include adoption outcome evidence."), + ("Raw content blocked", "raw_content_allowed", "false", "Telemetry must stay metadata-only."), + ], +} + + +def source_check_passed(actual: Any, expected: str, observed_state: dict[str, Any]) -> bool: + if expected == ">0": + return isinstance(actual, (int, float)) and actual > 0 + if expected == "==0": + return actual == 0 + if expected == "true": + return actual is True + if expected == "false": + return actual is False + if expected == "==pair_count": + return actual == observed_state.get("pair_count") and isinstance(actual, (int, float)) and actual > 0 + return False + + +def item_evidence_key(item: dict[str, Any]) -> str: + return str(item.get("evidence_key") or item.get("key") or "") + + +def item_observed_state(item: dict[str, Any]) -> dict[str, Any]: + observed = item.get("observed_state", {}) + return observed if isinstance(observed, dict) else {} + + +def item_source_accepted(item: dict[str, Any], observed_state: dict[str, Any]) -> bool: + return item.get("source_accepted") is True or observed_state.get("accepted") is True + + +def build_source_checklist(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for item in items: + key = item_evidence_key(item) + observed_state = item_observed_state(item) + for label, field, expected, next_action in SOURCE_CHECK_SPECS.get(key, []): + actual = observed_state.get(field) + passed = source_check_passed(actual, expected, observed_state) + rows.append( + { + "evidence_key": key, + "label": label, + "field": field, + "expected": expected, + "actual": actual, + "status": "pass" if passed else "blocked", + "source_accepted": item_source_accepted(item, observed_state), + "next_action": next_action, + } + ) + return rows + + +def summarize_source_checklist(rows: list[dict[str, Any]]) -> dict[str, int]: + pass_count = sum(1 for item in rows if item.get("status") == "pass") + blocked_count = sum(1 for item in rows if item.get("status") != "pass") + return { + "source_check_count": len(rows), + "source_pass_count": pass_count, + "source_blocked_count": blocked_count, + } diff --git a/scripts/world_class_submission_kit_rendering.py b/scripts/world_class_submission_kit_rendering.py new file mode 100644 index 00000000..ae0da581 --- /dev/null +++ b/scripts/world_class_submission_kit_rendering.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python3 +"""Render Markdown and HTML artifacts for world-class submission kits.""" + +from typing import Any + +from html_rendering import html_text + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Shared renderer for world-class submission kit Markdown and HTML artifacts." + + +def render_readme(report: dict[str, Any]) -> str: + commands = report["commands"] + lines = [ + "# World-Class Evidence Submission Kit", + "", + f"Generated at: `{report['generated_at']}`", + "", + "This kit contains editable drafts for human and external evidence packets. Drafts are not accepted evidence.", + "", + "## Workflow", + "", + "1. Run the real provider, human review, native permission, or native client telemetry work first.", + "2. Edit the matching JSON draft with only aggregate artifact references and provenance metadata.", + "3. Set `template_only` to `false` only after real evidence exists.", + "4. Set attestation booleans truthfully; do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content.", + "5. Validate the packet before asking the ledger reviewer to set `attestation.ledger_reviewer`, `attestation.ledger_reviewed_at`, and `attestation.ledger_reviewer_approved` truthfully.", + "6. Optional artifact prefill only inserts SHA-256 digests for current local aggregate artifacts; it does not mark a draft as real evidence.", + "", + "## Commands", + "", + f"- validate intake: `{commands['validate_intake']}`", + f"- review submission: `{commands['review_submission']}`", + f"- refresh ledger: `{commands['refresh_ledger']}`", + f"- guard public claims: `{commands['guard_claim']}`", + "", + "## Operator Handoff", + "", + "Follow these steps in order when handing the kit from operator to reviewer. Handoff rows are procedural and never count as completion evidence.", + "", + "| Step | Status | Command | Completion signal |", + "| --- | --- | --- | --- |", + ] + for item in report.get("operator_handoff", []): + command = item.get("command") or "manual" + lines.append( + f"| `{item['step_id']}` | `{item['status']}` | `{command}` | {item['completion_signal']} |" + ) + lines.extend( + [ + "", + "## Phase Queue", + "", + "This queue groups repair rows by execution phase so operators can clear access, artifact, and source blockers in order. Queue rows are procedural guidance only.", + "", + "| Phase | Status | Rows | Owners | Next action | Verify |", + "| --- | --- | ---: | --- | --- | --- |", + ] + ) + phase_queue = report.get("phase_queue", []) + if phase_queue: + for item in phase_queue: + owners = ", ".join(str(owner) for owner in item.get("owners", [])) or "n/a" + lines.append( + f"| `{item['phase']}` | `{item['status']}` | `{item['row_count']}` | {owners} | " + f"{item['next_action']} | `{item['verification_command']}` |" + ) + else: + lines.append("| `all` | `ready` | `0` | n/a | No phase blockers were generated. | `n/a` |") + lines.extend( + [ + "", + "## Drafts", + "", + "| Evidence | Draft | Status | Prefilled refs |", + "| --- | --- | --- | ---: |", + ] + ) + for item in report["files"]: + lines.append( + f"| `{item['evidence_key']}` | `{item['output_path']}` | `{item['status']}` | `{item.get('prefilled_artifact_ref_count', 0)}` |" + ) + lines.extend( + [ + "", + "## Evidence Matrix", + "", + "This matrix combines draft, artifact, and source-check readiness into one operator action list. Matrix rows are guidance only; they do not count as completion evidence.", + "", + "| Evidence | Stage | Draft | Submission refs | Supporting assets | Source checks | Next action |", + "| --- | --- | --- | ---: | ---: | ---: | --- |", + ] + ) + for item in report.get("evidence_matrix", []): + lines.append( + f"| `{item['evidence_key']}` | `{item['stage']}` | `{item['draft_status']}` | " + f"`{item.get('submission_ref_ready_count', 0)}/{item.get('submission_ref_total_count', 0)}` | " + f"`{item.get('supporting_artifact_ready_count', 0)}/{item.get('supporting_artifact_total_count', 0)}` | " + f"`{item['source_pass_count']}/{item['source_check_count']}` | {item['next_action']} |" + ) + lines.extend( + [ + "", + "## Repair Checklist", + "", + "This checklist turns every draft, artifact, and source blocker into a concrete repair row. Repair rows are procedural guidance and do not count as completion evidence.", + "", + "| Evidence | Type | Target | Status | Next action |", + "| --- | --- | --- | --- | --- |", + ] + ) + repair_rows = report.get("repair_checklist", []) + if repair_rows: + for item in repair_rows: + lines.append( + f"| `{item['evidence_key']}` | `{item['repair_type']}` | `{item['target']}` | " + f"`{item['status']}` | {item['next_action']} |" + ) + else: + lines.append("| `all` | `none` | `n/a` | `ready` | No repair rows were generated. |") + lines.extend(["", "## Execution Runbook", ""]) + for item in report.get("evidence_items", []): + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + runbook = must_collect.get("runbook", []) + lines.extend(["", f"### {item.get('label', item.get('evidence_key', 'Evidence'))}", ""]) + if runbook: + for step in runbook: + lines.append(f"- `{step}`" if str(step).startswith("python3 ") or "=" in str(step) else f"- {step}") + else: + lines.append("- No source runbook listed.") + lines.extend( + [ + "", + "## Artifact Checklist", + "", + "Use these paths and SHA-256 digests when filling `artifact_refs`. Glob patterns are expanded into concrete files; submissions must reference concrete paths, not globs.", + "", + "| Evidence | Role | Path | Status | SHA-256 |", + "| --- | --- | --- | --- | --- |", + ] + ) + for item in report.get("artifact_checklist", []): + digest = item.get("sha256") or "n/a" + lines.append( + f"| `{item['evidence_key']}` | `{item.get('artifact_role', 'supporting-evidence')}` | " + f"`{item['path']}` | `{item['status']}` | `{digest}` |" + ) + lines.extend( + [ + "", + "## Source Evidence Snapshot", + "", + "These checks explain why a draft is not ready for ledger acceptance yet. They mirror current aggregate reports and do not accept evidence by themselves.", + "", + "| Evidence | Check | Current | Expected | Status |", + "| --- | --- | --- | --- | --- |", + ] + ) + for item in report.get("source_checklist", []): + lines.append( + f"| `{item['evidence_key']}` | {item['label']} | `{item['actual']}` | `{item['expected']}` | `{item['status']}` |" + ) + lines.extend( + [ + "", + "## Anti-Overclaim", + "", + "- This kit never marks ledger evidence as accepted.", + "- Planned work, metadata fallback, pending review, and local command-runner output remain non-evidence.", + "- A valid intake packet means ready for ledger review, not world-class completion.", + ] + ) + return "\n".join(lines).rstrip() + "\n" + + +def render_html_list(values: list[Any], empty: str) -> str: + if not values: + return f"
  • {html_text(empty)}
    • " + return "".join(f"
  • {html_text(value)}
    • " for value in values) + + +def render_html_commands(commands: dict[str, str]) -> str: + return "".join( + f"
  • {html_text(label.replace('_', ' '))}{html_text(command)}
    • " + for label, command in commands.items() + ) + + +def render_html_files(files: list[dict[str, Any]]) -> str: + if not files: + return '

    No submission drafts were requested.

    ' + return "".join( + """ +
    +
    + {status} +

    {key}

    +
    +
    +
    Template
    {template}
    +
    Draft
    {output}
    +
    Prefill
    {prefill} artifact refs
    +
    + {errors} +
    + """.format( + status=html_text(item.get("status", "")), + key=html_text(item.get("evidence_key", "")), + template=html_text(item.get("template_path", "")), + output=html_text(item.get("output_path", "")), + prefill=html_text(item.get("prefilled_artifact_ref_count", 0)), + errors=( + '
      ' + + render_html_list(item.get("errors", []), "No errors.") + + "
    " + if item.get("errors") + else "" + ), + ) + for item in files + ) + + +def render_html_artifact_checklist(items: list[dict[str, Any]]) -> str: + if not items: + return '

    No required artifacts were listed for the requested evidence.

    ' + return "".join( + """ +
    +
    + {key} +

    {path}

    +
    +
    +
    Pattern
    {pattern}
    +
    Role
    {role}
    +
    Status
    {status}
    +
    SHA-256
    {sha}
    +
    +
    + """.format( + status=html_text(item.get("status", "")), + key=html_text(item.get("evidence_key", "")), + path=html_text(item.get("path", "")), + pattern=html_text(item.get("source_pattern", "")), + role=html_text(item.get("artifact_role", "supporting-evidence")), + sha=html_text(item.get("sha256") or "n/a"), + ) + for item in items + ) + + +def render_html_source_checklist(items: list[dict[str, Any]]) -> str: + if not items: + return '

    No source checks were listed for the requested evidence.

    ' + return "".join( + """ +
    +
    + {key} +

    {label}

    +
    +
    +
    Field
    {field}
    +
    Current
    {actual}
    +
    Expected
    {expected}
    +
    Action
    {action}
    +
    +
    + """.format( + status=html_text(item.get("status", "")), + key=html_text(item.get("evidence_key", "")), + label=html_text(item.get("label", "")), + field=html_text(item.get("field", "")), + actual=html_text(item.get("actual", "")), + expected=html_text(item.get("expected", "")), + action=html_text(item.get("next_action", "")), + ) + for item in items + ) + + +def render_html_matrix(items: list[dict[str, Any]]) -> str: + if not items: + return '

    No evidence matrix rows were generated.

    ' + return "".join( + """ +
    +
    + {stage} +

    {key}

    +
    +
    +
    Draft
    {draft}
    +
    Submission refs
    {submission_ref_ready}/{submission_ref_total} ready
    +
    Supporting assets
    {supporting_ready}/{supporting_total} ready
    +
    Source
    {source_pass}/{source_total} pass
    +
    Owner
    {owner}
    +
    +

    {action}

    +
    + """.format( + stage=html_text(item.get("stage", "")), + key=html_text(item.get("evidence_key", "")), + draft=html_text(item.get("draft_status", "")), + submission_ref_ready=html_text(item.get("submission_ref_ready_count", 0)), + submission_ref_total=html_text(item.get("submission_ref_total_count", 0)), + supporting_ready=html_text(item.get("supporting_artifact_ready_count", 0)), + supporting_total=html_text(item.get("supporting_artifact_total_count", 0)), + source_pass=html_text(item.get("source_pass_count", 0)), + source_total=html_text(item.get("source_check_count", 0)), + owner=html_text(item.get("owner", "")), + action=html_text(item.get("next_action", "")), + ) + for item in items + ) + + +def render_html_repair_checklist(items: list[dict[str, Any]]) -> str: + if not items: + return '

    No repair rows were generated.

    ' + return "".join( + """ +
    +
    + {key} · {repair_type} +

    {target}

    +
    +
    +
    Priority
    {priority}
    +
    Phase
    {phase}
    +
    Owner
    {owner}
    +
    Status
    {status}
    +
    Reason
    {reason}
    +
    Action
    {action}
    +
    Verify
    {verify}
    +
    Evidence
    does not count as completion
    +
    +
    + """.format( + status=html_text(item.get("status", "")), + key=html_text(item.get("evidence_key", "")), + repair_type=html_text(item.get("repair_type", "")), + target=html_text(item.get("target", "")), + priority=html_text(item.get("priority", "")), + phase=html_text(item.get("phase", "")), + owner=html_text(item.get("owner", "")), + reason=html_text(item.get("blocking_reason", "")), + action=html_text(item.get("next_action", "")), + verify=html_text(item.get("verification_command", "")), + ) + for item in items + ) + + +def render_html_phase_queue(items: list[dict[str, Any]]) -> str: + if not items: + return '

    No phase queue rows were generated.

    ' + return "".join( + """ +
    +
    + #{priority} · {status} +

    {label}

    +
    +
    +
    Phase
    {phase}
    +
    Rows
    {blocked}/{total} blocked
    +
    Owners
    {owners}
    +
    Evidence
    {evidence}
    +
    Next
    {action}
    +
    Verify
    {verify}
    +
    Counts
    does not count as completion
    +
    +
    + """.format( + status=html_text(item.get("status", "")), + priority=html_text(item.get("priority", "")), + label=html_text(item.get("label", "")), + phase=html_text(item.get("phase", "")), + blocked=html_text(item.get("blocked_count", 0)), + total=html_text(item.get("row_count", 0)), + owners=html_text(", ".join(str(owner) for owner in item.get("owners", [])) or "n/a"), + evidence=html_text(", ".join(str(key) for key in item.get("evidence_keys", [])) or "n/a"), + action=html_text(item.get("next_action", "")), + verify=html_text(item.get("verification_command", "")), + ) + for item in items + ) + + +def render_html_handoff(items: list[dict[str, Any]]) -> str: + if not items: + return '

    No operator handoff steps were generated.

    ' + return "".join( + """ +
    +
    + {status} +

    {label}

    +
    +
    +
    Step
    {step}
    +
    Command
    {command}
    +
    Signal
    {signal}
    +
    Evidence
    {counts}
    +
    +

    {blocking}

    +
    + """.format( + status=html_text(item.get("status", "")), + label=html_text(item.get("label", "")), + step=html_text(item.get("step_id", "")), + command=html_text(item.get("command") or "manual"), + signal=html_text(item.get("completion_signal", "")), + counts="does not count as completion" if item.get("counts_as_completion") is False else "review required", + blocking=html_text(item.get("blocking_condition", "")), + ) + for item in items + ) + + +def render_html_item(item: dict[str, Any]) -> str: + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + runbook = must_collect.get("runbook", []) + return f""" +
    +
    + {html_text(item.get('category', ''))} · {html_text(item.get('readiness', ''))} +

    {html_text(item.get('label', item.get('evidence_key', '')))}

    +
    +

    {html_text(item.get('blocking_reason', ''))}

    +
    +
    Owner
    {html_text(item.get('owner', ''))}
    +
    Evidence
    {html_text(item.get('evidence_key', ''))}
    +
    Submission
    {html_text(item.get('submission_path', ''))}
    +
    +
    +

    Execution Runbook

    +
      {render_html_list(runbook, 'No source runbook listed.')}
    +
    +
    +
    +

    Must Collect

    +
      {render_html_list(must_collect.get('provenance_requirements', []), 'No provenance requirements listed.')}
    +
    +
    +

    Pass Checks

    +
      {render_html_list(must_collect.get('success_checks', []), 'No success checks listed.')}
    +
    +
    +

    Privacy

    +
      {render_html_list(must_collect.get('privacy_contract', []), 'No privacy contract listed.')}
    +
    +
    +
    + """ + + +def render_html(report: dict[str, Any]) -> str: + summary = report["summary"] + artifact_ready = summary.get("artifact_ready_count", 0) + artifact_total = summary.get("artifact_checklist_count", 0) + submission_ref_ready = summary.get("submission_ref_ready_count", 0) + submission_ref_total = summary.get("submission_ref_count", 0) + stats = [ + ("Requested", summary["requested_count"]), + ("Written", summary["written_count"]), + ("Existing", summary["existing_count"]), + ("Skipped", summary["skipped_count"]), + ("Submit refs", f"{submission_ref_ready}/{submission_ref_total}"), + ("Support", f"{artifact_ready - submission_ref_ready}/{artifact_total - submission_ref_total}"), + ("Prefilled", summary.get("artifact_ref_prefill_count", 0)), + ] + stat_html = "".join( + f"
    {html_text(label)}{html_text(value)}
    " + for label, value in stats + ) + evidence_html = "".join(render_html_item(item) for item in report.get("evidence_items", [])) + matrix_html = render_html_matrix(report.get("evidence_matrix", [])) + repair_html = render_html_repair_checklist(report.get("repair_checklist", [])) + phase_queue_html = render_html_phase_queue(report.get("phase_queue", [])) + handoff_html = render_html_handoff(report.get("operator_handoff", [])) + artifact_html = render_html_artifact_checklist(report.get("artifact_checklist", [])) + source_html = render_html_source_checklist(report.get("source_checklist", [])) + return f""" + + + + + World-Class Evidence Submission Kit + + + + +
    +
    + Evidence Intake +

    World-Class Evidence Submission Kit

    +

    Use this cockpit to prepare human and external evidence packets. Drafts are not accepted evidence, artifact prefill only inserts local SHA-256 digests, and this page never changes the ledger result.

    +
    {stat_html}
    +
    +
    +

    Workflow

    1. Run the real provider, human review, native permission, or native client telemetry work first.
    2. Edit the matching JSON draft with aggregate artifact references and provenance metadata.
    3. Set template_only to false only after real evidence exists.
    4. Use prefilled SHA-256 values as convenience data, not evidence acceptance.
    5. Validate intake, ask the ledger reviewer to approve explicitly, refresh the ledger, then guard public claims.
    + +
    +

    Operator Handoff

    These ordered steps make the operator-to-reviewer handoff auditable. They are procedural guardrails and never count as world-class evidence.

    {handoff_html}
    +

    Phase Queue

    Repair rows are grouped into execution phases so operators can clear the first blocker before moving to later evidence work. Queue rows remain procedural and never count as accepted evidence.

    {phase_queue_html}
    +

    Evidence Matrix

    The matrix separates submission artifact_refs from supporting assets, then combines draft status, source checks, and the next operator action. It is guidance only and never counts as accepted evidence.

    {matrix_html}
    +

    Repair Checklist

    Each row names one concrete blocker and the next action required before ledger review. This checklist does not count as completion evidence.

    {repair_html}
    +

    Drafts

    {render_html_files(report['files'])}
    +

    Artifact Checklist

    Rows marked submission-ref are the paths expected in artifact_refs. Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into the submission. Glob patterns are expanded for operator convenience only.

    {artifact_html}
    +

    Source Evidence Snapshot

    This section shows current aggregate source checks. It explains remaining blockers without changing the ledger.

    {source_html}
    +

    Evidence Requirements

    {evidence_html}
    +

    Safety Boundary

    • Drafts never count as accepted ledger evidence.
    • Valid intake means ready for ledger review, not world-class completion.
    • Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content.
    +
    + + +""" diff --git a/scripts/world_class_submission_matrix.py b/scripts/world_class_submission_matrix.py new file mode 100644 index 00000000..943b5ef7 --- /dev/null +++ b/scripts/world_class_submission_matrix.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""Build operator-facing readiness matrices for world-class submission kits.""" + +from collections import defaultdict +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Shared by submission kit rendering to summarize draft, artifact, and source-check readiness." + + +def _by_key(rows: list[dict[str, Any]], key_name: str = "evidence_key") -> dict[str, list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + key = str(row.get(key_name, "")).strip() + if key: + grouped[key].append(row) + return dict(grouped) + + +def _draft_status(files_by_key: dict[str, list[dict[str, Any]]], key: str) -> tuple[str, str]: + rows = files_by_key.get(key, []) + if not rows: + return "missing", "" + row = rows[0] + return str(row.get("status", "missing")), str(row.get("output_path", "")) + + +def _first_blocked_action(source_rows: list[dict[str, Any]]) -> str: + for row in source_rows: + if row.get("status") != "pass": + action = str(row.get("next_action", "")).strip() + if action: + return action + return "" + + +def _stage( + *, + draft_status: str, + artifact_missing_count: int, + source_blocked_count: int, + invalid_draft: bool, +) -> str: + if draft_status == "skipped" or invalid_draft: + return "fix-draft" + if draft_status == "missing": + return "prepare-draft" + if source_blocked_count: + return "collect-source" + if artifact_missing_count: + return "fix-artifacts" + return "validate-packet" + + +def _next_action( + *, + stage: str, + draft_path: str, + first_blocked_action: str, + artifact_missing_count: int, +) -> str: + if stage == "fix-draft": + return "Fix skipped or invalid draft generation before evidence review." + if stage == "prepare-draft": + return "Generate the evidence draft, then keep template_only true until real evidence exists." + if stage == "collect-source": + return first_blocked_action or "Complete blocked source checks before ledger review." + if stage == "fix-artifacts": + return f"Resolve {artifact_missing_count} missing artifact reference(s) or replace globs with concrete paths." + if draft_path: + return f"Edit {draft_path}, set real provenance fields, validate intake, then refresh the ledger." + return "Validate intake, refresh the ledger, then run the public claim guard." + + +def build_evidence_matrix( + evidence_items: list[dict[str, Any]], + files: list[dict[str, Any]], + artifact_checklist: list[dict[str, Any]], + source_checklist: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Combine draft, artifact, and source-check readiness into one operator matrix.""" + files_by_key = _by_key(files) + artifacts_by_key = _by_key(artifact_checklist) + sources_by_key = _by_key(source_checklist) + matrix: list[dict[str, Any]] = [] + for item in evidence_items: + key = str(item.get("evidence_key", "")).strip() + if not key: + continue + artifact_rows = artifacts_by_key.get(key, []) + source_rows = sources_by_key.get(key, []) + draft_status, draft_path = _draft_status(files_by_key, key) + artifact_ready_count = sum(1 for row in artifact_rows if row.get("artifact_ref_ready")) + artifact_missing_count = len(artifact_rows) - artifact_ready_count + submission_ref_rows = [row for row in artifact_rows if row.get("submission_ref_required")] + supporting_rows = [row for row in artifact_rows if not row.get("submission_ref_required")] + submission_ref_ready_count = sum(1 for row in submission_ref_rows if row.get("artifact_ref_ready")) + supporting_artifact_ready_count = sum(1 for row in supporting_rows if row.get("artifact_ref_ready")) + source_pass_count = sum(1 for row in source_rows if row.get("status") == "pass") + source_blocked_count = len(source_rows) - source_pass_count + invalid_draft = draft_status not in {"written", "exists"} + stage = _stage( + draft_status=draft_status, + artifact_missing_count=artifact_missing_count, + source_blocked_count=source_blocked_count, + invalid_draft=invalid_draft, + ) + first_action = _first_blocked_action(source_rows) + matrix.append( + { + "evidence_key": key, + "label": item.get("label", key), + "category": item.get("category", ""), + "owner": item.get("owner", ""), + "stage": stage, + "draft_status": draft_status, + "draft_path": draft_path, + "artifact_ready_count": artifact_ready_count, + "artifact_total_count": len(artifact_rows), + "artifact_missing_count": artifact_missing_count, + "submission_ref_ready_count": submission_ref_ready_count, + "submission_ref_total_count": len(submission_ref_rows), + "submission_ref_missing_count": len(submission_ref_rows) - submission_ref_ready_count, + "supporting_artifact_ready_count": supporting_artifact_ready_count, + "supporting_artifact_total_count": len(supporting_rows), + "supporting_artifact_missing_count": len(supporting_rows) - supporting_artifact_ready_count, + "source_pass_count": source_pass_count, + "source_check_count": len(source_rows), + "source_blocked_count": source_blocked_count, + "next_action": _next_action( + stage=stage, + draft_path=draft_path, + first_blocked_action=first_action, + artifact_missing_count=artifact_missing_count, + ), + "counts_as_completion": False, + } + ) + return matrix + + +def summarize_evidence_matrix(matrix: list[dict[str, Any]]) -> dict[str, int]: + return { + "evidence_matrix_count": len(matrix), + "evidence_matrix_collect_source_count": sum(1 for item in matrix if item.get("stage") == "collect-source"), + "evidence_matrix_prepare_draft_count": sum(1 for item in matrix if item.get("stage") == "prepare-draft"), + "evidence_matrix_fix_artifacts_count": sum(1 for item in matrix if item.get("stage") == "fix-artifacts"), + "evidence_matrix_validate_packet_count": sum(1 for item in matrix if item.get("stage") == "validate-packet"), + "evidence_matrix_submission_ref_count": sum(int(item.get("submission_ref_total_count", 0)) for item in matrix), + "evidence_matrix_submission_ref_ready_count": sum( + int(item.get("submission_ref_ready_count", 0)) for item in matrix + ), + "evidence_matrix_supporting_artifact_count": sum( + int(item.get("supporting_artifact_total_count", 0)) for item in matrix + ), + "evidence_matrix_supporting_artifact_ready_count": sum( + int(item.get("supporting_artifact_ready_count", 0)) for item in matrix + ), + "evidence_matrix_counts_as_completion": 0, + } diff --git a/scripts/yao.py b/scripts/yao.py index 013ed5a9..09ffb3e0 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -2,370 +2,74 @@ import argparse import json import subprocess -import sys from pathlib import Path -from github_benchmark_scan import build_query -from render_intent_confidence import assess_intent_confidence from yao_cli_config import ( - ARCHETYPE_MODE, - archetype_guidance, baseline_compare_args, - diagnose_skill_candidates, - diagnosis_note, - discovery_summary, - infer_archetype, local_output_runner_command, - recommendation_from_synthesis, - reference_visibility, resolve_promotion_target, resolve_target, ) - - -ROOT = Path(__file__).resolve().parent.parent -SCRIPTS = ROOT / "scripts" - -def script_path(name: str) -> str: - return str(SCRIPTS / name) - - -def load_json_maybe(text: str) -> dict | None: - text = text.strip() - if not text: - return None - try: - return json.loads(text) - except json.JSONDecodeError: - return None - - -def run_script(name: str, args: list[str], cwd: Path | None = None) -> dict: - proc = subprocess.run( - [sys.executable, script_path(name), *args], - cwd=cwd or ROOT, - capture_output=True, - text=True, - ) - payload = load_json_maybe(proc.stdout) - return { - "command": f"{name} {' '.join(args)}".strip(), - "returncode": proc.returncode, - "ok": proc.returncode == 0, - "stdout": proc.stdout, - "stderr": proc.stderr, - "payload": payload, - } - - -def prompt_with_default(label: str, default: str) -> str: - sys.stderr.write(f"{label} [{default}]: ") - sys.stderr.flush() - value = sys.stdin.readline().strip() - return value or default - - -def prompt_optional(label: str, default: str = "skip") -> str: - sys.stderr.write(f"{label} [{default}]: ") - sys.stderr.flush() - value = sys.stdin.readline().strip() - return value or default - - -def prompt_optional_entries(label: str) -> list[str]: - sys.stderr.write(f"{label} [none]: ") - sys.stderr.flush() - value = sys.stdin.readline().strip() - if not value or value.lower() in {"none", "no", "n"}: - return [] - return [item.strip() for item in value.split(",") if item.strip()] - - -def update_context_slot(context: dict, slot: str, answer: str, list_mode: bool) -> None: - value = answer.strip() - if not value or value.lower() in {"skip", "none", "no", "n"}: - return - if list_mode: - context[slot] = [item.strip() for item in value.split(",") if item.strip()] - else: - context[slot] = value - - -def intent_confidence_note(summary: dict) -> str: - lines = [ - f"\nIntent confidence: {summary['score']}/100 ({summary['band']}).", - f"- Recommended action: {summary['recommended_action']}", - ] - if summary.get("gaps"): - top_gap = summary["gaps"][0] - lines.append(f"- Biggest gap: {top_gap['label']} — {top_gap['reason']}") - return "\n".join(lines) + "\n" - - -def maybe_emit_update_notice(args: argparse.Namespace) -> None: - if getattr(args, "no_update_check", False): - return - result = run_script("check_update.py", []) - payload = result["payload"] if result["payload"] is not None else {} - if not result["ok"] and not payload: - return - if payload.get("update_available"): - sys.stderr.write( - "\nUpdate available for yao-meta-skill: " - f"{payload.get('local_version')} -> {payload.get('remote_version')}.\n" - f"Run: {payload.get('install_hint')}\n" - ) - - -def command_init(args: argparse.Namespace) -> int: - cmd = [ - args.name, - "--description", - args.description, - "--output-dir", - args.output_dir, - "--mode", - args.mode, - "--archetype", - args.archetype, - *(["--title", args.title] if args.title else []), - ] - for reference in args.external_reference: - cmd.extend(["--external-reference", reference]) - for reference in args.user_reference: - cmd.extend(["--user-reference", reference]) - for constraint in args.local_constraint: - cmd.extend(["--local-constraint", constraint]) - if args.github_query: - cmd.extend(["--github-query", args.github_query]) - cmd.extend(["--github-top-n", str(args.github_top_n)]) - if args.github_fixture_dir: - cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) - if args.intent_job: - cmd.extend(["--intent-job", args.intent_job]) - for item in args.intent_real_input: - cmd.extend(["--intent-real-input", item]) - if args.intent_primary_output: - cmd.extend(["--intent-primary-output", args.intent_primary_output]) - for item in args.intent_exclusion: - cmd.extend(["--intent-exclusion", item]) - for item in args.intent_constraint: - cmd.extend(["--intent-constraint", item]) - for item in args.intent_standard: - cmd.extend(["--intent-standard", item]) - if args.intent_correction: - cmd.extend(["--intent-correction", args.intent_correction]) - result = run_script("init_skill.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_quickstart(args: argparse.Namespace) -> int: - maybe_emit_update_notice(args) - sys.stderr.write("Let's start gently. You do not need a polished brief here.\n") - sys.stderr.write("Give me the real work in your own words, and I will help turn it into a clean first-pass skill.\n") - sys.stderr.write("While we shape the first pass, I will quietly check a few strong public patterns in the background and only surface them if there is real uncertainty or a design conflict.\n") - name = args.name or prompt_with_default("Skill name", "my-skill") - job = args.job or prompt_with_default( - "In your own words, what repeated work do you most want this skill to reliably handle", - "Turn a repeated workflow into a reusable skill.", - ) - real_inputs = args.real_input or prompt_optional_entries( - "What material will people actually hand to this skill in practice (comma-separated)" - ) - primary_output = args.primary_output or prompt_with_default( - "If it works beautifully, what should it hand back so you or the next person can keep moving", - "A reusable skill package.", - ) - description = args.description or f"{job.rstrip('.')} Primary output: {primary_output.rstrip('.')}." - intent_context = { - "job": job, - "real_inputs": real_inputs, - "primary_output": primary_output, - "description": description, - "exclusions": [], - "constraints": [], - "standards": [], - "correction": "", - "user_references": [], - } - inferred_archetype, archetype_reason = infer_archetype(job, description) - guidance = archetype_guidance(inferred_archetype) - sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) - correction = prompt_optional( - "If I am off, what is the first thing I should correct before I package this idea", - "looks right", - ) - if correction.lower() not in {"looks right", "skip", "none", "no"}: - description = f"{description.rstrip('.')} Keep this correction in scope: {correction.rstrip('.')}." - intent_context["description"] = description - intent_context["correction"] = correction - inferred_archetype, archetype_reason = infer_archetype(job, description) - guidance = archetype_guidance(inferred_archetype) - sys.stderr.write("\nThanks. I tightened the frame before moving on.\n") - sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) - confidence = assess_intent_confidence(intent_context) - sys.stderr.write(intent_confidence_note(confidence)) - diagnosis = diagnose_skill_candidates(job, primary_output, inferred_archetype, confidence) - if diagnosis["fuzzy"]: - sys.stderr.write(diagnosis_note(diagnosis)) - if not confidence["gate_passed"]: - sys.stderr.write("Before I package this idea, I want to close the highest-leverage gaps instead of guessing.\n") - for follow_up in confidence.get("follow_up_questions", [])[:2]: - answer = prompt_optional(follow_up["question"], "skip") - update_context_slot(intent_context, follow_up["slot"], answer, follow_up["list"]) - confidence = assess_intent_confidence(intent_context) - sys.stderr.write("\nI tightened the intent frame once more before moving on.\n") - sys.stderr.write(intent_confidence_note(confidence)) - diagnosis = diagnose_skill_candidates(job, primary_output, inferred_archetype, confidence) - if diagnosis["fuzzy"]: - sys.stderr.write(diagnosis_note(diagnosis)) - archetype = args.archetype or prompt_with_default("I would start with this archetype (scaffold/production/library/governed)", inferred_archetype) - archetype = archetype if archetype in ARCHETYPE_MODE else inferred_archetype - default_mode = ARCHETYPE_MODE[archetype] - mode = args.mode or prompt_with_default("For the first pass, I would keep the mode here (scaffold/production/library/governed)", default_mode) - mode = mode if mode in ARCHETYPE_MODE.values() else default_mode - diagnosis = diagnose_skill_candidates(job, primary_output, archetype, confidence) - guidance = archetype_guidance(archetype) - sys.stderr.write( - f"\nGood. I will treat this as `{archetype}` in `{mode}` mode, so the first pass stays focused on {guidance['focus']}.\n" - ) - user_references = args.user_reference or prompt_optional_entries( - "If there is anything you admire and want me to learn from as pattern hints, send it here (repo, product, page, workflow; comma-separated)" - ) - external_references = args.external_reference or [] - prompted_constraints = args.constraint if getattr(args, "constraint", None) else ([] if args.local_constraint else prompt_optional_entries( - "Tell me any local constraints I must keep in view (privacy, naming, compatibility; comma-separated)" - )) - local_constraints = args.local_constraint or prompted_constraints or intent_context.get("constraints", []) - intent_context["user_references"] = user_references - intent_context["constraints"] = local_constraints - confidence = assess_intent_confidence(intent_context) - github_query = args.github_query or build_query(" ".join(filter(None, [job, primary_output, description]))) - title = args.title or name.replace("-", " ").title() - guidance = archetype_guidance(archetype) - cmd = [ - name, - "--description", - description, - "--title", - title, - "--output-dir", - args.output_dir, - "--mode", - mode, - "--archetype", - archetype, - "--github-query", - github_query, - "--github-top-n", - str(args.github_top_n), - "--intent-job", - job, - "--intent-primary-output", - primary_output, - ] - for item in real_inputs: - cmd.extend(["--intent-real-input", item]) - for item in intent_context.get("exclusions", []): - cmd.extend(["--intent-exclusion", item]) - for item in intent_context.get("constraints", []): - cmd.extend(["--intent-constraint", item]) - for item in intent_context.get("standards", []): - cmd.extend(["--intent-standard", item]) - if intent_context.get("correction"): - cmd.extend(["--intent-correction", intent_context["correction"]]) - if args.github_fixture_dir: - cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) - for reference in external_references: - cmd.extend(["--external-reference", reference]) - for reference in user_references: - cmd.extend(["--user-reference", reference]) - for constraint in local_constraints: - cmd.extend(["--local-constraint", constraint]) - result = run_script("init_skill.py", cmd) - payload = result["payload"] if result["payload"] is not None else result - reference_synthesis = payload.get("reference_synthesis") or {} - visibility = reference_visibility(reference_synthesis) - recommendation = recommendation_from_synthesis(reference_synthesis, visibility) - sys.stderr.write(f"\nRecommendation: {recommendation['summary']}\n") - if visibility["user_decision_required"]: - if visibility["conflicts"]: - sys.stderr.write(f"I am surfacing this because there is a real design conflict: {visibility['conflicts'][0]['summary']}\n") - else: - sys.stderr.write("I am surfacing this because intent is still settling and the package should not deepen on guesswork.\n") - else: - sys.stderr.write("I will keep the underlying benchmark evidence in the reviewer reports and move ahead with this recommendation.\n") - if payload.get("report_view", {}).get("html_report"): - sys.stderr.write(f"Skill report: {payload['report_view']['html_report']}\n") - - next_steps = [ - "Open reports/skill-overview.html to review the generated Skill audit report.", - "Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.", - "Open reports/review-studio.html to inspect the Review Studio 2.0 gate view before release.", - "Open reports/review-viewer.html to explain the package to a first-time reviewer.", - "Use reports/iteration-directions.md to choose only one high-value next move before adding more files.", - ] - if visibility["user_decision_required"]: - next_steps.insert( - 1, - "Open reports/reference-synthesis.md if you want to inspect why the recommendation was surfaced and which tradeoff needs a call.", - ) - report = { - "ok": result["ok"], - "root": payload.get("root"), - "mode": mode, - "archetype": archetype, - "artifacts": payload.get("artifacts", {}), - "report_view": payload.get("report_view", {}), - "intent_confidence": { - "score": confidence["score"], - "band": confidence["band"], - "gate_passed": confidence["gate_passed"], - "recommended_action": confidence["recommended_action"], - }, - "recommendation": recommendation, - "reference_mode": { - "mode": visibility["mode"], - "user_decision_required": visibility["user_decision_required"], - }, - "reviewer_evidence": { - "visibility": "full evidence in reports and review-viewer", - "artifacts": { - "benchmark_scan": payload.get("artifacts", {}).get("github_benchmark_scan_md"), - "reference_synthesis": payload.get("artifacts", {}).get("reference_synthesis_md"), - "artifact_design_profile": payload.get("artifacts", {}).get("artifact_design_profile_md"), - "prompt_quality_profile": payload.get("artifacts", {}).get("prompt_quality_profile_md"), - "system_model": payload.get("artifacts", {}).get("system_model_md"), - "review_studio": payload.get("artifacts", {}).get("review_studio_html"), - "review_viewer": payload.get("artifacts", {}).get("review_viewer_html"), - }, - }, - "guidance": { - "archetype_reason": archetype_reason, - "problem_diagnosis": diagnosis, - "why_this_mode": ( - "Scaffold mode keeps the first package light and lets you postpone governance-heavy work until reuse becomes real." - if mode == "scaffold" - else "This mode expects stronger lifecycle metadata, validation, and review discipline." - ), - "first_gate": guidance["first_gate"], - "focus": guidance["focus"], - "next_steps": next_steps, - "experience_note": ( - "The first pass should feel more like guided co-creation than a worksheet. " - "The system should make benchmark and pattern calls quietly unless there is a real reason to ask you to choose." - ), - }, - } - if visibility["user_decision_required"]: - report["uncertainty_or_conflict"] = { - "reasons": visibility["reasons"], - "conflicts": visibility["conflicts"], - "note": "A design decision still needs your input before the package should be deepened.", - } - print(json.dumps(report, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 +from yao_cli_adaptation_commands import ( + command_adapt_apply, + command_adapt_propose, + command_adapt_scan, + command_daily_skillops, + command_weekly_curator, +) +from yao_cli_create_commands import command_init, command_quickstart +from yao_cli_distribution_commands import ( + command_compile_skill, + command_conformance, + command_install_simulate, + command_package, + command_package_verify, + command_registry_audit, + command_runtime_permissions, + command_skill_atlas, + command_skill_ir, + command_trust, + command_upgrade_check, +) +from yao_cli_output_commands import ( + command_output_eval, + command_output_execution, + command_output_review, + command_output_review_import, + command_output_review_kit, +) +from yao_cli_parser import build_parser as build_cli_parser +from yao_cli_report_commands import ( + command_artifact_design_profile, + command_benchmark_reproducibility, + command_evidence_consistency, + command_github_benchmark_scan, + command_intent_confidence, + command_intent_dialogue, + command_iteration_directions, + command_output_risk_profile, + command_prompt_quality_profile, + command_reference_scan, + command_reference_synthesis, + command_report, + command_review_studio, + command_review_viewer, + command_skill_os2_audit, + command_skill_os2_coverage, + command_skill_interpretation, + command_skill_report, + command_system_model, + command_world_class_claim_guard, + command_world_class_evidence, + command_world_class_intake, + command_world_class_ledger, + command_world_class_preflight, + command_world_class_runbook, + command_world_class_submission_kit, + command_world_class_submission_review, +) +from yao_cli_runtime import ROOT, allow_report_status, run_adoption_drift_if_source_exists, run_script +from yao_cli_telemetry import add_telemetry_args, maybe_record_cli_event def command_validate(args: argparse.Namespace) -> int: @@ -433,224 +137,38 @@ def command_promote_check(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 -def command_report(args: argparse.Namespace) -> int: - steps = [] - if args.refresh_optimization: - steps.append(run_script("run_description_optimization_suite.py", [])) - steps.extend( - [ - run_script("build_confusion_matrix.py", []), - run_script("promotion_checker.py", []), - run_script("render_eval_dashboard.py", []), - run_script("render_intent_confidence.py", [str(ROOT)]), - run_script("render_description_drift_history.py", []), - run_script("render_iteration_ledger.py", []), - run_script("render_baseline_compare.py", baseline_compare_args()), - run_script("render_regression_history.py", []), - run_script("render_context_reports.py", []), - run_script("render_portability_report.py", []), - run_script("render_reference_synthesis.py", [str(ROOT)]), - run_script("render_artifact_design_profile.py", [str(ROOT)]), - run_script("render_prompt_quality_profile.py", [str(ROOT)]), - run_script("render_system_model.py", [str(ROOT)]), - run_script("compile_skill.py", [str(ROOT)]), - run_script("run_output_eval.py", []), - run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]), - run_script("adjudicate_output_review.py", []), - run_script("render_adoption_drift_report.py", [str(ROOT)]), - run_script("render_review_waivers.py", [str(ROOT)]), - run_script("render_review_annotations.py", [str(ROOT)]), - ] - ) - report = { - "ok": all(step["ok"] for step in steps), - "steps": [{"command": step["command"], "ok": step["ok"], "returncode": step["returncode"]} for step in steps], - "artifacts": { - "eval_results": "reports/eval_suite.json", - "route_scorecard": "reports/route_scorecard.json", - "promotion_decisions": "reports/promotion_decisions.json", - "intent_confidence": "reports/intent-confidence.json", - "iteration_ledger": "reports/iteration_ledger.md", - "baseline_compare": "reports/baseline-compare.json", - "regression_history": "reports/regression_history.md", - "context_budget": "reports/context_budget.json", - "portability_score": "reports/portability_score.json", - "reference_synthesis": "reports/reference-synthesis.json", - "artifact_design_profile": "reports/artifact-design-profile.json", - "prompt_quality_profile": "reports/prompt-quality-profile.json", - "system_model": "reports/system-model.json", - "compiled_targets": "reports/compiled_targets.json", - "output_execution": "reports/output_execution_runs.json", - "output_review_adjudication": "reports/output_review_adjudication.json", - "adoption_drift": "reports/adoption_drift_report.json", - "review_waivers": "reports/review_waivers.json", - "review_annotations": "reports/review_annotations.json", - }, - } - print(json.dumps(report, ensure_ascii=False, indent=2)) - return 0 if report["ok"] else 2 - - -def command_skill_report(args: argparse.Namespace) -> int: +def command_python_compat(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] - if args.output_html: - cmd.extend(["--output-html", args.output_html]) + for path in args.path: + cmd.extend(["--path", path]) + if args.target_python: + cmd.extend(["--target-python", args.target_python]) if args.output_json: cmd.extend(["--output-json", args.output_json]) - result = run_script("render_skill_overview.py", cmd) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("python_compat_check.py", cmd) print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) return 0 if result["ok"] else 2 -def command_review_viewer(args: argparse.Namespace) -> int: +def command_architecture_audit(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] - if args.output_html: - cmd.extend(["--output-html", args.output_html]) if args.output_json: cmd.extend(["--output-json", args.output_json]) - result = run_script("render_review_viewer.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_review_studio(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir] - if args.output_html: - cmd.extend(["--output-html", args.output_html]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_review_studio.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_reference_scan(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir] - for reference in args.external_reference: - cmd.extend(["--external-reference", reference]) - for reference in args.user_reference: - cmd.extend(["--user-reference", reference]) - for constraint in args.local_constraint: - cmd.extend(["--local-constraint", constraint]) - for reference in args.reference: - cmd.extend(["--reference", reference]) if args.output_md: cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_reference_scan.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_github_benchmark_scan(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir, "--query", args.query, "--top-n", str(args.top_n)] - if args.fixture_dir: - cmd.extend(["--fixture-dir", args.fixture_dir]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("github_benchmark_scan.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_intent_confidence(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir] - if args.context_json: - cmd.extend(["--context-json", args.context_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_intent_confidence.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_intent_dialogue(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_intent_dialogue.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_reference_synthesis(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_reference_synthesis.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_risk_profile(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_output_risk_profile.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_artifact_design_profile(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_artifact_design_profile.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_prompt_quality_profile(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_prompt_quality_profile.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_system_model(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_system_model.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_iteration_directions(args: argparse.Namespace) -> int: - skill_dir = str(Path(args.skill_dir).resolve()) - cmd = [skill_dir] - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - result = run_script("render_iteration_directions.py", cmd) + if args.warn_lines is not None: + cmd.extend(["--warn-lines", str(args.warn_lines)]) + if args.block_lines is not None: + cmd.extend(["--block-lines", str(args.block_lines)]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("render_architecture_maintainability.py", cmd) print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) return 0 if result["ok"] else 2 @@ -688,6 +206,8 @@ def command_adoption_drift(args: argparse.Namespace) -> int: cmd.extend(["--activation-type", args.activation_type]) cmd.extend(["--outcome", args.outcome]) cmd.extend(["--failure-type", args.failure_type]) + cmd.extend(["--source", args.source]) + cmd.extend(["--command", args.telemetry_command]) if args.timestamp: cmd.extend(["--timestamp", args.timestamp]) if args.skill_name: @@ -699,6 +219,74 @@ def command_adoption_drift(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_telemetry_import(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir, "--input-jsonl", args.input_jsonl] + if args.events_jsonl: + cmd.extend(["--events-jsonl", args.events_jsonl]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.source: + cmd.extend(["--source", args.source]) + if args.telemetry_command: + cmd.extend(["--command", args.telemetry_command]) + if args.dry_run: + cmd.append("--dry-run") + result = run_script("import_telemetry_events.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_telemetry_emit(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [ + skill_dir, + "--event", + args.event, + "--activation-type", + args.activation_type, + "--outcome", + args.outcome, + "--failure-type", + args.failure_type, + "--source", + args.source, + "--command", + args.telemetry_command, + ] + if args.output_jsonl: + cmd.extend(["--output-jsonl", args.output_jsonl]) + if args.timestamp: + cmd.extend(["--timestamp", args.timestamp]) + if args.skill_name: + cmd.extend(["--skill-name", args.skill_name]) + if args.version: + cmd.extend(["--version", args.version]) + if args.dry_run: + cmd.append("--dry-run") + result = run_script("emit_telemetry_event.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_telemetry_hooks(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_jsonl: + cmd.extend(["--output-jsonl", args.output_jsonl]) + result = run_script("render_telemetry_hook_recipes.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_review_waivers(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -784,212 +372,6 @@ def command_baseline_compare(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 -def command_skill_ir(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.validate_only: - cmd.append("--validate-only") - result = run_script("export_skill_ir.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_compile_skill(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - for target in args.target or []: - cmd.extend(["--target", target]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("compile_skill.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_eval(args: argparse.Namespace) -> int: - cmd = [] - if args.cases: - cmd.extend(["--cases", args.cases]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.blind_pack_json: - cmd.extend(["--blind-pack-json", args.blind_pack_json]) - if args.blind_pack_md: - cmd.extend(["--blind-pack-md", args.blind_pack_md]) - if args.blind_answer_key_json: - cmd.extend(["--blind-answer-key-json", args.blind_answer_key_json]) - result = run_script("run_output_eval.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_execution(args: argparse.Namespace) -> int: - cmd = [] - if args.cases: - cmd.extend(["--cases", args.cases]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.runner_command: - cmd.extend(["--runner-command", args.runner_command]) - if args.timeout_seconds is not None: - cmd.extend(["--timeout-seconds", str(args.timeout_seconds)]) - result = run_script("run_output_execution.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_review(args: argparse.Namespace) -> int: - cmd = [] - if args.blind_pack: - cmd.extend(["--blind-pack", args.blind_pack]) - if args.answer_key: - cmd.extend(["--answer-key", args.answer_key]) - if args.decisions: - cmd.extend(["--decisions", args.decisions]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.write_template: - cmd.append("--write-template") - result = run_script("adjudicate_output_review.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_conformance(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - for target in args.target or []: - cmd.extend(["--target", target]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - result = run_script("run_conformance_suite.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_runtime_permissions(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.package_dir: - cmd.extend(["--package-dir", args.package_dir]) - for target in args.target or []: - cmd.extend(["--target", target]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - result = run_script("probe_runtime_permissions.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_trust(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - result = run_script("trust_check.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_skill_atlas(args: argparse.Namespace) -> int: - cmd = ["--workspace-root", str(Path(args.workspace_root).resolve())] - if args.output_dir: - cmd.extend(["--output-dir", args.output_dir]) - if args.report_html: - cmd.extend(["--report-html", args.report_html]) - if args.report_json: - cmd.extend(["--report-json", args.report_json]) - if args.overlap_threshold is not None: - cmd.extend(["--overlap-threshold", str(args.overlap_threshold)]) - if args.today: - cmd.extend(["--today", args.today]) - result = run_script("build_skill_atlas.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_registry_audit(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.registry_dir: - cmd.extend(["--registry-dir", args.registry_dir]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("registry_audit.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_package_verify(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.package_dir: - cmd.extend(["--package-dir", args.package_dir]) - if args.expectations: - cmd.extend(["--expectations", args.expectations]) - if args.registry_json: - cmd.extend(["--registry-json", args.registry_json]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.require_zip: - cmd.append("--require-zip") - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("verify_package.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_install_simulate(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.package_dir: - cmd.extend(["--package-dir", args.package_dir]) - if args.install_root: - cmd.extend(["--install-root", args.install_root]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("simulate_install.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_upgrade_check(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - cmd.extend(["--previous-package-json", args.previous_package_json]) - if args.current_package_json: - cmd.extend(["--current-package-json", args.current_package_json]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("upgrade_check.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - def command_review(args: argparse.Namespace) -> int: target_name = resolve_promotion_target(args.target) bundle_dir = ROOT / "reports" / "iteration_bundles" / target_name @@ -1050,10 +432,26 @@ def command_workspace_flow(args: argparse.Namespace) -> int: {"phase": "report-refresh", "result": run_script("render_regression_history.py", [])}, {"phase": "report-refresh", "result": run_script("render_context_reports.py", [])}, {"phase": "report-refresh", "result": run_script("render_portability_report.py", [])}, + {"phase": "report-refresh", "result": run_script("python_compat_check.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_architecture_maintainability.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("compile_skill.py", [str(ROOT)])}, - {"phase": "report-refresh", "result": run_script("render_adoption_drift_report.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_adoption_drift_if_source_exists()}, + {"phase": "report-refresh", "result": run_script("render_telemetry_hook_recipes.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_review_waivers.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_review_annotations.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_evidence_intake.py", [str(ROOT)])}, + { + "phase": "report-refresh", + "result": allow_report_status(run_script("render_world_class_submission_review.py", [str(ROOT)])), + }, + {"phase": "report-refresh", "result": run_script("render_world_class_operator_runbook.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_claim_guard.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_skill_os2_coverage.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_skill_overview.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_review_viewer.py", [str(ROOT)])}, ] ) @@ -1102,23 +500,6 @@ def command_workspace_flow(args: argparse.Namespace) -> int: return 0 if report["ok"] else 2 -def command_package(args: argparse.Namespace) -> int: - cmd = [ - str(Path(args.skill_dir).resolve()), - "--output-dir", - args.output_dir, - ] - for platform in args.platform or ["generic"]: - cmd.extend(["--platform", platform]) - if args.expectations: - cmd.extend(["--expectations", args.expectations]) - if args.zip: - cmd.append("--zip") - result = run_script("cross_packager.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - def command_test(args: argparse.Namespace) -> int: proc = subprocess.run( ["make", args.target], @@ -1157,505 +538,20 @@ def command_check_update(args: argparse.Namespace) -> int: def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="Unified authoring CLI for yao-meta-skill.") - subparsers = parser.add_subparsers(dest="command", required=True) - - init_cmd = subparsers.add_parser("init", help="Initialize a minimal skill package.") - init_cmd.add_argument("name") - init_cmd.add_argument("--description", default="Describe what the skill does and when to use it.") - init_cmd.add_argument("--title") - init_cmd.add_argument("--output-dir", default=".") - init_cmd.add_argument("--mode", choices=["scaffold", "production", "library", "governed"], default="scaffold") - init_cmd.add_argument("--archetype", choices=["scaffold", "production", "library", "governed"], default="scaffold") - init_cmd.add_argument("--external-reference", action="append", default=[]) - init_cmd.add_argument("--user-reference", action="append", default=[]) - init_cmd.add_argument("--local-constraint", action="append", default=[]) - init_cmd.add_argument("--github-query") - init_cmd.add_argument("--github-top-n", type=int, default=3) - init_cmd.add_argument("--github-fixture-dir") - init_cmd.add_argument("--intent-job") - init_cmd.add_argument("--intent-real-input", action="append", default=[]) - init_cmd.add_argument("--intent-primary-output") - init_cmd.add_argument("--intent-exclusion", action="append", default=[]) - init_cmd.add_argument("--intent-constraint", action="append", default=[]) - init_cmd.add_argument("--intent-standard", action="append", default=[]) - init_cmd.add_argument("--intent-correction") - init_cmd.set_defaults(func=command_init) - - quickstart_cmd = subparsers.add_parser( - "quickstart", - help="Interactive fast path for creating a scaffold-first skill package.", - ) - quickstart_cmd.add_argument("--name") - quickstart_cmd.add_argument("--job") - quickstart_cmd.add_argument("--real-input", action="append", default=[]) - quickstart_cmd.add_argument("--primary-output") - quickstart_cmd.add_argument("--description") - quickstart_cmd.add_argument("--title") - quickstart_cmd.add_argument("--output-dir", default=".") - quickstart_cmd.add_argument("--mode", choices=["scaffold", "production", "library", "governed"]) - quickstart_cmd.add_argument("--archetype", choices=["scaffold", "production", "library", "governed"]) - quickstart_cmd.add_argument("--external-reference", action="append", default=[]) - quickstart_cmd.add_argument("--user-reference", action="append", default=[]) - quickstart_cmd.add_argument("--local-constraint", action="append", default=[]) - quickstart_cmd.add_argument("--constraint", action="append", default=[]) - quickstart_cmd.add_argument("--github-query") - quickstart_cmd.add_argument("--github-top-n", type=int, default=3) - quickstart_cmd.add_argument("--github-fixture-dir") - quickstart_cmd.add_argument("--no-update-check", action="store_true") - quickstart_cmd.set_defaults(func=command_quickstart) - - validate_cmd = subparsers.add_parser("validate", help="Run validate, lint, governance, and resource checks.") - validate_cmd.add_argument("skill_dir", nargs="?", default=".") - validate_cmd.add_argument("--require-manifest", action="store_true") - validate_cmd.set_defaults(func=command_validate) - - optimize_cmd = subparsers.add_parser("optimize-description", help="Optimize description candidates for a target.") - optimize_cmd.add_argument( - "--target", - choices=["root", "team-frontend-review", "governed-incident-command", "all"], - default="root", - ) - optimize_cmd.add_argument("--write", action="store_true", help="Write default report artifacts for the target.") - optimize_cmd.set_defaults(func=command_optimize_description) - - promote_cmd = subparsers.add_parser("promote-check", help="Apply promotion policy and build iteration bundles.") - promote_cmd.set_defaults(func=command_promote_check) - - review_cmd = subparsers.add_parser("review", help="Locate the current bundle and human review stub for a target.") - review_cmd.add_argument( - "--target", - choices=["root", "team-frontend-review", "governed-incident-command"], - default="root", - ) - review_cmd.set_defaults(func=command_review) - - snapshot_cmd = subparsers.add_parser("release-snapshot", help="Create a versioned snapshot from current promotion outputs.") - snapshot_cmd.add_argument( - "--target", - choices=["root", "team-frontend-review", "governed-incident-command"], - default="root", - ) - snapshot_cmd.add_argument("--label", default="manual") - snapshot_cmd.set_defaults(func=command_release_snapshot) - - flow_cmd = subparsers.add_parser( - "workspace-flow", - help="Run optimize, promotion, review refresh, and release snapshots as one authoring flow.", - ) - flow_cmd.add_argument( - "--target", - choices=["root", "team-frontend-review", "governed-incident-command", "all"], - default="root", - ) - flow_cmd.add_argument("--label", default="manual") - flow_cmd.set_defaults(func=command_workspace_flow) - - report_cmd = subparsers.add_parser("report", help="Render route, iteration, regression, and context reports.") - report_cmd.add_argument("--refresh-optimization", action="store_true") - report_cmd.set_defaults(func=command_report) - - skill_report_cmd = subparsers.add_parser("skill-report", help="Render the HTML skill report for a skill package.") - skill_report_cmd.add_argument("skill_dir", nargs="?", default=".") - skill_report_cmd.add_argument("--output-html") - skill_report_cmd.add_argument("--output-json") - skill_report_cmd.set_defaults(func=command_skill_report) - - review_viewer_cmd = subparsers.add_parser("review-viewer", help="Render a compact HTML review page for a skill package.") - review_viewer_cmd.add_argument("skill_dir", nargs="?", default=".") - review_viewer_cmd.add_argument("--output-html") - review_viewer_cmd.add_argument("--output-json") - review_viewer_cmd.set_defaults(func=command_review_viewer) - - review_studio_cmd = subparsers.add_parser("review-studio", help="Render Review Studio 2.0 for a skill package.") - review_studio_cmd.add_argument("skill_dir", nargs="?", default=".") - review_studio_cmd.add_argument("--output-html") - review_studio_cmd.add_argument("--output-json") - review_studio_cmd.set_defaults(func=command_review_studio) - - reference_scan_cmd = subparsers.add_parser( - "reference-scan", - help="Render a controlled benchmark scan report for a skill package.", - ) - reference_scan_cmd.add_argument("skill_dir", nargs="?", default=".") - reference_scan_cmd.add_argument("--reference", action="append", default=[]) - reference_scan_cmd.add_argument("--external-reference", action="append", default=[]) - reference_scan_cmd.add_argument("--user-reference", action="append", default=[]) - reference_scan_cmd.add_argument("--local-constraint", action="append", default=[]) - reference_scan_cmd.add_argument("--output-md") - reference_scan_cmd.add_argument("--output-json") - reference_scan_cmd.set_defaults(func=command_reference_scan) - - github_scan_cmd = subparsers.add_parser( - "github-benchmark-scan", - help="Search top public GitHub repositories and extract borrow or avoid patterns for a skill.", - ) - github_scan_cmd.add_argument("skill_dir", nargs="?", default=".") - github_scan_cmd.add_argument("--query", required=True) - github_scan_cmd.add_argument("--top-n", type=int, default=3) - github_scan_cmd.add_argument("--fixture-dir") - github_scan_cmd.add_argument("--output-md") - github_scan_cmd.add_argument("--output-json") - github_scan_cmd.set_defaults(func=command_github_benchmark_scan) - - intent_confidence_cmd = subparsers.add_parser( - "intent-confidence", - help="Render a confidence report for how well the real job, inputs, outputs, and exclusions are understood.", - ) - intent_confidence_cmd.add_argument("skill_dir", nargs="?", default=".") - intent_confidence_cmd.add_argument("--context-json") - intent_confidence_cmd.add_argument("--output-md") - intent_confidence_cmd.add_argument("--output-json") - intent_confidence_cmd.set_defaults(func=command_intent_confidence) - - intent_dialogue_cmd = subparsers.add_parser( - "intent-dialogue", - help="Render a front-loaded intent dialogue guide for a skill package.", - ) - intent_dialogue_cmd.add_argument("skill_dir", nargs="?", default=".") - intent_dialogue_cmd.add_argument("--output-md") - intent_dialogue_cmd.add_argument("--output-json") - intent_dialogue_cmd.set_defaults(func=command_intent_dialogue) - - reference_synthesis_cmd = subparsers.add_parser( - "reference-synthesis", - help="Render a multi-source reference synthesis report for a skill package.", - ) - reference_synthesis_cmd.add_argument("skill_dir", nargs="?", default=".") - reference_synthesis_cmd.add_argument("--output-md") - reference_synthesis_cmd.add_argument("--output-json") - reference_synthesis_cmd.set_defaults(func=command_reference_synthesis) - - output_risk_cmd = subparsers.add_parser( - "output-risk-profile", - help="Render predicted output failure modes and self-repair checks for a skill package.", - ) - output_risk_cmd.add_argument("skill_dir", nargs="?", default=".") - output_risk_cmd.add_argument("--output-md") - output_risk_cmd.add_argument("--output-json") - output_risk_cmd.set_defaults(func=command_output_risk_profile) - - artifact_design_cmd = subparsers.add_parser( - "artifact-design-profile", - help="Render artifact design direction and visual quality gates for a skill package.", - ) - artifact_design_cmd.add_argument("skill_dir", nargs="?", default=".") - artifact_design_cmd.add_argument("--output-md") - artifact_design_cmd.add_argument("--output-json") - artifact_design_cmd.set_defaults(func=command_artifact_design_profile) - - prompt_quality_cmd = subparsers.add_parser( - "prompt-quality-profile", - help="Render prompt-facing need model, RTF mapping, complexity, and quality checks for a skill package.", - ) - prompt_quality_cmd.add_argument("skill_dir", nargs="?", default=".") - prompt_quality_cmd.add_argument("--output-md") - prompt_quality_cmd.add_argument("--output-json") - prompt_quality_cmd.set_defaults(func=command_prompt_quality_profile) - - system_model_cmd = subparsers.add_parser( - "system-model", - help="Render a systems-thinking model for boundaries, feedback loops, drift, and leverage points.", - ) - system_model_cmd.add_argument("skill_dir", nargs="?", default=".") - system_model_cmd.add_argument("--output-md") - system_model_cmd.add_argument("--output-json") - system_model_cmd.set_defaults(func=command_system_model) - - iteration_directions_cmd = subparsers.add_parser( - "iteration-directions", - help="Render the top three next iteration directions for a skill package.", - ) - iteration_directions_cmd.add_argument("skill_dir", nargs="?", default=".") - iteration_directions_cmd.add_argument("--output-md") - iteration_directions_cmd.add_argument("--output-json") - iteration_directions_cmd.set_defaults(func=command_iteration_directions) - - feedback_cmd = subparsers.add_parser( - "feedback", - help="Capture lightweight reviewer feedback without running the full promotion flow.", - ) - feedback_cmd.add_argument("skill_dir", nargs="?", default=".") - feedback_cmd.add_argument("--note") - feedback_cmd.add_argument("--rating", type=int, default=3) - feedback_cmd.add_argument("--category", default="general") - feedback_cmd.add_argument("--recommended-action", default="review") - feedback_cmd.set_defaults(func=command_feedback) - - adoption_drift_cmd = subparsers.add_parser( - "adoption-drift", - help="Render local-first metadata-only adoption and drift telemetry for a skill package.", - ) - adoption_drift_cmd.add_argument("skill_dir", nargs="?", default=".") - adoption_drift_cmd.add_argument("--events-jsonl") - adoption_drift_cmd.add_argument("--output-json") - adoption_drift_cmd.add_argument("--output-md") - adoption_drift_cmd.add_argument("--generated-at") - adoption_drift_cmd.add_argument( - "--record-event", - choices=["review_event", "script_run", "skill_activation", "skill_output"], - ) - adoption_drift_cmd.add_argument( - "--activation-type", - choices=["explicit", "implicit", "manual", "unknown"], - default="unknown", - ) - adoption_drift_cmd.add_argument( - "--outcome", - choices=["accepted", "edited", "failed", "missed", "rejected", "reviewed", "unknown"], - default="unknown", - ) - adoption_drift_cmd.add_argument( - "--failure-type", - choices=[ - "bad_output", - "missing_resource", - "none", - "review_overdue", - "script_error", - "under_trigger", - "wrong_trigger", - ], - default="none", - ) - adoption_drift_cmd.add_argument("--timestamp") - adoption_drift_cmd.add_argument("--skill-name") - adoption_drift_cmd.add_argument("--version") - adoption_drift_cmd.set_defaults(func=command_adoption_drift) - - review_waivers_cmd = subparsers.add_parser( - "review-waivers", - help="Render or update human reviewer waiver evidence for Review Studio.", - ) - review_waivers_cmd.add_argument("skill_dir", nargs="?", default=".") - review_waivers_cmd.add_argument("--waivers-json") - review_waivers_cmd.add_argument("--output-json") - review_waivers_cmd.add_argument("--output-md") - review_waivers_cmd.add_argument("--generated-at") - review_waivers_cmd.add_argument("--add-waiver", action="store_true") - review_waivers_cmd.add_argument( - "--gate-key", - choices=[ - "context-budget", - "intent-canvas", - "operations-loop", - "output-lab", - "registry-audit", - "release-notes", - "runtime-matrix", - "skill-atlas", - "trigger-lab", - "trust-report", - "permission-gates", - "permission-runtime", - ], - ) - review_waivers_cmd.add_argument( - "--decision", - choices=["accepted-risk", "false-positive", "temporary-exception"], - default="accepted-risk", - ) - review_waivers_cmd.add_argument("--reviewer") - review_waivers_cmd.add_argument("--reason") - review_waivers_cmd.add_argument("--expires-at") - review_waivers_cmd.add_argument("--created-at") - review_waivers_cmd.add_argument("--evidence") - review_waivers_cmd.add_argument("--scope", default="current-release") - review_waivers_cmd.set_defaults(func=command_review_waivers) - - review_annotations_cmd = subparsers.add_parser( - "review-annotations", - help="Render or update inline reviewer annotations for Review Studio gates and source paths.", - ) - review_annotations_cmd.add_argument("skill_dir", nargs="?", default=".") - review_annotations_cmd.add_argument("--annotations-json") - review_annotations_cmd.add_argument("--output-json") - review_annotations_cmd.add_argument("--output-md") - review_annotations_cmd.add_argument("--write-template", action="store_true") - review_annotations_cmd.add_argument("--add-annotation", action="store_true") - review_annotations_cmd.add_argument("--annotation-id") - review_annotations_cmd.add_argument( - "--gate-key", - choices=[ - "context-budget", - "intent-canvas", - "operations-loop", - "output-lab", - "registry-audit", - "release-notes", - "review-waivers", - "runtime-matrix", - "skill-atlas", - "trigger-lab", - "trust-report", - "permission-gates", - "permission-runtime", - ], - ) - review_annotations_cmd.add_argument("--target-path") - review_annotations_cmd.add_argument("--line", type=int) - review_annotations_cmd.add_argument("--severity", choices=["blocker", "info", "note", "warning"], default="note") - review_annotations_cmd.add_argument("--status", choices=["deferred", "open", "resolved"], default="open") - review_annotations_cmd.add_argument("--reviewer") - review_annotations_cmd.add_argument("--created-at") - review_annotations_cmd.add_argument("--body") - review_annotations_cmd.add_argument("--suggested-action") - review_annotations_cmd.add_argument("--evidence") - review_annotations_cmd.set_defaults(func=command_review_annotations) - - baseline_compare_cmd = subparsers.add_parser( - "baseline-compare", - help="Render a lightweight with-skill vs baseline comparison across tracked targets.", - ) - baseline_compare_cmd.set_defaults(func=command_baseline_compare) - - skill_ir_cmd = subparsers.add_parser("skill-ir", help="Export platform-neutral Skill IR for a skill package.") - skill_ir_cmd.add_argument("skill_dir", nargs="?", default=".") - skill_ir_cmd.add_argument("--output-json") - skill_ir_cmd.add_argument("--validate-only", action="store_true") - skill_ir_cmd.set_defaults(func=command_skill_ir) - - compile_skill_cmd = subparsers.add_parser( - "compile-skill", - help="Compile Skill IR into target-specific semantic contracts.", - ) - compile_skill_cmd.add_argument("skill_dir", nargs="?", default=".") - compile_skill_cmd.add_argument("--target", action="append") - compile_skill_cmd.add_argument("--output-json") - compile_skill_cmd.add_argument("--output-md") - compile_skill_cmd.add_argument("--generated-at") - compile_skill_cmd.set_defaults(func=command_compile_skill) - - output_eval_cmd = subparsers.add_parser("output-eval", help="Run Output Eval Lab assertion grading.") - output_eval_cmd.add_argument("--cases") - output_eval_cmd.add_argument("--output-json") - output_eval_cmd.add_argument("--output-md") - output_eval_cmd.add_argument("--blind-pack-json") - output_eval_cmd.add_argument("--blind-pack-md") - output_eval_cmd.add_argument("--blind-answer-key-json") - output_eval_cmd.set_defaults(func=command_output_eval) - - output_execution_cmd = subparsers.add_parser( - "output-exec", - help="Record output-eval execution evidence, timing, and token usage.", - ) - output_execution_cmd.add_argument("--cases") - output_execution_cmd.add_argument("--output-json") - output_execution_cmd.add_argument("--output-md") - output_execution_cmd.add_argument("--runner-command") - output_execution_cmd.add_argument("--timeout-seconds", type=float) - output_execution_cmd.set_defaults(func=command_output_execution) - - output_review_cmd = subparsers.add_parser( - "output-review", - help="Adjudicate blind A/B output review decisions against the answer key.", - ) - output_review_cmd.add_argument("--blind-pack") - output_review_cmd.add_argument("--answer-key") - output_review_cmd.add_argument("--decisions") - output_review_cmd.add_argument("--output-json") - output_review_cmd.add_argument("--output-md") - output_review_cmd.add_argument("--write-template", action="store_true") - output_review_cmd.set_defaults(func=command_output_review) - - conformance_cmd = subparsers.add_parser("conformance", help="Run runtime conformance checks for Skill OS targets.") - conformance_cmd.add_argument("skill_dir", nargs="?", default=".") - conformance_cmd.add_argument("--target", action="append", choices=["openai", "claude", "agent-skills", "vscode", "generic"]) - conformance_cmd.add_argument("--output-json") - conformance_cmd.add_argument("--output-md") - conformance_cmd.set_defaults(func=command_conformance) - - runtime_permissions_cmd = subparsers.add_parser( - "runtime-permissions", - help="Probe generated target adapters for runtime permission enforcement metadata.", - ) - runtime_permissions_cmd.add_argument("skill_dir", nargs="?", default=".") - runtime_permissions_cmd.add_argument("--package-dir", default="dist") - runtime_permissions_cmd.add_argument("--target", action="append", choices=["openai", "claude", "generic"]) - runtime_permissions_cmd.add_argument("--output-json") - runtime_permissions_cmd.add_argument("--output-md") - runtime_permissions_cmd.set_defaults(func=command_runtime_permissions) - - trust_cmd = subparsers.add_parser("trust", help="Run trust and security checks for a skill package.") - trust_cmd.add_argument("skill_dir", nargs="?", default=".") - trust_cmd.add_argument("--output-json") - trust_cmd.add_argument("--output-md") - trust_cmd.set_defaults(func=command_trust) - - skill_atlas_cmd = subparsers.add_parser("skill-atlas", help="Build a portfolio-level Skill Atlas for a workspace.") - skill_atlas_cmd.add_argument("--workspace-root", default=".") - skill_atlas_cmd.add_argument("--output-dir") - skill_atlas_cmd.add_argument("--report-html") - skill_atlas_cmd.add_argument("--report-json") - skill_atlas_cmd.add_argument("--overlap-threshold", type=float) - skill_atlas_cmd.add_argument("--today") - skill_atlas_cmd.set_defaults(func=command_skill_atlas) - - registry_audit_cmd = subparsers.add_parser("registry-audit", help="Build and audit Skill OS registry package metadata.") - registry_audit_cmd.add_argument("skill_dir", nargs="?", default=".") - registry_audit_cmd.add_argument("--registry-dir") - registry_audit_cmd.add_argument("--output-json") - registry_audit_cmd.add_argument("--output-md") - registry_audit_cmd.add_argument("--generated-at") - registry_audit_cmd.set_defaults(func=command_registry_audit) - - package_cmd = subparsers.add_parser("package", help="Export compatibility artifacts for selected targets.") - package_cmd.add_argument("skill_dir", nargs="?", default=".") - package_cmd.add_argument("--platform", action="append") - package_cmd.add_argument("--output-dir", default="dist") - package_cmd.add_argument("--expectations") - package_cmd.add_argument("--zip", action="store_true") - package_cmd.set_defaults(func=command_package) - - package_verify_cmd = subparsers.add_parser("package-verify", help="Verify generated package artifacts, archive safety, and registry parity.") - package_verify_cmd.add_argument("skill_dir", nargs="?", default=".") - package_verify_cmd.add_argument("--package-dir", default="dist") - package_verify_cmd.add_argument("--expectations", default="evals/packaging_expectations.json") - package_verify_cmd.add_argument("--registry-json", default="reports/registry_audit.json") - package_verify_cmd.add_argument("--output-json") - package_verify_cmd.add_argument("--output-md") - package_verify_cmd.add_argument("--require-zip", action="store_true") - package_verify_cmd.add_argument("--generated-at") - package_verify_cmd.set_defaults(func=command_package_verify) - - install_simulate_cmd = subparsers.add_parser("install-simulate", help="Simulate installing a generated package into a temporary local skill root.") - install_simulate_cmd.add_argument("skill_dir", nargs="?", default=".") - install_simulate_cmd.add_argument("--package-dir", default="dist") - install_simulate_cmd.add_argument("--install-root") - install_simulate_cmd.add_argument("--output-json") - install_simulate_cmd.add_argument("--output-md") - install_simulate_cmd.add_argument("--generated-at") - install_simulate_cmd.set_defaults(func=command_install_simulate) - - upgrade_check_cmd = subparsers.add_parser("upgrade-check", help="Compare current and previous registry package metadata for upgrade readiness.") - upgrade_check_cmd.add_argument("skill_dir", nargs="?", default=".") - upgrade_check_cmd.add_argument("--previous-package-json", required=True) - upgrade_check_cmd.add_argument("--current-package-json", default="reports/registry_audit.json") - upgrade_check_cmd.add_argument("--output-json") - upgrade_check_cmd.add_argument("--output-md") - upgrade_check_cmd.add_argument("--generated-at") - upgrade_check_cmd.set_defaults(func=command_upgrade_check) - - test_cmd = subparsers.add_parser("test", help="Run a Makefile test target.") - test_cmd.add_argument("--target", default="test") - test_cmd.set_defaults(func=command_test) - - update_cmd = subparsers.add_parser("check-update", help="Check whether a newer yao-meta-skill version is available.") - update_cmd.add_argument("--force", action="store_true") - update_cmd.add_argument("--no-cache", action="store_true") - update_cmd.add_argument("--version-url") - update_cmd.add_argument("--manifest-url") - update_cmd.add_argument("--timeout", type=float, default=3.0) - update_cmd.add_argument("--allow-custom-update-url", action="store_true") - update_cmd.set_defaults(func=command_check_update) - + parser = build_cli_parser({name: value for name, value in globals().items() if name.startswith("command_")}) + add_telemetry_args(parser) return parser def main() -> None: parser = build_parser() args = parser.parse_args() - raise SystemExit(args.func(args)) + returncode = 2 + try: + returncode = args.func(args) + finally: + maybe_record_cli_event(ROOT, args, returncode) + raise SystemExit(returncode) if __name__ == "__main__": diff --git a/scripts/yao_cli_adaptation_commands.py b/scripts/yao_cli_adaptation_commands.py new file mode 100644 index 00000000..184e5537 --- /dev/null +++ b/scripts/yao_cli_adaptation_commands.py @@ -0,0 +1,121 @@ +"""Adaptive scan, proposal, and apply command handlers for the Yao CLI.""" + +import argparse +import json +from pathlib import Path + +from yao_cli_runtime import run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep adaptive scan/proposal/apply command handlers outside the thin CLI orchestrator." + + +def command_adapt_scan(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir, "--source", args.source] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.min_support is not None: + cmd.extend(["--min-support", str(args.min_support)]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.allow_history_source: + cmd.append("--allow-history-source") + result = run_script("summarize_user_signals.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_adapt_propose(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.patterns_json: + cmd.extend(["--patterns-json", args.patterns_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("propose_adaptation.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_adapt_apply(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.proposal_id: + cmd.extend(["--proposal-id", args.proposal_id]) + if args.patch_file: + cmd.extend(["--patch-file", args.patch_file]) + if args.proposals_json: + cmd.extend(["--proposals-json", args.proposals_json]) + if args.approval_ledger: + cmd.extend(["--approval-ledger", args.approval_ledger]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.today: + cmd.extend(["--today", args.today]) + if args.write_template: + cmd.append("--write-template") + if args.prepare_approval: + cmd.append("--prepare-approval") + if args.apply: + cmd.append("--apply") + if args.run_verification: + cmd.append("--run-verification") + if not args.rollback_on_failure: + cmd.append("--no-rollback-on-failure") + result = run_script("apply_adaptation.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_daily_skillops(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.source: + cmd.extend(["--source", args.source]) + if args.patterns_json: + cmd.extend(["--patterns-json", args.patterns_json]) + if args.proposals_json: + cmd.extend(["--proposals-json", args.proposals_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.min_support is not None: + cmd.extend(["--min-support", str(args.min_support)]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.allow_history_source: + cmd.append("--allow-history-source") + if args.no_refresh_source_reports: + cmd.append("--no-refresh-source-reports") + result = run_script("render_daily_skillops_report.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_weekly_curator(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + for path in args.daily_json: + cmd.extend(["--daily-json", path]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("render_weekly_curator_report.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 diff --git a/scripts/yao_cli_config.py b/scripts/yao_cli_config.py index f8bf5bbe..e3383caf 100644 --- a/scripts/yao_cli_config.py +++ b/scripts/yao_cli_config.py @@ -68,6 +68,41 @@ def local_output_runner_command() -> str: return json.dumps(["python3", "scripts/local_output_eval_runner.py"]) +def provider_output_runner_command( + provider: str, + model: str | None = None, + base_url: str | None = None, + api_format: str | None = None, + thinking: str | None = None, + temperature: float | None = None, + api_key_env: str | None = None, + allow_insecure_localhost: bool = False, + allow_custom_base_url: bool = False, +) -> str: + command = ["python3", "scripts/provider_output_eval_runner.py", "--provider", provider] + if provider == "deepseek": + api_format = api_format or "chat-completions" + thinking = thinking or "disabled" + api_key_env = api_key_env or "DEEPSEEK_API_KEY" + if model: + command.extend(["--model", model]) + if base_url: + command.extend(["--base-url", base_url]) + if api_format: + command.extend(["--api-format", api_format]) + if thinking: + command.extend(["--thinking", thinking]) + if temperature is not None: + command.extend(["--temperature", str(temperature)]) + if api_key_env: + command.extend(["--api-key-env", api_key_env]) + if allow_insecure_localhost: + command.append("--allow-insecure-localhost") + if allow_custom_base_url: + command.append("--allow-custom-base-url") + return json.dumps(command) + + def resolve_target(name: str) -> dict: if name not in TARGETS: raise KeyError(f"Unknown target: {name}") diff --git a/scripts/yao_cli_create_commands.py b/scripts/yao_cli_create_commands.py new file mode 100644 index 00000000..265e6eda --- /dev/null +++ b/scripts/yao_cli_create_commands.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Creation command handlers for the Yao CLI.""" + +import argparse +import json +import sys + +from github_benchmark_scan import build_query +from render_intent_confidence import assess_intent_confidence +from yao_cli_config import ( + ARCHETYPE_MODE, + archetype_guidance, + diagnose_skill_candidates, + diagnosis_note, + discovery_summary, + infer_archetype, + recommendation_from_synthesis, + reference_visibility, +) +from yao_cli_runtime import run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator." + + +def prompt_with_default(label: str, default: str) -> str: + sys.stderr.write(f"{label} [{default}]: ") + sys.stderr.flush() + value = sys.stdin.readline().strip() + return value or default + + +def prompt_optional(label: str, default: str = "skip") -> str: + sys.stderr.write(f"{label} [{default}]: ") + sys.stderr.flush() + value = sys.stdin.readline().strip() + return value or default + + +def prompt_optional_entries(label: str) -> list[str]: + sys.stderr.write(f"{label} [none]: ") + sys.stderr.flush() + value = sys.stdin.readline().strip() + if not value or value.lower() in {"none", "no", "n"}: + return [] + return [item.strip() for item in value.split(",") if item.strip()] + + +def update_context_slot(context: dict, slot: str, answer: str, list_mode: bool) -> None: + value = answer.strip() + if not value or value.lower() in {"skip", "none", "no", "n"}: + return + if list_mode: + context[slot] = [item.strip() for item in value.split(",") if item.strip()] + else: + context[slot] = value + + +def intent_confidence_note(summary: dict) -> str: + lines = [ + f"\nIntent confidence: {summary['score']}/100 ({summary['band']}).", + f"- Recommended action: {summary['recommended_action']}", + ] + if summary.get("gaps"): + top_gap = summary["gaps"][0] + lines.append(f"- Biggest gap: {top_gap['label']} — {top_gap['reason']}") + return "\n".join(lines) + "\n" + + +def maybe_emit_update_notice(args: argparse.Namespace) -> None: + if getattr(args, "no_update_check", False): + return + result = run_script("check_update.py", []) + payload = result["payload"] if result["payload"] is not None else {} + if not result["ok"] and not payload: + return + if payload.get("update_available"): + sys.stderr.write( + "\nUpdate available for yao-meta-skill: " + f"{payload.get('local_version')} -> {payload.get('remote_version')}.\n" + f"Run: {payload.get('install_hint')}\n" + ) + + +def command_init(args: argparse.Namespace) -> int: + cmd = [ + args.name, + "--description", + args.description, + "--output-dir", + args.output_dir, + "--mode", + args.mode, + "--archetype", + args.archetype, + *(["--title", args.title] if args.title else []), + ] + for reference in args.external_reference: + cmd.extend(["--external-reference", reference]) + for reference in args.user_reference: + cmd.extend(["--user-reference", reference]) + for constraint in args.local_constraint: + cmd.extend(["--local-constraint", constraint]) + if args.github_query: + cmd.extend(["--github-query", args.github_query]) + cmd.extend(["--github-top-n", str(args.github_top_n)]) + if args.github_fixture_dir: + cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) + if args.intent_job: + cmd.extend(["--intent-job", args.intent_job]) + for item in args.intent_real_input: + cmd.extend(["--intent-real-input", item]) + if args.intent_primary_output: + cmd.extend(["--intent-primary-output", args.intent_primary_output]) + for item in args.intent_exclusion: + cmd.extend(["--intent-exclusion", item]) + for item in args.intent_constraint: + cmd.extend(["--intent-constraint", item]) + for item in args.intent_standard: + cmd.extend(["--intent-standard", item]) + if args.intent_correction: + cmd.extend(["--intent-correction", args.intent_correction]) + result = run_script("init_skill.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_quickstart(args: argparse.Namespace) -> int: + maybe_emit_update_notice(args) + sys.stderr.write("Let's start gently. You do not need a polished brief here.\n") + sys.stderr.write("Give me the real work in your own words, and I will help turn it into a clean first-pass skill.\n") + sys.stderr.write("While we shape the first pass, I will quietly check a few strong public patterns in the background and only surface them if there is real uncertainty or a design conflict.\n") + name = args.name or prompt_with_default("Skill name", "my-skill") + job = args.job or prompt_with_default( + "In your own words, what repeated work do you most want this skill to reliably handle", + "Turn a repeated workflow into a reusable skill.", + ) + real_inputs = args.real_input or prompt_optional_entries( + "What material will people actually hand to this skill in practice (comma-separated)" + ) + primary_output = args.primary_output or prompt_with_default( + "If it works beautifully, what should it hand back so you or the next person can keep moving", + "A reusable skill package.", + ) + description = args.description or f"{job.rstrip('.')} Primary output: {primary_output.rstrip('.')}." + intent_context = { + "job": job, + "real_inputs": real_inputs, + "primary_output": primary_output, + "description": description, + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [], + } + inferred_archetype, archetype_reason = infer_archetype(job, description) + guidance = archetype_guidance(inferred_archetype) + sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) + correction = prompt_optional( + "If I am off, what is the first thing I should correct before I package this idea", + "looks right", + ) + if correction.lower() not in {"looks right", "skip", "none", "no"}: + description = f"{description.rstrip('.')} Keep this correction in scope: {correction.rstrip('.')}." + intent_context["description"] = description + intent_context["correction"] = correction + inferred_archetype, archetype_reason = infer_archetype(job, description) + guidance = archetype_guidance(inferred_archetype) + sys.stderr.write("\nThanks. I tightened the frame before moving on.\n") + sys.stderr.write(discovery_summary(job, primary_output, inferred_archetype, guidance)) + confidence = assess_intent_confidence(intent_context) + sys.stderr.write(intent_confidence_note(confidence)) + diagnosis = diagnose_skill_candidates(job, primary_output, inferred_archetype, confidence) + if diagnosis["fuzzy"]: + sys.stderr.write(diagnosis_note(diagnosis)) + if not confidence["gate_passed"]: + sys.stderr.write("Before I package this idea, I want to close the highest-leverage gaps instead of guessing.\n") + for follow_up in confidence.get("follow_up_questions", [])[:2]: + answer = prompt_optional(follow_up["question"], "skip") + update_context_slot(intent_context, follow_up["slot"], answer, follow_up["list"]) + confidence = assess_intent_confidence(intent_context) + sys.stderr.write("\nI tightened the intent frame once more before moving on.\n") + sys.stderr.write(intent_confidence_note(confidence)) + diagnosis = diagnose_skill_candidates(job, primary_output, inferred_archetype, confidence) + if diagnosis["fuzzy"]: + sys.stderr.write(diagnosis_note(diagnosis)) + archetype = args.archetype or prompt_with_default("I would start with this archetype (scaffold/production/library/governed)", inferred_archetype) + archetype = archetype if archetype in ARCHETYPE_MODE else inferred_archetype + default_mode = ARCHETYPE_MODE[archetype] + mode = args.mode or prompt_with_default("For the first pass, I would keep the mode here (scaffold/production/library/governed)", default_mode) + mode = mode if mode in ARCHETYPE_MODE.values() else default_mode + diagnosis = diagnose_skill_candidates(job, primary_output, archetype, confidence) + guidance = archetype_guidance(archetype) + sys.stderr.write( + f"\nGood. I will treat this as `{archetype}` in `{mode}` mode, so the first pass stays focused on {guidance['focus']}.\n" + ) + user_references = args.user_reference or prompt_optional_entries( + "If there is anything you admire and want me to learn from as pattern hints, send it here (repo, product, page, workflow; comma-separated)" + ) + external_references = args.external_reference or [] + prompted_constraints = args.constraint if getattr(args, "constraint", None) else ([] if args.local_constraint else prompt_optional_entries( + "Tell me any local constraints I must keep in view (privacy, naming, compatibility; comma-separated)" + )) + local_constraints = args.local_constraint or prompted_constraints or intent_context.get("constraints", []) + intent_context["user_references"] = user_references + intent_context["constraints"] = local_constraints + confidence = assess_intent_confidence(intent_context) + github_query = args.github_query or build_query(" ".join(filter(None, [job, primary_output, description]))) + title = args.title or name.replace("-", " ").title() + guidance = archetype_guidance(archetype) + cmd = [ + name, + "--description", + description, + "--title", + title, + "--output-dir", + args.output_dir, + "--mode", + mode, + "--archetype", + archetype, + "--github-query", + github_query, + "--github-top-n", + str(args.github_top_n), + "--intent-job", + job, + "--intent-primary-output", + primary_output, + ] + for item in real_inputs: + cmd.extend(["--intent-real-input", item]) + for item in intent_context.get("exclusions", []): + cmd.extend(["--intent-exclusion", item]) + for item in intent_context.get("constraints", []): + cmd.extend(["--intent-constraint", item]) + for item in intent_context.get("standards", []): + cmd.extend(["--intent-standard", item]) + if intent_context.get("correction"): + cmd.extend(["--intent-correction", intent_context["correction"]]) + if args.github_fixture_dir: + cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) + for reference in external_references: + cmd.extend(["--external-reference", reference]) + for reference in user_references: + cmd.extend(["--user-reference", reference]) + for constraint in local_constraints: + cmd.extend(["--local-constraint", constraint]) + result = run_script("init_skill.py", cmd) + payload = result["payload"] if result["payload"] is not None else result + reference_synthesis = payload.get("reference_synthesis") or {} + visibility = reference_visibility(reference_synthesis) + recommendation = recommendation_from_synthesis(reference_synthesis, visibility) + sys.stderr.write(f"\nRecommendation: {recommendation['summary']}\n") + if visibility["user_decision_required"]: + if visibility["conflicts"]: + sys.stderr.write(f"I am surfacing this because there is a real design conflict: {visibility['conflicts'][0]['summary']}\n") + else: + sys.stderr.write("I am surfacing this because intent is still settling and the package should not deepen on guesswork.\n") + else: + sys.stderr.write("I will keep the underlying benchmark evidence in the reviewer reports and move ahead with this recommendation.\n") + if payload.get("report_view", {}).get("html_report"): + sys.stderr.write(f"Skill report: {payload['report_view']['html_report']}\n") + if payload.get("report_view", {}).get("interpretation_report"): + sys.stderr.write(f"Skill interpretation: {payload['report_view']['interpretation_report']}\n") + + next_steps = [ + "Open reports/skill-interpretation.html to review the generated Skill interpretation report.", + "Open reports/skill-overview.html to review the generated Skill audit report.", + "Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.", + "Open reports/review-studio.html to inspect the Review Studio 2.0 gate view before release.", + "Open reports/review-viewer.html to explain the package to a first-time reviewer.", + "Use reports/iteration-directions.md to choose only one high-value next move before adding more files.", + ] + if visibility["user_decision_required"]: + next_steps.insert( + 1, + "Open reports/reference-synthesis.md if you want to inspect why the recommendation was surfaced and which tradeoff needs a call.", + ) + report = { + "ok": result["ok"], + "root": payload.get("root"), + "mode": mode, + "archetype": archetype, + "artifacts": payload.get("artifacts", {}), + "report_view": payload.get("report_view", {}), + "intent_confidence": { + "score": confidence["score"], + "band": confidence["band"], + "gate_passed": confidence["gate_passed"], + "recommended_action": confidence["recommended_action"], + }, + "recommendation": recommendation, + "reference_mode": { + "mode": visibility["mode"], + "user_decision_required": visibility["user_decision_required"], + }, + "reviewer_evidence": { + "visibility": "full evidence in reports and review-viewer", + "artifacts": { + "benchmark_scan": payload.get("artifacts", {}).get("github_benchmark_scan_md"), + "reference_synthesis": payload.get("artifacts", {}).get("reference_synthesis_md"), + "artifact_design_profile": payload.get("artifacts", {}).get("artifact_design_profile_md"), + "prompt_quality_profile": payload.get("artifacts", {}).get("prompt_quality_profile_md"), + "system_model": payload.get("artifacts", {}).get("system_model_md"), + "skill_interpretation": payload.get("artifacts", {}).get("skill_interpretation_html"), + "review_studio": payload.get("artifacts", {}).get("review_studio_html"), + "review_viewer": payload.get("artifacts", {}).get("review_viewer_html"), + }, + }, + "guidance": { + "archetype_reason": archetype_reason, + "problem_diagnosis": diagnosis, + "why_this_mode": ( + "Scaffold mode keeps the first package light and lets you postpone governance-heavy work until reuse becomes real." + if mode == "scaffold" + else "This mode expects stronger lifecycle metadata, validation, and review discipline." + ), + "first_gate": guidance["first_gate"], + "focus": guidance["focus"], + "next_steps": next_steps, + "experience_note": ( + "The first pass should feel more like guided co-creation than a worksheet. " + "The system should make benchmark and pattern calls quietly unless there is a real reason to ask you to choose." + ), + }, + } + if visibility["user_decision_required"]: + report["uncertainty_or_conflict"] = { + "reasons": visibility["reasons"], + "conflicts": visibility["conflicts"], + "note": "A design decision still needs your input before the package should be deepened.", + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 diff --git a/scripts/yao_cli_distribution_commands.py b/scripts/yao_cli_distribution_commands.py new file mode 100644 index 00000000..95856e96 --- /dev/null +++ b/scripts/yao_cli_distribution_commands.py @@ -0,0 +1,164 @@ +"""Distribution, packaging, and runtime-gate command handlers for the Yao CLI.""" + +import argparse +import json +from pathlib import Path + +from yao_cli_runtime import run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep distribution and runtime gate handlers outside the thin CLI orchestrator." + + +def emit_result(result: dict) -> int: + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_skill_ir(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.validate_only: + cmd.append("--validate-only") + return emit_result(run_script("export_skill_ir.py", cmd)) + + +def command_compile_skill(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("compile_skill.py", cmd)) + + +def command_conformance(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + return emit_result(run_script("run_conformance_suite.py", cmd)) + + +def command_runtime_permissions(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + for target in args.target or []: + cmd.extend(["--target", target]) + if getattr(args, "install_simulation_json", None): + cmd.extend(["--install-simulation-json", args.install_simulation_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + return emit_result(run_script("probe_runtime_permissions.py", cmd)) + + +def command_trust(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + return emit_result(run_script("trust_check.py", cmd)) + + +def command_skill_atlas(args: argparse.Namespace) -> int: + cmd = ["--workspace-root", str(Path(args.workspace_root).resolve())] + if args.output_dir: + cmd.extend(["--output-dir", args.output_dir]) + if args.report_html: + cmd.extend(["--report-html", args.report_html]) + if args.report_json: + cmd.extend(["--report-json", args.report_json]) + if args.overlap_threshold is not None: + cmd.extend(["--overlap-threshold", str(args.overlap_threshold)]) + if args.today: + cmd.extend(["--today", args.today]) + return emit_result(run_script("build_skill_atlas.py", cmd)) + + +def command_registry_audit(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.registry_dir: + cmd.extend(["--registry-dir", args.registry_dir]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("registry_audit.py", cmd)) + + +def command_package_verify(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + if args.expectations: + cmd.extend(["--expectations", args.expectations]) + if args.registry_json: + cmd.extend(["--registry-json", args.registry_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.require_zip: + cmd.append("--require-zip") + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("verify_package.py", cmd)) + + +def command_install_simulate(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + if args.install_root: + cmd.extend(["--install-root", args.install_root]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("simulate_install.py", cmd)) + + +def command_upgrade_check(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + cmd.extend(["--previous-package-json", args.previous_package_json]) + if args.current_package_json: + cmd.extend(["--current-package-json", args.current_package_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("upgrade_check.py", cmd)) + + +def command_package(args: argparse.Namespace) -> int: + cmd = [ + str(Path(args.skill_dir).resolve()), + "--output-dir", + args.output_dir, + ] + for platform in args.platform or ["generic"]: + cmd.extend(["--platform", platform]) + if args.expectations: + cmd.extend(["--expectations", args.expectations]) + if args.zip: + cmd.append("--zip") + return emit_result(run_script("cross_packager.py", cmd)) diff --git a/scripts/yao_cli_output_commands.py b/scripts/yao_cli_output_commands.py new file mode 100644 index 00000000..170a3aeb --- /dev/null +++ b/scripts/yao_cli_output_commands.py @@ -0,0 +1,134 @@ +"""Output evaluation and human-review command handlers for the Yao CLI.""" + +import argparse +import json + +from yao_cli_config import provider_output_runner_command +from yao_cli_runtime import run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep output evaluation and review handlers outside the thin CLI orchestrator." + + +def emit_result(result: dict) -> int: + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_output_eval(args: argparse.Namespace) -> int: + cmd = [] + if args.cases: + cmd.extend(["--cases", args.cases]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.blind_pack_json: + cmd.extend(["--blind-pack-json", args.blind_pack_json]) + if args.blind_pack_md: + cmd.extend(["--blind-pack-md", args.blind_pack_md]) + if args.blind_answer_key_json: + cmd.extend(["--blind-answer-key-json", args.blind_answer_key_json]) + return emit_result(run_script("run_output_eval.py", cmd)) + + +def command_output_execution(args: argparse.Namespace) -> int: + cmd = [] + if args.cases: + cmd.extend(["--cases", args.cases]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.runner_command and args.provider_runner: + payload = { + "schema_version": "1.0", + "ok": False, + "failures": ["Use either --runner-command or --provider-runner, not both."], + } + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 2 + if args.provider_runner: + cmd.extend( + [ + "--runner-command", + provider_output_runner_command( + args.provider_runner, + model=args.provider_model, + base_url=args.provider_base_url, + api_format=args.provider_api_format, + thinking=args.provider_thinking, + temperature=args.provider_temperature, + api_key_env=args.api_key_env, + allow_insecure_localhost=args.allow_insecure_localhost, + allow_custom_base_url=args.allow_custom_base_url, + ), + ] + ) + elif args.runner_command: + cmd.extend(["--runner-command", args.runner_command]) + if args.timeout_seconds is not None: + cmd.extend(["--timeout-seconds", str(args.timeout_seconds)]) + return emit_result(run_script("run_output_execution.py", cmd)) + + +def command_output_review_kit(args: argparse.Namespace) -> int: + cmd = [] + if args.blind_pack_json: + cmd.extend(["--blind-pack-json", args.blind_pack_json]) + if args.blind_pack_md: + cmd.extend(["--blind-pack-md", args.blind_pack_md]) + if args.decisions: + cmd.extend(["--decisions", args.decisions]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_html: + cmd.extend(["--output-html", args.output_html]) + if args.write_template: + cmd.append("--write-template") + return emit_result(run_script("prepare_output_review_kit.py", cmd)) + + +def command_output_review(args: argparse.Namespace) -> int: + cmd = [] + if args.blind_pack: + cmd.extend(["--blind-pack", args.blind_pack]) + if args.answer_key: + cmd.extend(["--answer-key", args.answer_key]) + if args.decisions: + cmd.extend(["--decisions", args.decisions]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.write_template: + cmd.append("--write-template") + return emit_result(run_script("adjudicate_output_review.py", cmd)) + + +def command_output_review_import(args: argparse.Namespace) -> int: + cmd = ["--input", args.input] + if args.format: + cmd.extend(["--format", args.format]) + if args.blind_pack: + cmd.extend(["--blind-pack", args.blind_pack]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.reviewer: + cmd.extend(["--reviewer", args.reviewer]) + if args.reviewed_at: + cmd.extend(["--reviewed-at", args.reviewed_at]) + if args.blind_review_attested: + cmd.append("--blind-review-attested") + if args.run_adjudication: + cmd.append("--run-adjudication") + if args.answer_key: + cmd.extend(["--answer-key", args.answer_key]) + if args.adjudication_json: + cmd.extend(["--adjudication-json", args.adjudication_json]) + if args.adjudication_md: + cmd.extend(["--adjudication-md", args.adjudication_md]) + return emit_result(run_script("import_output_review_decisions.py", cmd)) diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py new file mode 100644 index 00000000..60e5e916 --- /dev/null +++ b/scripts/yao_cli_parser.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +"""Argparse surface for the Yao CLI.""" + +import argparse +from collections.abc import Callable + +from yao_cli_parser_evidence import add_evidence_commands +from yao_cli_parser_operations import add_operating_loop_commands + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep CLI parser declarations separate from command orchestration." + + +def _handler(command_handlers: dict[str, Callable[[argparse.Namespace], int]], name: str) -> Callable[[argparse.Namespace], int]: + if name not in command_handlers: + raise KeyError(f"Missing CLI command handler: {name}") + return command_handlers[name] + + +def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]]) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Unified authoring CLI for yao-meta-skill.") + subparsers = parser.add_subparsers(dest="command", required=True) + + init_cmd = subparsers.add_parser("init", help="Initialize a minimal skill package.") + init_cmd.add_argument("name") + init_cmd.add_argument("--description", default="Describe what the skill does and when to use it.") + init_cmd.add_argument("--title") + init_cmd.add_argument("--output-dir", default=".") + init_cmd.add_argument("--mode", choices=["scaffold", "production", "library", "governed"], default="scaffold") + init_cmd.add_argument("--archetype", choices=["scaffold", "production", "library", "governed"], default="scaffold") + init_cmd.add_argument("--external-reference", action="append", default=[]) + init_cmd.add_argument("--user-reference", action="append", default=[]) + init_cmd.add_argument("--local-constraint", action="append", default=[]) + init_cmd.add_argument("--github-query") + init_cmd.add_argument("--github-top-n", type=int, default=3) + init_cmd.add_argument("--github-fixture-dir") + init_cmd.add_argument("--intent-job") + init_cmd.add_argument("--intent-real-input", action="append", default=[]) + init_cmd.add_argument("--intent-primary-output") + init_cmd.add_argument("--intent-exclusion", action="append", default=[]) + init_cmd.add_argument("--intent-constraint", action="append", default=[]) + init_cmd.add_argument("--intent-standard", action="append", default=[]) + init_cmd.add_argument("--intent-correction") + init_cmd.set_defaults(func=_handler(command_handlers, "command_init")) + + quickstart_cmd = subparsers.add_parser( + "quickstart", + help="Interactive fast path for creating a scaffold-first skill package.", + ) + quickstart_cmd.add_argument("--name") + quickstart_cmd.add_argument("--job") + quickstart_cmd.add_argument("--real-input", action="append", default=[]) + quickstart_cmd.add_argument("--primary-output") + quickstart_cmd.add_argument("--description") + quickstart_cmd.add_argument("--title") + quickstart_cmd.add_argument("--output-dir", default=".") + quickstart_cmd.add_argument("--mode", choices=["scaffold", "production", "library", "governed"]) + quickstart_cmd.add_argument("--archetype", choices=["scaffold", "production", "library", "governed"]) + quickstart_cmd.add_argument("--external-reference", action="append", default=[]) + quickstart_cmd.add_argument("--user-reference", action="append", default=[]) + quickstart_cmd.add_argument("--local-constraint", action="append", default=[]) + quickstart_cmd.add_argument("--constraint", action="append", default=[]) + quickstart_cmd.add_argument("--github-query") + quickstart_cmd.add_argument("--github-top-n", type=int, default=3) + quickstart_cmd.add_argument("--github-fixture-dir") + quickstart_cmd.add_argument("--no-update-check", action="store_true") + quickstart_cmd.set_defaults(func=_handler(command_handlers, "command_quickstart")) + + validate_cmd = subparsers.add_parser("validate", help="Run validate, lint, governance, and resource checks.") + validate_cmd.add_argument("skill_dir", nargs="?", default=".") + validate_cmd.add_argument("--require-manifest", action="store_true") + validate_cmd.set_defaults(func=_handler(command_handlers, "command_validate")) + + optimize_cmd = subparsers.add_parser("optimize-description", help="Optimize description candidates for a target.") + optimize_cmd.add_argument( + "--target", + choices=["root", "team-frontend-review", "governed-incident-command", "all"], + default="root", + ) + optimize_cmd.add_argument("--write", action="store_true", help="Write default report artifacts for the target.") + optimize_cmd.set_defaults(func=_handler(command_handlers, "command_optimize_description")) + + promote_cmd = subparsers.add_parser("promote-check", help="Apply promotion policy and build iteration bundles.") + promote_cmd.set_defaults(func=_handler(command_handlers, "command_promote_check")) + + python_compat_cmd = subparsers.add_parser( + "python-compat", + help="Check Python source compatibility for the supported CI/runtime interpreter.", + ) + python_compat_cmd.add_argument("skill_dir", nargs="?", default=".") + python_compat_cmd.add_argument("--path", action="append", default=[]) + python_compat_cmd.add_argument("--target-python", default="3.11") + python_compat_cmd.add_argument("--output-json") + python_compat_cmd.add_argument("--output-md") + python_compat_cmd.add_argument("--generated-at") + python_compat_cmd.set_defaults(func=_handler(command_handlers, "command_python_compat")) + + architecture_audit_cmd = subparsers.add_parser( + "architecture-audit", + help="Render maintainability evidence for large Python files and CLI command surface.", + ) + architecture_audit_cmd.add_argument("skill_dir", nargs="?", default=".") + architecture_audit_cmd.add_argument("--output-json") + architecture_audit_cmd.add_argument("--output-md") + architecture_audit_cmd.add_argument("--warn-lines", type=int, default=900) + architecture_audit_cmd.add_argument("--block-lines", type=int, default=1500) + architecture_audit_cmd.add_argument("--generated-at") + architecture_audit_cmd.set_defaults(func=_handler(command_handlers, "command_architecture_audit")) + + review_cmd = subparsers.add_parser("review", help="Locate the current bundle and human review stub for a target.") + review_cmd.add_argument( + "--target", + choices=["root", "team-frontend-review", "governed-incident-command"], + default="root", + ) + review_cmd.set_defaults(func=_handler(command_handlers, "command_review")) + + snapshot_cmd = subparsers.add_parser("release-snapshot", help="Create a versioned snapshot from current promotion outputs.") + snapshot_cmd.add_argument( + "--target", + choices=["root", "team-frontend-review", "governed-incident-command"], + default="root", + ) + snapshot_cmd.add_argument("--label", default="manual") + snapshot_cmd.set_defaults(func=_handler(command_handlers, "command_release_snapshot")) + + flow_cmd = subparsers.add_parser( + "workspace-flow", + help="Run optimize, promotion, review refresh, and release snapshots as one authoring flow.", + ) + flow_cmd.add_argument( + "--target", + choices=["root", "team-frontend-review", "governed-incident-command", "all"], + default="root", + ) + flow_cmd.add_argument("--label", default="manual") + flow_cmd.set_defaults(func=_handler(command_handlers, "command_workspace_flow")) + + report_cmd = subparsers.add_parser("report", help="Render route, iteration, regression, and context reports.") + report_cmd.add_argument("--refresh-optimization", action="store_true") + report_cmd.set_defaults(func=_handler(command_handlers, "command_report")) + + skill_report_cmd = subparsers.add_parser("skill-report", help="Render the HTML skill report for a skill package.") + skill_report_cmd.add_argument("skill_dir", nargs="?", default=".") + skill_report_cmd.add_argument("--output-html") + skill_report_cmd.add_argument("--output-json") + skill_report_cmd.set_defaults(func=_handler(command_handlers, "command_skill_report")) + + skill_interpretation_cmd = subparsers.add_parser( + "skill-interpretation", + help="Render the first-class skill interpretation report for a skill package.", + ) + skill_interpretation_cmd.add_argument("skill_dir", nargs="?", default=".") + skill_interpretation_cmd.add_argument("--output-html") + skill_interpretation_cmd.add_argument("--output-json") + skill_interpretation_cmd.set_defaults(func=_handler(command_handlers, "command_skill_interpretation")) + + review_viewer_cmd = subparsers.add_parser("review-viewer", help="Render a compact HTML review page for a skill package.") + review_viewer_cmd.add_argument("skill_dir", nargs="?", default=".") + review_viewer_cmd.add_argument("--output-html") + review_viewer_cmd.add_argument("--output-json") + review_viewer_cmd.set_defaults(func=_handler(command_handlers, "command_review_viewer")) + + review_studio_cmd = subparsers.add_parser("review-studio", help="Render Review Studio 2.0 for a skill package.") + review_studio_cmd.add_argument("skill_dir", nargs="?", default=".") + review_studio_cmd.add_argument("--output-html") + review_studio_cmd.add_argument("--output-json") + review_studio_cmd.set_defaults(func=_handler(command_handlers, "command_review_studio")) + + skill_os2_audit_cmd = subparsers.add_parser( + "skill-os2-audit", + help="Render a requirement-by-requirement Skill OS 2.0 completion audit.", + ) + skill_os2_audit_cmd.add_argument("skill_dir", nargs="?", default=".") + skill_os2_audit_cmd.add_argument("--output-json") + skill_os2_audit_cmd.add_argument("--output-md") + skill_os2_audit_cmd.add_argument("--generated-at") + skill_os2_audit_cmd.set_defaults(func=_handler(command_handlers, "command_skill_os2_audit")) + + skill_os2_coverage_cmd = subparsers.add_parser( + "skill-os2-coverage", + help="Render Skill OS 2.0 blueprint-to-evidence coverage.", + ) + skill_os2_coverage_cmd.add_argument("skill_dir", nargs="?", default=".") + skill_os2_coverage_cmd.add_argument("--output-json") + skill_os2_coverage_cmd.add_argument("--output-md") + skill_os2_coverage_cmd.add_argument("--generated-at") + skill_os2_coverage_cmd.set_defaults(func=_handler(command_handlers, "command_skill_os2_coverage")) + + add_evidence_commands(subparsers, command_handlers) + + reference_scan_cmd = subparsers.add_parser( + "reference-scan", + help="Render a controlled benchmark scan report for a skill package.", + ) + reference_scan_cmd.add_argument("skill_dir", nargs="?", default=".") + reference_scan_cmd.add_argument("--reference", action="append", default=[]) + reference_scan_cmd.add_argument("--external-reference", action="append", default=[]) + reference_scan_cmd.add_argument("--user-reference", action="append", default=[]) + reference_scan_cmd.add_argument("--local-constraint", action="append", default=[]) + reference_scan_cmd.add_argument("--output-md") + reference_scan_cmd.add_argument("--output-json") + reference_scan_cmd.set_defaults(func=_handler(command_handlers, "command_reference_scan")) + + github_scan_cmd = subparsers.add_parser( + "github-benchmark-scan", + help="Search top public GitHub repositories and extract borrow or avoid patterns for a skill.", + ) + github_scan_cmd.add_argument("skill_dir", nargs="?", default=".") + github_scan_cmd.add_argument("--query", required=True) + github_scan_cmd.add_argument("--top-n", type=int, default=3) + github_scan_cmd.add_argument("--fixture-dir") + github_scan_cmd.add_argument("--output-md") + github_scan_cmd.add_argument("--output-json") + github_scan_cmd.set_defaults(func=_handler(command_handlers, "command_github_benchmark_scan")) + + intent_confidence_cmd = subparsers.add_parser( + "intent-confidence", + help="Render a confidence report for how well the real job, inputs, outputs, and exclusions are understood.", + ) + intent_confidence_cmd.add_argument("skill_dir", nargs="?", default=".") + intent_confidence_cmd.add_argument("--context-json") + intent_confidence_cmd.add_argument("--output-md") + intent_confidence_cmd.add_argument("--output-json") + intent_confidence_cmd.set_defaults(func=_handler(command_handlers, "command_intent_confidence")) + + intent_dialogue_cmd = subparsers.add_parser( + "intent-dialogue", + help="Render a front-loaded intent dialogue guide for a skill package.", + ) + intent_dialogue_cmd.add_argument("skill_dir", nargs="?", default=".") + intent_dialogue_cmd.add_argument("--output-md") + intent_dialogue_cmd.add_argument("--output-json") + intent_dialogue_cmd.set_defaults(func=_handler(command_handlers, "command_intent_dialogue")) + + reference_synthesis_cmd = subparsers.add_parser( + "reference-synthesis", + help="Render a multi-source reference synthesis report for a skill package.", + ) + reference_synthesis_cmd.add_argument("skill_dir", nargs="?", default=".") + reference_synthesis_cmd.add_argument("--output-md") + reference_synthesis_cmd.add_argument("--output-json") + reference_synthesis_cmd.set_defaults(func=_handler(command_handlers, "command_reference_synthesis")) + + output_risk_cmd = subparsers.add_parser( + "output-risk-profile", + help="Render predicted output failure modes and self-repair checks for a skill package.", + ) + output_risk_cmd.add_argument("skill_dir", nargs="?", default=".") + output_risk_cmd.add_argument("--output-md") + output_risk_cmd.add_argument("--output-json") + output_risk_cmd.set_defaults(func=_handler(command_handlers, "command_output_risk_profile")) + + artifact_design_cmd = subparsers.add_parser( + "artifact-design-profile", + help="Render artifact design direction and visual quality gates for a skill package.", + ) + artifact_design_cmd.add_argument("skill_dir", nargs="?", default=".") + artifact_design_cmd.add_argument("--output-md") + artifact_design_cmd.add_argument("--output-json") + artifact_design_cmd.set_defaults(func=_handler(command_handlers, "command_artifact_design_profile")) + + prompt_quality_cmd = subparsers.add_parser( + "prompt-quality-profile", + help="Render prompt-facing need model, RTF mapping, complexity, and quality checks for a skill package.", + ) + prompt_quality_cmd.add_argument("skill_dir", nargs="?", default=".") + prompt_quality_cmd.add_argument("--output-md") + prompt_quality_cmd.add_argument("--output-json") + prompt_quality_cmd.set_defaults(func=_handler(command_handlers, "command_prompt_quality_profile")) + + system_model_cmd = subparsers.add_parser( + "system-model", + help="Render a systems-thinking model for boundaries, feedback loops, drift, and leverage points.", + ) + system_model_cmd.add_argument("skill_dir", nargs="?", default=".") + system_model_cmd.add_argument("--output-md") + system_model_cmd.add_argument("--output-json") + system_model_cmd.set_defaults(func=_handler(command_handlers, "command_system_model")) + + iteration_directions_cmd = subparsers.add_parser( + "iteration-directions", + help="Render the top three next iteration directions for a skill package.", + ) + iteration_directions_cmd.add_argument("skill_dir", nargs="?", default=".") + iteration_directions_cmd.add_argument("--output-md") + iteration_directions_cmd.add_argument("--output-json") + iteration_directions_cmd.set_defaults(func=_handler(command_handlers, "command_iteration_directions")) + + feedback_cmd = subparsers.add_parser( + "feedback", + help="Capture lightweight reviewer feedback without running the full promotion flow.", + ) + feedback_cmd.add_argument("skill_dir", nargs="?", default=".") + feedback_cmd.add_argument("--note") + feedback_cmd.add_argument("--rating", type=int, default=3) + feedback_cmd.add_argument("--category", default="general") + feedback_cmd.add_argument("--recommended-action", default="review") + feedback_cmd.set_defaults(func=_handler(command_handlers, "command_feedback")) + + add_operating_loop_commands(subparsers, command_handlers) + + baseline_compare_cmd = subparsers.add_parser( + "baseline-compare", + help="Render a lightweight with-skill vs baseline comparison across tracked targets.", + ) + baseline_compare_cmd.set_defaults(func=_handler(command_handlers, "command_baseline_compare")) + + skill_ir_cmd = subparsers.add_parser("skill-ir", help="Export platform-neutral Skill IR for a skill package.") + skill_ir_cmd.add_argument("skill_dir", nargs="?", default=".") + skill_ir_cmd.add_argument("--output-json") + skill_ir_cmd.add_argument("--validate-only", action="store_true") + skill_ir_cmd.set_defaults(func=_handler(command_handlers, "command_skill_ir")) + + compile_skill_cmd = subparsers.add_parser( + "compile-skill", + help="Compile Skill IR into target-specific semantic contracts.", + ) + compile_skill_cmd.add_argument("skill_dir", nargs="?", default=".") + compile_skill_cmd.add_argument("--target", action="append") + compile_skill_cmd.add_argument("--output-json") + compile_skill_cmd.add_argument("--output-md") + compile_skill_cmd.add_argument("--generated-at") + compile_skill_cmd.set_defaults(func=_handler(command_handlers, "command_compile_skill")) + + output_eval_cmd = subparsers.add_parser("output-eval", help="Run Output Eval Lab assertion grading.") + output_eval_cmd.add_argument("--cases") + output_eval_cmd.add_argument("--output-json") + output_eval_cmd.add_argument("--output-md") + output_eval_cmd.add_argument("--blind-pack-json") + output_eval_cmd.add_argument("--blind-pack-md") + output_eval_cmd.add_argument("--blind-answer-key-json") + output_eval_cmd.set_defaults(func=_handler(command_handlers, "command_output_eval")) + + output_review_kit_cmd = subparsers.add_parser( + "output-review-kit", + help="Prepare a reviewer-facing blind A/B output review kit without exposing the answer key.", + ) + output_review_kit_cmd.add_argument("--blind-pack-json") + output_review_kit_cmd.add_argument("--blind-pack-md") + output_review_kit_cmd.add_argument("--decisions") + output_review_kit_cmd.add_argument("--output-json") + output_review_kit_cmd.add_argument("--output-md") + output_review_kit_cmd.add_argument("--output-html") + output_review_kit_cmd.add_argument("--write-template", action="store_true") + output_review_kit_cmd.set_defaults(func=_handler(command_handlers, "command_output_review_kit")) + + output_execution_cmd = subparsers.add_parser( + "output-exec", + help="Record output-eval execution evidence, timing, and token usage.", + ) + output_execution_cmd.add_argument("--cases") + output_execution_cmd.add_argument("--output-json") + output_execution_cmd.add_argument("--output-md") + output_execution_cmd.add_argument("--runner-command") + output_execution_cmd.add_argument( + "--provider-runner", + choices=["openai", "deepseek"], + help="Use the bundled provider-backed runner instead of a custom runner command.", + ) + output_execution_cmd.add_argument("--provider-model", help="Model for --provider-runner; otherwise use YAO_OUTPUT_EVAL_MODEL.") + output_execution_cmd.add_argument("--provider-base-url", help="Override provider endpoint for compatible APIs.") + output_execution_cmd.add_argument( + "--provider-api-format", + choices=["responses", "chat-completions"], + help="Provider API shape used by the bundled provider runner. Defaults are selected from --provider-runner.", + ) + output_execution_cmd.add_argument( + "--provider-thinking", + choices=["enabled", "disabled"], + help="Optional thinking mode override for chat-completions providers that support it.", + ) + output_execution_cmd.add_argument( + "--provider-temperature", + type=float, + help="Sampling temperature for the bundled provider runner. Defaults to 0 for reproducibility.", + ) + output_execution_cmd.add_argument("--api-key-env") + output_execution_cmd.add_argument("--allow-insecure-localhost", action="store_true") + output_execution_cmd.add_argument("--allow-custom-base-url", action="store_true") + output_execution_cmd.add_argument("--timeout-seconds", type=float) + output_execution_cmd.set_defaults(func=_handler(command_handlers, "command_output_execution")) + + output_review_cmd = subparsers.add_parser( + "output-review", + help="Adjudicate blind A/B output review decisions against the answer key.", + ) + output_review_cmd.add_argument("--blind-pack") + output_review_cmd.add_argument("--answer-key") + output_review_cmd.add_argument("--decisions") + output_review_cmd.add_argument("--output-json") + output_review_cmd.add_argument("--output-md") + output_review_cmd.add_argument("--write-template", action="store_true") + output_review_cmd.set_defaults(func=_handler(command_handlers, "command_output_review")) + + output_review_import_cmd = subparsers.add_parser( + "output-review-import", + help="Import human blind A/B reviewer decisions into the canonical decision file.", + ) + output_review_import_cmd.add_argument("--input", required=True) + output_review_import_cmd.add_argument("--format", choices=["auto", "json", "jsonl", "csv"], default="auto") + output_review_import_cmd.add_argument("--blind-pack") + output_review_import_cmd.add_argument("--output-json") + output_review_import_cmd.add_argument("--reviewer") + output_review_import_cmd.add_argument("--reviewed-at") + output_review_import_cmd.add_argument("--blind-review-attested", action="store_true") + output_review_import_cmd.add_argument("--run-adjudication", action="store_true") + output_review_import_cmd.add_argument("--answer-key") + output_review_import_cmd.add_argument("--adjudication-json") + output_review_import_cmd.add_argument("--adjudication-md") + output_review_import_cmd.set_defaults(func=_handler(command_handlers, "command_output_review_import")) + + conformance_cmd = subparsers.add_parser("conformance", help="Run runtime conformance checks for Skill OS targets.") + conformance_cmd.add_argument("skill_dir", nargs="?", default=".") + conformance_cmd.add_argument("--target", action="append", choices=["openai", "claude", "agent-skills", "vscode", "generic"]) + conformance_cmd.add_argument("--output-json") + conformance_cmd.add_argument("--output-md") + conformance_cmd.set_defaults(func=_handler(command_handlers, "command_conformance")) + + runtime_permissions_cmd = subparsers.add_parser( + "runtime-permissions", + help="Probe generated target adapters for runtime permission enforcement metadata.", + ) + runtime_permissions_cmd.add_argument("skill_dir", nargs="?", default=".") + runtime_permissions_cmd.add_argument("--package-dir", default="dist") + runtime_permissions_cmd.add_argument("--target", action="append", choices=["openai", "claude", "generic", "vscode"]) + runtime_permissions_cmd.add_argument("--install-simulation-json") + runtime_permissions_cmd.add_argument("--output-json") + runtime_permissions_cmd.add_argument("--output-md") + runtime_permissions_cmd.set_defaults(func=_handler(command_handlers, "command_runtime_permissions")) + + trust_cmd = subparsers.add_parser("trust", help="Run trust and security checks for a skill package.") + trust_cmd.add_argument("skill_dir", nargs="?", default=".") + trust_cmd.add_argument("--output-json") + trust_cmd.add_argument("--output-md") + trust_cmd.set_defaults(func=_handler(command_handlers, "command_trust")) + + skill_atlas_cmd = subparsers.add_parser("skill-atlas", help="Build a portfolio-level Skill Atlas for a workspace.") + skill_atlas_cmd.add_argument("--workspace-root", default=".") + skill_atlas_cmd.add_argument("--output-dir") + skill_atlas_cmd.add_argument("--report-html") + skill_atlas_cmd.add_argument("--report-json") + skill_atlas_cmd.add_argument("--overlap-threshold", type=float) + skill_atlas_cmd.add_argument("--today") + skill_atlas_cmd.set_defaults(func=_handler(command_handlers, "command_skill_atlas")) + + registry_audit_cmd = subparsers.add_parser("registry-audit", help="Build and audit Skill OS registry package metadata.") + registry_audit_cmd.add_argument("skill_dir", nargs="?", default=".") + registry_audit_cmd.add_argument("--registry-dir") + registry_audit_cmd.add_argument("--output-json") + registry_audit_cmd.add_argument("--output-md") + registry_audit_cmd.add_argument("--generated-at") + registry_audit_cmd.set_defaults(func=_handler(command_handlers, "command_registry_audit")) + + package_cmd = subparsers.add_parser("package", help="Export compatibility artifacts for selected targets.") + package_cmd.add_argument("skill_dir", nargs="?", default=".") + package_cmd.add_argument("--platform", action="append") + package_cmd.add_argument("--output-dir", default="dist") + package_cmd.add_argument("--expectations") + package_cmd.add_argument("--zip", action="store_true") + package_cmd.set_defaults(func=_handler(command_handlers, "command_package")) + + package_verify_cmd = subparsers.add_parser("package-verify", help="Verify generated package artifacts, archive safety, and registry parity.") + package_verify_cmd.add_argument("skill_dir", nargs="?", default=".") + package_verify_cmd.add_argument("--package-dir", default="dist") + package_verify_cmd.add_argument("--expectations", default="evals/packaging_expectations.json") + package_verify_cmd.add_argument("--registry-json", default="reports/registry_audit.json") + package_verify_cmd.add_argument("--output-json") + package_verify_cmd.add_argument("--output-md") + package_verify_cmd.add_argument("--require-zip", action="store_true") + package_verify_cmd.add_argument("--generated-at") + package_verify_cmd.set_defaults(func=_handler(command_handlers, "command_package_verify")) + + install_simulate_cmd = subparsers.add_parser("install-simulate", help="Simulate installing a generated package into a temporary local skill root.") + install_simulate_cmd.add_argument("skill_dir", nargs="?", default=".") + install_simulate_cmd.add_argument("--package-dir", default="dist") + install_simulate_cmd.add_argument("--install-root") + install_simulate_cmd.add_argument("--output-json") + install_simulate_cmd.add_argument("--output-md") + install_simulate_cmd.add_argument("--generated-at") + install_simulate_cmd.set_defaults(func=_handler(command_handlers, "command_install_simulate")) + + upgrade_check_cmd = subparsers.add_parser("upgrade-check", help="Compare current and previous registry package metadata for upgrade readiness.") + upgrade_check_cmd.add_argument("skill_dir", nargs="?", default=".") + upgrade_check_cmd.add_argument("--previous-package-json", required=True) + upgrade_check_cmd.add_argument("--current-package-json", default="reports/registry_audit.json") + upgrade_check_cmd.add_argument("--output-json") + upgrade_check_cmd.add_argument("--output-md") + upgrade_check_cmd.add_argument("--generated-at") + upgrade_check_cmd.set_defaults(func=_handler(command_handlers, "command_upgrade_check")) + + test_cmd = subparsers.add_parser("test", help="Run a Makefile test target.") + test_cmd.add_argument("--target", default="test") + test_cmd.set_defaults(func=_handler(command_handlers, "command_test")) + + update_cmd = subparsers.add_parser("check-update", help="Check whether a newer yao-meta-skill version is available.") + update_cmd.add_argument("--force", action="store_true") + update_cmd.add_argument("--no-cache", action="store_true") + update_cmd.add_argument("--version-url") + update_cmd.add_argument("--manifest-url") + update_cmd.add_argument("--timeout", type=float, default=3.0) + update_cmd.add_argument("--allow-custom-update-url", action="store_true") + update_cmd.set_defaults(func=_handler(command_handlers, "command_check_update")) + + return parser diff --git a/scripts/yao_cli_parser_evidence.py b/scripts/yao_cli_parser_evidence.py new file mode 100644 index 00000000..e4d6c121 --- /dev/null +++ b/scripts/yao_cli_parser_evidence.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Evidence command declarations for the Yao CLI parser.""" + +import argparse +from collections.abc import Callable + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao_cli_parser.py to keep evidence command declarations out of the main parser module." + + +def _handler(command_handlers: dict[str, Callable[[argparse.Namespace], int]], name: str) -> Callable[[argparse.Namespace], int]: + if name not in command_handlers: + raise KeyError(f"Missing CLI command handler: {name}") + return command_handlers[name] + + +def add_evidence_commands( + subparsers: argparse._SubParsersAction, + command_handlers: dict[str, Callable[[argparse.Namespace], int]], +) -> None: + world_class_evidence_cmd = subparsers.add_parser( + "world-class-evidence", + help="Render the evidence collection plan for remaining world-class readiness gaps.", + ) + world_class_evidence_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_evidence_cmd.add_argument("--output-json") + world_class_evidence_cmd.add_argument("--output-md") + world_class_evidence_cmd.add_argument("--generated-at") + world_class_evidence_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_evidence")) + + world_class_ledger_cmd = subparsers.add_parser( + "world-class-ledger", + help="Render the machine-checkable ledger for world-class evidence gaps.", + ) + world_class_ledger_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_ledger_cmd.add_argument("--output-json") + world_class_ledger_cmd.add_argument("--output-md") + world_class_ledger_cmd.add_argument("--submissions-dir") + world_class_ledger_cmd.add_argument("--generated-at") + world_class_ledger_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_ledger")) + + world_class_intake_cmd = subparsers.add_parser( + "world-class-intake", + help="Validate world-class human and external evidence intake packets.", + ) + world_class_intake_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_intake_cmd.add_argument("--submissions-dir") + world_class_intake_cmd.add_argument("--output-json") + world_class_intake_cmd.add_argument("--output-md") + world_class_intake_cmd.add_argument("--generated-at") + world_class_intake_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_intake")) + + world_class_preflight_cmd = subparsers.add_parser( + "world-class-preflight", + help="Render operator preflight checks for collecting pending world-class evidence.", + ) + world_class_preflight_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_preflight_cmd.add_argument("--submissions-dir") + world_class_preflight_cmd.add_argument("--output-json") + world_class_preflight_cmd.add_argument("--output-md") + world_class_preflight_cmd.add_argument("--output-html") + world_class_preflight_cmd.add_argument("--generated-at") + world_class_preflight_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_preflight")) + + world_class_submission_kit_cmd = subparsers.add_parser( + "world-class-submission-kit", + help="Prepare editable world-class evidence submission drafts.", + ) + world_class_submission_kit_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_submission_kit_cmd.add_argument("--output-dir") + world_class_submission_kit_cmd.add_argument("--evidence-key", action="append", default=[]) + world_class_submission_kit_cmd.add_argument("--overwrite", action="store_true") + world_class_submission_kit_cmd.add_argument( + "--prefill-artifacts", + action="store_true", + help="Insert SHA-256 digests for currently available aggregate artifacts while keeping drafts template-only.", + ) + world_class_submission_kit_cmd.add_argument("--generated-at") + world_class_submission_kit_cmd.add_argument("--output-html") + world_class_submission_kit_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_submission_kit")) + + world_class_submission_review_cmd = subparsers.add_parser( + "world-class-submission-review", + help="Render a read-only review queue for world-class evidence submissions.", + ) + world_class_submission_review_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_submission_review_cmd.add_argument("--submissions-dir") + world_class_submission_review_cmd.add_argument("--output-json") + world_class_submission_review_cmd.add_argument("--output-md") + world_class_submission_review_cmd.add_argument("--generated-at") + world_class_submission_review_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_submission_review")) + + world_class_runbook_cmd = subparsers.add_parser( + "world-class-runbook", + help="Render an operator runbook for pending world-class evidence.", + ) + world_class_runbook_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_runbook_cmd.add_argument("--submissions-dir") + world_class_runbook_cmd.add_argument("--output-json") + world_class_runbook_cmd.add_argument("--output-md") + world_class_runbook_cmd.add_argument("--output-html") + world_class_runbook_cmd.add_argument("--generated-at") + world_class_runbook_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_runbook")) + + world_class_claim_guard_cmd = subparsers.add_parser( + "world-class-claim-guard", + help="Scan public claim surfaces for premature world-class completion claims.", + ) + world_class_claim_guard_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_claim_guard_cmd.add_argument("--claim-surface", action="append", default=[]) + world_class_claim_guard_cmd.add_argument("--output-json") + world_class_claim_guard_cmd.add_argument("--output-md") + world_class_claim_guard_cmd.add_argument("--generated-at") + world_class_claim_guard_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_claim_guard")) + + benchmark_reproducibility_cmd = subparsers.add_parser( + "benchmark-reproducibility", + help="Render benchmark methodology, artifact, failure-disclosure, and reproduction-command evidence.", + ) + benchmark_reproducibility_cmd.add_argument("skill_dir", nargs="?", default=".") + benchmark_reproducibility_cmd.add_argument("--output-json") + benchmark_reproducibility_cmd.add_argument("--output-md") + benchmark_reproducibility_cmd.add_argument("--generated-at") + benchmark_reproducibility_cmd.set_defaults(func=_handler(command_handlers, "command_benchmark_reproducibility")) + + evidence_consistency_cmd = subparsers.add_parser( + "evidence-consistency", + help="Render cross-report evidence consistency checks.", + ) + evidence_consistency_cmd.add_argument("skill_dir", nargs="?", default=".") + evidence_consistency_cmd.add_argument("--output-json") + evidence_consistency_cmd.add_argument("--output-md") + evidence_consistency_cmd.add_argument("--generated-at") + evidence_consistency_cmd.set_defaults(func=_handler(command_handlers, "command_evidence_consistency")) diff --git a/scripts/yao_cli_parser_operations.py b/scripts/yao_cli_parser_operations.py new file mode 100644 index 00000000..263e2535 --- /dev/null +++ b/scripts/yao_cli_parser_operations.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Operating-loop command declarations for the Yao CLI parser.""" + +import argparse +from collections.abc import Callable + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao_cli_parser.py to keep SkillOps, telemetry, and review-loop command declarations out of the main parser module." + + +TELEMETRY_EVENT_CHOICES = ["review_event", "script_run", "skill_activation", "skill_output"] +TELEMETRY_ACTIVATION_CHOICES = ["explicit", "implicit", "manual", "unknown"] +TELEMETRY_OUTCOME_CHOICES = ["accepted", "edited", "failed", "missed", "rejected", "reviewed", "unknown"] +TELEMETRY_FAILURE_CHOICES = [ + "bad_output", + "missing_resource", + "none", + "review_overdue", + "script_error", + "under_trigger", + "wrong_trigger", +] +TELEMETRY_SOURCE_CHOICES = ["external", "manual", "unknown", "yao_cli"] + +REVIEW_WAIVER_GATE_CHOICES = [ + "architecture-maintainability", + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "python-compat", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "permission-gates", + "permission-runtime", +] + +REVIEW_ANNOTATION_GATE_CHOICES = [ + *REVIEW_WAIVER_GATE_CHOICES[:8], + "review-waivers", + *REVIEW_WAIVER_GATE_CHOICES[8:12], + "world-class-evidence", + *REVIEW_WAIVER_GATE_CHOICES[12:], +] + + +def _handler(command_handlers: dict[str, Callable[[argparse.Namespace], int]], name: str) -> Callable[[argparse.Namespace], int]: + if name not in command_handlers: + raise KeyError(f"Missing CLI command handler: {name}") + return command_handlers[name] + + +def _add_telemetry_metadata_arguments( + command: argparse.ArgumentParser, + *, + source_default: str, + command_default: str, + activation_default: str, + event_flag: str | None = None, + event_default: str | None = None, +) -> None: + if event_flag is not None: + kwargs = {"choices": TELEMETRY_EVENT_CHOICES} + if event_default is not None: + kwargs["default"] = event_default + command.add_argument(event_flag, **kwargs) + command.add_argument("--activation-type", choices=TELEMETRY_ACTIVATION_CHOICES, default=activation_default) + command.add_argument("--outcome", choices=TELEMETRY_OUTCOME_CHOICES, default="unknown") + command.add_argument("--failure-type", choices=TELEMETRY_FAILURE_CHOICES, default="none") + command.add_argument("--source", choices=TELEMETRY_SOURCE_CHOICES, default=source_default) + command.add_argument("--command", dest="telemetry_command", default=command_default) + command.add_argument("--timestamp") + command.add_argument("--skill-name") + command.add_argument("--version") + + +def add_operating_loop_commands( + subparsers: argparse._SubParsersAction, + command_handlers: dict[str, Callable[[argparse.Namespace], int]], +) -> None: + adapt_scan_cmd = subparsers.add_parser( + "adapt-scan", + help="Scan one explicit local source file for redacted repeated user preference signals.", + ) + adapt_scan_cmd.add_argument("skill_dir", nargs="?", default=".") + adapt_scan_cmd.add_argument("--source", required=True) + adapt_scan_cmd.add_argument("--output-json") + adapt_scan_cmd.add_argument("--output-md") + adapt_scan_cmd.add_argument("--min-support", type=int, default=2) + adapt_scan_cmd.add_argument("--generated-at") + adapt_scan_cmd.add_argument("--allow-history-source", action="store_true") + adapt_scan_cmd.set_defaults(func=_handler(command_handlers, "command_adapt_scan")) + + adapt_propose_cmd = subparsers.add_parser( + "adapt-propose", + help="Create proposal-only adaptation plans from redacted repeated preference patterns.", + ) + adapt_propose_cmd.add_argument("skill_dir", nargs="?", default=".") + adapt_propose_cmd.add_argument("--patterns-json") + adapt_propose_cmd.add_argument("--output-json") + adapt_propose_cmd.add_argument("--output-md") + adapt_propose_cmd.add_argument("--generated-at") + adapt_propose_cmd.set_defaults(func=_handler(command_handlers, "command_adapt_propose")) + + adapt_apply_cmd = subparsers.add_parser( + "adapt-apply", + help="Dry-run or apply an approved adaptation patch with allowlist, regression, and rollback evidence.", + ) + adapt_apply_cmd.add_argument("skill_dir", nargs="?", default=".") + adapt_apply_cmd.add_argument("--proposal-id") + adapt_apply_cmd.add_argument("--patch-file") + adapt_apply_cmd.add_argument("--proposals-json") + adapt_apply_cmd.add_argument("--approval-ledger") + adapt_apply_cmd.add_argument("--output-json") + adapt_apply_cmd.add_argument("--output-md") + adapt_apply_cmd.add_argument("--generated-at") + adapt_apply_cmd.add_argument("--today") + adapt_apply_cmd.add_argument("--write-template", action="store_true") + adapt_apply_cmd.add_argument("--prepare-approval", action="store_true") + adapt_apply_cmd.add_argument("--apply", action="store_true") + adapt_apply_cmd.add_argument("--run-verification", action="store_true") + adapt_apply_cmd.add_argument( + "--no-rollback-on-failure", + dest="rollback_on_failure", + action="store_false", + help="Leave an applied patch in place if verification fails. Default is to reverse the patch.", + ) + adapt_apply_cmd.set_defaults(rollback_on_failure=True) + adapt_apply_cmd.set_defaults(func=_handler(command_handlers, "command_adapt_apply")) + + daily_skillops_cmd = subparsers.add_parser( + "daily-skillops", + help="Render a Daily SkillOps report from explicit-source adaptive evidence without scanning private logs or applying patches.", + ) + daily_skillops_cmd.add_argument("skill_dir", nargs="?", default=".") + daily_skillops_cmd.add_argument("--source") + daily_skillops_cmd.add_argument("--patterns-json", default="reports/user_patterns.json") + daily_skillops_cmd.add_argument("--proposals-json", default="reports/adaptation_proposals.json") + daily_skillops_cmd.add_argument("--output-json") + daily_skillops_cmd.add_argument("--output-md") + daily_skillops_cmd.add_argument("--min-support", type=int, default=2) + daily_skillops_cmd.add_argument("--generated-at") + daily_skillops_cmd.add_argument("--allow-history-source", action="store_true") + daily_skillops_cmd.add_argument("--no-refresh-source-reports", action="store_true") + daily_skillops_cmd.set_defaults(func=_handler(command_handlers, "command_daily_skillops")) + + weekly_curator_cmd = subparsers.add_parser( + "weekly-curator", + help="Render a weekly SkillOps curator report from generated daily reports and portfolio evidence.", + ) + weekly_curator_cmd.add_argument("skill_dir", nargs="?", default=".") + weekly_curator_cmd.add_argument("--daily-json", action="append", default=[]) + weekly_curator_cmd.add_argument("--output-json") + weekly_curator_cmd.add_argument("--output-md") + weekly_curator_cmd.add_argument("--generated-at") + weekly_curator_cmd.set_defaults(func=_handler(command_handlers, "command_weekly_curator")) + + adoption_drift_cmd = subparsers.add_parser( + "adoption-drift", + help="Render local-first metadata-only adoption and drift telemetry for a skill package.", + ) + adoption_drift_cmd.add_argument("skill_dir", nargs="?", default=".") + adoption_drift_cmd.add_argument("--events-jsonl") + adoption_drift_cmd.add_argument("--output-json") + adoption_drift_cmd.add_argument("--output-md") + adoption_drift_cmd.add_argument("--generated-at") + _add_telemetry_metadata_arguments( + adoption_drift_cmd, + event_flag="--record-event", + source_default="manual", + command_default="unknown", + activation_default="unknown", + ) + adoption_drift_cmd.set_defaults(func=_handler(command_handlers, "command_adoption_drift")) + + telemetry_import_cmd = subparsers.add_parser( + "telemetry-import", + help="Import external metadata-only telemetry JSONL and refresh the adoption drift report.", + ) + telemetry_import_cmd.add_argument("skill_dir", nargs="?", default=".") + telemetry_import_cmd.add_argument("--input-jsonl", required=True) + telemetry_import_cmd.add_argument("--events-jsonl") + telemetry_import_cmd.add_argument("--output-json") + telemetry_import_cmd.add_argument("--output-md") + telemetry_import_cmd.add_argument("--generated-at") + telemetry_import_cmd.add_argument("--source", choices=TELEMETRY_SOURCE_CHOICES, default="external") + telemetry_import_cmd.add_argument("--command", dest="telemetry_command", default="external-client") + telemetry_import_cmd.add_argument("--dry-run", action="store_true") + telemetry_import_cmd.set_defaults(func=_handler(command_handlers, "command_telemetry_import")) + + telemetry_emit_cmd = subparsers.add_parser( + "telemetry-emit", + help="Emit one metadata-only telemetry event for later import.", + ) + telemetry_emit_cmd.add_argument("skill_dir", nargs="?", default=".") + telemetry_emit_cmd.add_argument("--output-jsonl") + _add_telemetry_metadata_arguments( + telemetry_emit_cmd, + event_flag="--event", + event_default="script_run", + source_default="external", + command_default="external-client", + activation_default="manual", + ) + telemetry_emit_cmd.add_argument("--dry-run", action="store_true") + telemetry_emit_cmd.set_defaults(func=_handler(command_handlers, "command_telemetry_emit")) + + telemetry_hooks_cmd = subparsers.add_parser( + "telemetry-hooks", + help="Render metadata-only telemetry client hook recipes.", + ) + telemetry_hooks_cmd.add_argument("skill_dir", nargs="?", default=".") + telemetry_hooks_cmd.add_argument("--output-json") + telemetry_hooks_cmd.add_argument("--output-md") + telemetry_hooks_cmd.add_argument("--output-jsonl") + telemetry_hooks_cmd.set_defaults(func=_handler(command_handlers, "command_telemetry_hooks")) + + review_waivers_cmd = subparsers.add_parser( + "review-waivers", + help="Render or update human reviewer waiver evidence for Review Studio.", + ) + review_waivers_cmd.add_argument("skill_dir", nargs="?", default=".") + review_waivers_cmd.add_argument("--waivers-json") + review_waivers_cmd.add_argument("--output-json") + review_waivers_cmd.add_argument("--output-md") + review_waivers_cmd.add_argument("--generated-at") + review_waivers_cmd.add_argument("--add-waiver", action="store_true") + review_waivers_cmd.add_argument("--gate-key", choices=REVIEW_WAIVER_GATE_CHOICES) + review_waivers_cmd.add_argument( + "--decision", + choices=["accepted-risk", "false-positive", "temporary-exception"], + default="accepted-risk", + ) + review_waivers_cmd.add_argument("--reviewer") + review_waivers_cmd.add_argument("--reason") + review_waivers_cmd.add_argument("--expires-at") + review_waivers_cmd.add_argument("--created-at") + review_waivers_cmd.add_argument("--evidence") + review_waivers_cmd.add_argument("--scope", default="current-release") + review_waivers_cmd.set_defaults(func=_handler(command_handlers, "command_review_waivers")) + + review_annotations_cmd = subparsers.add_parser( + "review-annotations", + help="Render or update inline reviewer annotations for Review Studio gates and source paths.", + ) + review_annotations_cmd.add_argument("skill_dir", nargs="?", default=".") + review_annotations_cmd.add_argument("--annotations-json") + review_annotations_cmd.add_argument("--output-json") + review_annotations_cmd.add_argument("--output-md") + review_annotations_cmd.add_argument("--write-template", action="store_true") + review_annotations_cmd.add_argument("--add-annotation", action="store_true") + review_annotations_cmd.add_argument("--annotation-id") + review_annotations_cmd.add_argument("--gate-key", choices=REVIEW_ANNOTATION_GATE_CHOICES) + review_annotations_cmd.add_argument("--target-path") + review_annotations_cmd.add_argument("--line", type=int) + review_annotations_cmd.add_argument("--severity", choices=["blocker", "info", "note", "warning"], default="note") + review_annotations_cmd.add_argument("--status", choices=["deferred", "open", "resolved"], default="open") + review_annotations_cmd.add_argument("--reviewer") + review_annotations_cmd.add_argument("--created-at") + review_annotations_cmd.add_argument("--body") + review_annotations_cmd.add_argument("--suggested-action") + review_annotations_cmd.add_argument("--evidence") + review_annotations_cmd.set_defaults(func=_handler(command_handlers, "command_review_annotations")) diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py new file mode 100644 index 00000000..8920a391 --- /dev/null +++ b/scripts/yao_cli_report_commands.py @@ -0,0 +1,339 @@ +"""Report and evidence command handlers for the Yao CLI.""" + +import argparse +import json +from pathlib import Path + +from yao_cli_config import baseline_compare_args, local_output_runner_command +from yao_cli_runtime import ROOT, allow_report_status, run_adoption_drift_if_source_exists, run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep report and evidence command handlers out of the CLI orchestrator." + + +def emit_result(result: dict) -> int: + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def load_json_file(path: Path) -> dict: + if not path.exists(): + return {} + try: + return json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + + +def output_execution_has_provider_evidence() -> bool: + summary = load_json_file(ROOT / "reports" / "output_execution_runs.json").get("summary", {}) + return ( + int(summary.get("model_executed_count", 0) or 0) > 0 + and int(summary.get("timing_observed_count", 0) or 0) > 0 + and int(summary.get("token_observed_count", 0) or 0) > 0 + ) + + +def run_output_execution_refresh() -> dict: + command = f"run_output_execution.py --runner-command {local_output_runner_command()}" + if output_execution_has_provider_evidence(): + return { + "command": f"{command} skipped: provider-backed evidence already present", + "returncode": 0, + "ok": True, + "stdout": "", + "stderr": "", + "payload": { + "ok": True, + "skipped": True, + "reason": "provider-backed output execution evidence already has observed model, timing, and token metadata", + }, + } + return run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]) + + +def resolved_skill_dir(args: argparse.Namespace) -> str: + return str(Path(args.skill_dir).resolve()) + + +def append_outputs(cmd: list[str], args: argparse.Namespace, *, markdown: bool = True, generated_at: bool = False) -> None: + if getattr(args, "output_html", None): + cmd.extend(["--output-html", args.output_html]) + if getattr(args, "output_json", None): + cmd.extend(["--output-json", args.output_json]) + if markdown and getattr(args, "output_md", None): + cmd.extend(["--output-md", args.output_md]) + if generated_at and getattr(args, "generated_at", None): + cmd.extend(["--generated-at", args.generated_at]) + + +def render_skill_report_command(args: argparse.Namespace, script_name: str, *, markdown: bool = True, generated_at: bool = False) -> int: + cmd = [resolved_skill_dir(args)] + append_outputs(cmd, args, markdown=markdown, generated_at=generated_at) + return emit_result(run_script(script_name, cmd)) + + +def command_report(args: argparse.Namespace) -> int: + steps = [] + if args.refresh_optimization: + steps.append(run_script("run_description_optimization_suite.py", [])) + steps.extend( + [ + run_script("build_confusion_matrix.py", []), + run_script("promotion_checker.py", []), + run_script("render_eval_dashboard.py", []), + run_script("render_intent_confidence.py", [str(ROOT)]), + run_script("render_description_drift_history.py", []), + run_script("render_iteration_ledger.py", []), + run_script("render_baseline_compare.py", baseline_compare_args()), + run_script("render_regression_history.py", []), + run_script("trust_check.py", [str(ROOT)]), + run_script("python_compat_check.py", [str(ROOT)]), + run_script("render_architecture_maintainability.py", [str(ROOT)]), + run_script("render_context_reports.py", []), + run_script("render_portability_report.py", []), + run_script("render_reference_synthesis.py", [str(ROOT)]), + run_script("render_artifact_design_profile.py", [str(ROOT)]), + run_script("render_prompt_quality_profile.py", [str(ROOT)]), + run_script("render_system_model.py", [str(ROOT)]), + run_script("compile_skill.py", [str(ROOT)]), + run_script("run_output_eval.py", []), + run_output_execution_refresh(), + run_script("prepare_output_review_kit.py", []), + run_script("adjudicate_output_review.py", []), + run_adoption_drift_if_source_exists(), + run_script("render_telemetry_hook_recipes.py", [str(ROOT)]), + run_script("render_review_waivers.py", [str(ROOT)]), + run_script("render_review_annotations.py", [str(ROOT)]), + run_script("render_world_class_evidence_plan.py", [str(ROOT)]), + run_script("render_world_class_evidence_ledger.py", [str(ROOT)]), + run_script("render_world_class_evidence_intake.py", [str(ROOT)]), + run_script("render_world_class_preflight.py", [str(ROOT)]), + allow_report_status(run_script("render_world_class_submission_review.py", [str(ROOT)])), + run_script("render_world_class_operator_runbook.py", [str(ROOT)]), + run_script("render_world_class_claim_guard.py", [str(ROOT)]), + run_script("render_daily_skillops_report.py", [str(ROOT)]), + run_script("render_weekly_curator_report.py", [str(ROOT)]), + run_script("render_skill_os2_audit.py", [str(ROOT)]), + run_script("render_skill_os2_coverage.py", [str(ROOT)]), + run_script("render_benchmark_reproducibility.py", [str(ROOT)]), + run_script("render_skill_overview.py", [str(ROOT)]), + run_script("render_skill_interpretation.py", [str(ROOT)]), + run_script("render_review_studio.py", [str(ROOT)]), + run_script("render_evidence_consistency.py", [str(ROOT)]), + run_script("render_review_viewer.py", [str(ROOT)]), + ] + ) + report = { + "ok": all(step["ok"] for step in steps), + "steps": [{"command": step["command"], "ok": step["ok"], "returncode": step["returncode"]} for step in steps], + "artifacts": { + "eval_results": "reports/eval_suite.json", + "route_scorecard": "reports/route_scorecard.json", + "promotion_decisions": "reports/promotion_decisions.json", + "intent_confidence": "reports/intent-confidence.json", + "iteration_ledger": "reports/iteration_ledger.md", + "baseline_compare": "reports/baseline-compare.json", + "regression_history": "reports/regression_history.md", + "context_budget": "reports/context_budget.json", + "portability_score": "reports/portability_score.json", + "security_trust": "reports/security_trust_report.json", + "python_compatibility": "reports/python_compatibility.json", + "architecture_maintainability": "reports/architecture_maintainability.json", + "reference_synthesis": "reports/reference-synthesis.json", + "artifact_design_profile": "reports/artifact-design-profile.json", + "prompt_quality_profile": "reports/prompt-quality-profile.json", + "system_model": "reports/system-model.json", + "compiled_targets": "reports/compiled_targets.json", + "output_execution": "reports/output_execution_runs.json", + "output_review_kit": "reports/output_review_kit.json", + "output_review_kit_html": "reports/output_review_kit.html", + "output_review_adjudication": "reports/output_review_adjudication.json", + "adoption_drift": "reports/adoption_drift_report.json", + "telemetry_hooks": "reports/telemetry_hook_recipes.json", + "review_waivers": "reports/review_waivers.json", + "review_annotations": "reports/review_annotations.json", + "world_class_evidence_plan": "reports/world_class_evidence_plan.json", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.json", + "world_class_evidence_intake": "reports/world_class_evidence_intake.json", + "world_class_evidence_preflight": "reports/world_class_evidence_preflight.json", + "world_class_submission_review": "reports/world_class_submission_review.json", + "world_class_operator_runbook": "reports/world_class_operator_runbook.json", + "world_class_claim_guard": "reports/world_class_claim_guard.json", + "daily_skillops": "reports/skillops/daily", + "weekly_curator": "reports/skillops/weekly", + "skill_os2_audit": "reports/skill_os2_audit.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", + "skill_overview": "reports/skill-overview.json", + "skill_interpretation": "reports/skill-interpretation.json", + "skill_interpretation_html": "reports/skill-interpretation.html", + "review_studio": "reports/review-studio.json", + "review_studio_html": "reports/review-studio.html", + "evidence_consistency": "reports/evidence_consistency.json", + "review_viewer": "reports/review-viewer.json", + "review_viewer_html": "reports/review-viewer.html", + }, + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if report["ok"] else 2 + + +def command_skill_report(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_skill_overview.py", markdown=False) + + +def command_skill_interpretation(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_skill_interpretation.py", markdown=False) + + +def command_review_viewer(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_review_viewer.py", markdown=False) + + +def command_review_studio(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_review_studio.py", markdown=False) + + +def command_skill_os2_audit(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_skill_os2_audit.py", generated_at=True) + + +def command_skill_os2_coverage(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_skill_os2_coverage.py", generated_at=True) + + +def command_world_class_evidence(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_world_class_evidence_plan.py", generated_at=True) + + +def command_world_class_ledger(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if getattr(args, "submissions_dir", None): + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_evidence_ledger.py", cmd)) + + +def command_world_class_intake(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_evidence_intake.py", cmd)) + + +def command_world_class_preflight(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_preflight.py", cmd)) + + +def command_world_class_submission_kit(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.output_dir: + cmd.extend(["--output-dir", args.output_dir]) + for key in args.evidence_key: + cmd.extend(["--evidence-key", key]) + if args.overwrite: + cmd.append("--overwrite") + if args.prefill_artifacts: + cmd.append("--prefill-artifacts") + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.output_html: + cmd.extend(["--output-html", args.output_html]) + return emit_result(run_script("prepare_world_class_submission_kit.py", cmd)) + + +def command_world_class_submission_review(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_submission_review.py", cmd)) + + +def command_world_class_runbook(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.submissions_dir: + cmd.extend(["--submissions-dir", args.submissions_dir]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_operator_runbook.py", cmd)) + + +def command_world_class_claim_guard(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + for surface in args.claim_surface: + cmd.extend(["--claim-surface", surface]) + append_outputs(cmd, args, generated_at=True) + return emit_result(run_script("render_world_class_claim_guard.py", cmd)) + + +def command_benchmark_reproducibility(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_benchmark_reproducibility.py", generated_at=True) + + +def command_evidence_consistency(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_evidence_consistency.py", generated_at=True) + + +def command_reference_scan(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + for reference in args.external_reference: + cmd.extend(["--external-reference", reference]) + for reference in args.user_reference: + cmd.extend(["--user-reference", reference]) + for constraint in args.local_constraint: + cmd.extend(["--local-constraint", constraint]) + for reference in args.reference: + cmd.extend(["--reference", reference]) + append_outputs(cmd, args) + return emit_result(run_script("render_reference_scan.py", cmd)) + + +def command_github_benchmark_scan(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args), "--query", args.query, "--top-n", str(args.top_n)] + if args.fixture_dir: + cmd.extend(["--fixture-dir", args.fixture_dir]) + append_outputs(cmd, args) + return emit_result(run_script("github_benchmark_scan.py", cmd)) + + +def command_intent_confidence(args: argparse.Namespace) -> int: + cmd = [resolved_skill_dir(args)] + if args.context_json: + cmd.extend(["--context-json", args.context_json]) + append_outputs(cmd, args) + return emit_result(run_script("render_intent_confidence.py", cmd)) + + +def command_intent_dialogue(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_intent_dialogue.py") + + +def command_reference_synthesis(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_reference_synthesis.py") + + +def command_output_risk_profile(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_output_risk_profile.py") + + +def command_artifact_design_profile(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_artifact_design_profile.py") + + +def command_prompt_quality_profile(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_prompt_quality_profile.py") + + +def command_system_model(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_system_model.py") + + +def command_iteration_directions(args: argparse.Namespace) -> int: + return render_skill_report_command(args, "render_iteration_directions.py") diff --git a/scripts/yao_cli_runtime.py b/scripts/yao_cli_runtime.py new file mode 100644 index 00000000..bbed20ca --- /dev/null +++ b/scripts/yao_cli_runtime.py @@ -0,0 +1,75 @@ +"""Shared runtime helpers for the Yao CLI.""" + +import json +import subprocess +import sys +from pathlib import Path + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py and command modules for shared subprocess execution and JSON payload parsing." + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = ROOT / "scripts" + + +def script_path(name: str) -> str: + return str(SCRIPTS / name) + + +def load_json_maybe(text: str) -> dict | None: + text = text.strip() + if not text: + return None + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + +def run_script(name: str, args: list[str], cwd: Path | None = None) -> dict: + proc = subprocess.run( + [sys.executable, script_path(name), *args], + cwd=cwd or ROOT, + capture_output=True, + text=True, + ) + payload = load_json_maybe(proc.stdout) + return { + "command": f"{name} {' '.join(args)}".strip(), + "returncode": proc.returncode, + "ok": proc.returncode == 0, + "stdout": proc.stdout, + "stderr": proc.stderr, + "payload": payload, + } + + +def allow_report_status(result: dict, *, allowed_returncodes: set[int] | None = None) -> dict: + allowed = allowed_returncodes or {2} + if result.get("ok") or result.get("returncode") not in allowed or result.get("payload") is None: + return result + normalized = dict(result) + normalized["ok"] = True + normalized["soft_status_returncode"] = result.get("returncode") + normalized["soft_status_reason"] = "report generated with a non-passing evidence decision" + return normalized + + +def run_adoption_drift_if_source_exists(skill_dir: Path | None = None) -> dict: + target = (skill_dir or ROOT).resolve() + events_path = target / "reports" / "telemetry_events.jsonl" + if events_path.exists(): + return run_script("render_adoption_drift_report.py", [str(target)]) + return { + "command": "render_adoption_drift_report.py skipped: missing reports/telemetry_events.jsonl", + "returncode": 0, + "ok": True, + "stdout": "", + "stderr": "", + "payload": { + "ok": True, + "skipped": True, + "reason": "raw telemetry event logs are local-only; keeping committed adoption_drift_report artifacts", + }, + } diff --git a/scripts/yao_cli_telemetry.py b/scripts/yao_cli_telemetry.py new file mode 100644 index 00000000..4bb54b10 --- /dev/null +++ b/scripts/yao_cli_telemetry.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Metadata-only CLI telemetry helpers for yao.py.""" + +import argparse +import os +import sys +from pathlib import Path +from typing import Any + +from render_adoption_drift_report import append_event, normalize_event, skill_defaults, utc_now + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to record opt-in metadata-only CLI run telemetry." + +ENABLE_ENV = "YAO_CLI_TELEMETRY" +EVENTS_ENV = "YAO_CLI_TELEMETRY_EVENTS" +TRUTHY = {"1", "true", "yes", "on"} +FALSY = {"0", "false", "no", "off"} + + +def add_telemetry_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--record-cli-telemetry", + action="store_true", + help="Record metadata-only yao.py command telemetry to reports/telemetry_events.jsonl.", + ) + parser.add_argument( + "--no-cli-telemetry", + action="store_true", + help="Disable yao.py command telemetry even when YAO_CLI_TELEMETRY is enabled.", + ) + parser.add_argument( + "--telemetry-events-jsonl", + help="Override the local metadata-only telemetry JSONL path.", + ) + + +def telemetry_enabled(args: argparse.Namespace, environ: dict[str, str] | None = None) -> bool: + environ = environ or os.environ + if getattr(args, "no_cli_telemetry", False): + return False + if getattr(args, "record_cli_telemetry", False): + return True + raw = environ.get(ENABLE_ENV, "").strip().lower() + if raw in TRUTHY: + return True + if raw in FALSY: + return False + return False + + +def telemetry_path(root: Path, args: argparse.Namespace, environ: dict[str, str] | None = None) -> Path: + environ = environ or os.environ + configured = getattr(args, "telemetry_events_jsonl", None) or environ.get(EVENTS_ENV) + if configured: + return Path(configured).expanduser().resolve() + return root / "reports" / "telemetry_events.jsonl" + + +def normalize_command_name(value: Any) -> str: + raw = str(value or "unknown") + lowered = raw.strip().lower() + safe = "".join(char for char in lowered if char.isalnum() or char in {"-", "_"}) + return safe[:64] or "unknown" + + +def cli_event(root: Path, args: argparse.Namespace, returncode: int) -> dict[str, str]: + defaults = skill_defaults(root) + ok = returncode == 0 + return { + "event": "script_run", + "skill": defaults["skill"], + "version": defaults["version"], + "activation_type": "manual", + "outcome": "accepted" if ok else "failed", + "failure_type": "none" if ok else "script_error", + "timestamp": utc_now(), + "source": "yao_cli", + "command": normalize_command_name(getattr(args, "command", "unknown")), + } + + +def maybe_record_cli_event(root: Path, args: argparse.Namespace, returncode: int) -> None: + if not telemetry_enabled(args): + return + path = telemetry_path(root, args) + event, failures = normalize_event(cli_event(root, args, returncode), skill_defaults(root), "yao-cli") + if failures or event is None: + sys.stderr.write(f"Telemetry skipped: {'; '.join(failures)}\n") + return + try: + append_event(path, event) + except OSError as exc: + sys.stderr.write(f"Telemetry skipped: {exc}\n") diff --git a/security/network_policy.json b/security/network_policy.json index b2eb87d1..44111763 100644 --- a/security/network_policy.json +++ b/security/network_policy.json @@ -32,6 +32,22 @@ "custom_url_policy": "not supported", "timeout_seconds": 15, "auth": "optional GITHUB_TOKEN or GH_TOKEN for GitHub API rate limits" + }, + "scripts/provider_output_eval_runner.py": { + "purpose": "Call reviewed provider API endpoints to collect provider-backed output-eval model evidence.", + "allowed_hosts": [ + "api.deepseek.com", + "api.openai.com" + ], + "allowed_path_prefixes": [ + "/chat/completions", + "/v1/chat/completions", + "/v1/responses" + ], + "requires_https": true, + "custom_url_policy": "only via --provider-base-url plus --allow-custom-base-url; plain HTTP is restricted to --allow-insecure-localhost for local test servers", + "timeout_seconds": 120, + "auth": "OPENAI_API_KEY or DEEPSEEK_API_KEY by provider default, or a reviewed env var through --api-key-env" } } } diff --git a/security/permission_policy.json b/security/permission_policy.json index ec543116..633d9591 100644 --- a/security/permission_policy.json +++ b/security/permission_policy.json @@ -6,7 +6,7 @@ "decision": "approved", "reviewer": "Yao Team", "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", - "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "reason": "Network use is bounded to public GitHub metadata, benchmark discovery, update checks, and reviewed provider output-eval endpoints, with allowed hosts reviewed in security/network_policy.json.", "expires_at": "2026-09-30", "evidence": [ "security/network_policy.json", @@ -15,7 +15,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" } }, "file_write": { @@ -32,7 +33,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" } }, "subprocess": { @@ -48,7 +50,8 @@ "target_enforcement": { "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", "claude": "adapter.json target_permission_contract and README note", - "generic": "adapter.json target_permission_contract" + "generic": "adapter.json target_permission_contract", + "vscode": "adapter.json target_permission_contract plus VS Code workspace-trust note" } } } diff --git a/security/permission_policy.md b/security/permission_policy.md index a0d4c2d9..085531b4 100644 --- a/security/permission_policy.md +++ b/security/permission_policy.md @@ -22,10 +22,12 @@ Each high-permission approval must include: - `reason` - `expires_at` - `evidence` -- `target_enforcement` for OpenAI, Claude, and generic packages +- `target_enforcement` for OpenAI, Claude, generic, and VS Code / Copilot packages Review Studio exposes this as the `permission-gates` gate. Missing, invalid, or expired approvals block governed mode and stay reviewer-visible in lighter modes. -After packaging, run `python3 scripts/probe_runtime_permissions.py . --package-dir dist` to generate `reports/runtime_permission_probes.md` and `reports/runtime_permission_probes.json`. Review Studio exposes that evidence as the `permission-runtime` gate, separate from approval governance. A passing probe means target adapters make permission handling explicit and auditable; it does not claim client-native enforcement when the target only supports metadata fallback. +Install simulation also enforces this policy after archive extraction. `python3 scripts/simulate_install.py . --package-dir dist` reads the installed `security/permission_policy.json`, reads each installed target adapter's `target_permission_contract.declared_capabilities`, and fails when any target/capability pair lacks an active reviewer approval or target-specific enforcement note. This keeps package distribution from relying only on metadata presence in the source tree. + +After packaging, run `python3 scripts/probe_runtime_permissions.py . --package-dir dist` to generate `reports/runtime_permission_probes.md` and `reports/runtime_permission_probes.json`. The probe also reads `reports/install_simulation.json` when it matches the probed package directory, so reviewers can see local installer enforcement next to target-native enforcement. Review Studio exposes that evidence as the `permission-runtime` gate, separate from approval governance. A passing probe means target adapters make permission handling explicit and auditable; local installer enforcement is supporting distribution evidence, and does not claim client-native enforcement when the target only supports metadata fallback. Current targets preserve permission semantics as metadata. They do not enforce permissions at runtime. Any future native installer or client integration must map these capabilities into that client's permission model rather than dropping them. diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index 88756678..d1de6b8c 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -4,7 +4,7 @@ "title": "Yao Meta Skill", "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger_surface": { - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "should_trigger": [ "Create a skill from this repeated workflow.", "Turn this runbook into a reusable agent skill.", @@ -40,21 +40,18 @@ }, "workflow": { "steps": [ - "Decide whether the request should become a skill and choose the lightest fit.", - "Capture job, output, exclusions, constraints, and standards.", - "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", - "Write the `description` early and test route quality before expanding the package.", - "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", - "Add only folders and gates that earn their keep.", - "Surface the top three next iteration directions." + "For one-off/no reusable process: `Do not create a skill`; `near-neighbor`; require `repeated use` + `reusable output contract`.", + "Capture job, output, exclusions, constraints, standards, and the lightest fit.", + "Scan references in order: external benchmark, user source, local fit; surface only uncertainty or conflict.", + "Write `description` early, test route quality, then add only earned folders and gates.", + "Add output-risk, artifact-design, prompt-quality, system-model, and next directions only when useful." ], "decision_points": [ "Route by frontmatter `description`.", "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", "Use the lightest reliable process.", - "`Scaffold`: exploratory or personal.", - "`Production`: team reuse.", - "`Library`: shared infrastructure.", + "`Scaffold`: exploratory/personal. `Production`: team reuse. `Library`: shared infrastructure. `Governed`: high-trust, policy-sensitive, or release-critical.", + "Rules: [Method](references/skill-engineering-method.md), [Operating Modes](references/operating-modes.md), [Resource Boundaries](references/resource-boundaries.md).", "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." @@ -76,6 +73,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -103,39 +101,71 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", - "references/trust-security-method.md" + "references/trust-security-method.md", + "references/user-memory-policy.md" ], "scripts": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -150,32 +180,93 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", - "scripts/yao_cli_config.py" + "scripts/yao_cli_adaptation_commands.py", + "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py" ], "assets": [ + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], @@ -215,6 +306,8 @@ "reports/security_trust_report.md", "reports/runtime_permission_probes.json", "reports/runtime_permission_probes.md", + "reports/telemetry_hook_recipes.json", + "reports/telemetry_hook_recipes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", @@ -226,6 +319,7 @@ "eval_plan": { "trigger": [ "evals/README.md", + "evals/adaptation/user_signals.example.jsonl", "evals/adversarial/trigger_cases.json", "evals/blind_holdout/trigger_cases.json", "evals/confusion/route_cases.json", @@ -277,7 +371,8 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "source_files": [ "SKILL.md", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index cfa9108d..73e58a17 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -5,7 +5,7 @@ { "name": "yao-meta-skill", "path": ".", - "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.", "owner": "Yao Team", "version": "1.1.0", "status": "active", @@ -16,37 +16,68 @@ "openai", "claude", "generic", - "agent-skills-compatible" + "agent-skills-compatible", + "vscode" ], "resources": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", + "scripts/benchmark_release_lock.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", "scripts/compile_skill.py", + "scripts/compile_skill_targets.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_phase_queue.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", + "scripts/html_rendering.py", + "scripts/import_output_review_decisions.py", + "scripts/import_telemetry_events.py", "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", + "scripts/prepare_output_review_kit.py", + "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/propose_adaptation.py", + "scripts/provider_output_eval_runner.py", + "scripts/python_compat_check.py", + "scripts/reference_synthesis_markdown.py", "scripts/registry_audit.py", "scripts/render_adoption_drift_report.py", + "scripts/render_architecture_maintainability.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", + "scripts/render_daily_skillops_report.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/render_intent_confidence.py", "scripts/render_intent_dialogue.py", "scripts/render_iteration_directions.py", @@ -61,32 +92,90 @@ "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_review_waivers.py", + "scripts/render_skill_interpretation.py", + "scripts/render_skill_os2_audit.py", + "scripts/render_skill_os2_coverage.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", + "scripts/render_telemetry_hook_recipes.py", + "scripts/render_weekly_curator_report.py", + "scripts/render_world_class_claim_guard.py", + "scripts/render_world_class_evidence_intake.py", + "scripts/render_world_class_evidence_ledger.py", + "scripts/render_world_class_evidence_plan.py", + "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", + "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", + "scripts/review_studio_action_evidence.py", + "scripts/review_studio_actions.py", + "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_output_review.py", + "scripts/review_studio_panels.py", + "scripts/review_studio_skillops.py", + "scripts/review_studio_waivers.py", + "scripts/review_studio_world_class.py", + "scripts/review_viewer_data.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "scripts/simulate_install.py", + "scripts/skill_ir_paths.py", + "scripts/skill_os2_coverage_markdown.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skillops_opportunity.py", + "scripts/summarize_user_signals.py", "scripts/sync_local_install.py", + "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", + "scripts/world_class_operator_runbook_coordination.py", + "scripts/world_class_phase_queue.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_preflight_markdown.py", + "scripts/world_class_provider_evidence.py", + "scripts/world_class_repair_checklist.py", + "scripts/world_class_source_checks.py", + "scripts/world_class_submission_kit_rendering.py", + "scripts/world_class_submission_matrix.py", "scripts/yao.py", + "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", + "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_runtime.py", + "scripts/yao_cli_telemetry.py", "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/autonomous-adaptation.md", "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", @@ -114,35 +203,48 @@ "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", + "references/skillops-decision-policy.md", "references/systems-thinking-doctrine.md", "references/telemetry-drift-method.md", "references/trust-security-method.md", + "references/user-memory-policy.md", + "assets/review-studio.css", + "assets/review-viewer.css", + "assets/skill-overview.css", + "assets/skill-overview.js", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" ], "token_set": [ - "add", - "asked", + "creation", + "distribution", "docs", "evals", "evaluate", - "existing", - "improve", + "improvement", "notes", "or", "package", - "process", + "packaging", "prompts", "refactor", - "repeated", - "reuse", - "team", + "team-ready", "transcripts", + "workflow", "workflows" ], "atlas_scope": "release", "actionable": true, - "scope_reason": "default release-actionable skill" + "scope_reason": "default release-actionable skill", + "telemetry": { + "report_present": true, + "report": "reports/adoption_drift_report.json", + "risk_band": "low", + "event_count": 1, + "adoption_sample_count": 1, + "adoption_rate": 100.0, + "candidate_count": 0 + } }, { "name": "release-orchestrator", @@ -200,7 +302,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/complex-release-orchestrator/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -239,7 +350,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/final/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -262,7 +382,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v0/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -300,7 +429,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v1/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "incident-command-governor", @@ -341,7 +479,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/governed-incident-command/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "note-cleanup", @@ -378,7 +525,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/simple-note-cleanup/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -418,7 +574,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/team-frontend-review/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "geo-ranking-article-generator", @@ -478,7 +643,16 @@ ], "atlas_scope": "embedded-generated-skill", "actionable": false, - "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release.", + "telemetry": { + "report_present": false, + "report": "geo-ranking-article-generator/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "invalid-governance-skill", @@ -500,7 +674,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/governance_invalid_manifest/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-yaml-skill", @@ -524,7 +707,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_invalid_yaml/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-skill", @@ -547,7 +739,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_missing_interface_field/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } } ], "summary": { @@ -561,6 +762,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 } } diff --git a/skill_atlas/drift_signals.json b/skill_atlas/drift_signals.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/skill_atlas/drift_signals.json @@ -0,0 +1 @@ +[] diff --git a/skill_atlas/route_overlap_matrix.csv b/skill_atlas/route_overlap_matrix.csv index 6f598a2b..e245622c 100644 --- a/skill_atlas/route_overlap_matrix.csv +++ b/skill_atlas/route_overlap_matrix.csv @@ -1,12 +1,12 @@ skill_a,skill_b,path_a,path_b,score,status,actionable,scope_a,scope_b -yao-meta-skill,release-orchestrator,.,examples/complex-release-orchestrator/generated-skill,0.061,clear,False,release,example -yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/final,0.054,clear,False,release,example +yao-meta-skill,release-orchestrator,.,examples/complex-release-orchestrator/generated-skill,0.064,clear,False,release,example +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/final,0.028,clear,False,release,example yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v0,0.0,clear,False,release,example -yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v1,0.056,clear,False,release,example -yao-meta-skill,incident-command-governor,.,examples/governed-incident-command/generated-skill,0.067,clear,False,release,example -yao-meta-skill,note-cleanup,.,examples/simple-note-cleanup/generated-skill,0.088,clear,False,release,example -yao-meta-skill,frontend-review,.,examples/team-frontend-review/generated-skill,0.056,clear,False,release,example -yao-meta-skill,geo-ranking-article-generator,.,geo-ranking-article-generator,0.038,clear,False,release,embedded-generated-skill +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v1,0.029,clear,False,release,example +yao-meta-skill,incident-command-governor,.,examples/governed-incident-command/generated-skill,0.034,clear,False,release,example +yao-meta-skill,note-cleanup,.,examples/simple-note-cleanup/generated-skill,0.061,clear,False,release,example +yao-meta-skill,frontend-review,.,examples/team-frontend-review/generated-skill,0.029,clear,False,release,example +yao-meta-skill,geo-ranking-article-generator,.,geo-ranking-article-generator,0.02,clear,False,release,embedded-generated-skill yao-meta-skill,invalid-governance-skill,.,tests/fixtures/governance_invalid_manifest,0.0,clear,False,release,test-fixture yao-meta-skill,broken-yaml-skill,.,tests/fixtures/package_invalid_yaml,0.0,clear,False,release,test-fixture yao-meta-skill,broken-skill,.,tests/fixtures/package_missing_interface_field,0.0,clear,False,release,test-fixture diff --git a/tests/review_studio_helpers.py b/tests/review_studio_helpers.py new file mode 100644 index 00000000..be697f51 --- /dev/null +++ b/tests/review_studio_helpers.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = ROOT / "scripts" +SCRIPT = SCRIPTS / "render_review_studio.py" + + +def run(args: list[object], allowed_returncodes: tuple[int, ...] = (0,)) -> subprocess.CompletedProcess[str]: + proc = subprocess.run([str(item) for item in args], cwd=ROOT, check=False, capture_output=True, text=True) + if proc.returncode not in allowed_returncodes: + raise subprocess.CalledProcessError(proc.returncode, proc.args, output=proc.stdout, stderr=proc.stderr) + return proc + + +def run_script( + script_name: str, + *args: object, + allowed_returncodes: tuple[int, ...] = (0,), +) -> subprocess.CompletedProcess[str]: + return run([sys.executable, SCRIPTS / script_name, *args], allowed_returncodes=allowed_returncodes) + + +def prepare_tmp_root() -> Path: + tmp_root = ROOT / "tests" / "tmp_review_studio" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + return tmp_root + + +def copy_tmp_report(tmp_root: Path, stem: str) -> None: + for suffix in ("json", "md"): + (ROOT / "reports" / f"{stem}.{suffix}").write_text( + (tmp_root / f"{stem}.{suffix}").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + +def output_execution_has_provider_evidence() -> bool: + path = ROOT / "reports" / "output_execution_runs.json" + if not path.exists(): + return False + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return False + summary = payload.get("summary", {}) + if not isinstance(summary, dict): + return False + return ( + int(summary.get("model_executed_count") or 0) > 0 + and int(summary.get("timing_observed_count") or 0) > 0 + and int(summary.get("token_observed_count") or 0) > 0 + ) + + +def prepare_review_studio_inputs(tmp_root: Path) -> None: + run_script("run_output_eval.py") + run_script("prepare_output_review_kit.py") + if not output_execution_has_provider_evidence(): + run_script("run_output_execution.py", "--runner-command", json.dumps(["python3", "scripts/local_output_eval_runner.py"])) + run_script("adjudicate_output_review.py") + run_script("compile_skill.py", ROOT, "--generated-at", "2026-06-13") + + package_dir = tmp_root / "dist" + run_script( + "cross_packager.py", + ROOT, + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--platform", + "vscode", + "--expectations", + ROOT / "evals" / "packaging_expectations.json", + "--output-dir", + package_dir, + "--zip", + ) + run_script( + "simulate_install.py", + ROOT, + "--package-dir", + package_dir, + "--install-root", + tmp_root / "install-root", + "--output-json", + tmp_root / "install_simulation.json", + "--output-md", + tmp_root / "install_simulation.md", + "--generated-at", + "2026-06-13", + ) + copy_tmp_report(tmp_root, "install_simulation") + + run_script("registry_audit.py", ROOT, "--generated-at", "2026-06-13") + run_script("render_intent_confidence.py", ROOT) + run_script("trust_check.py", ROOT) + run_script("python_compat_check.py", ROOT, "--generated-at", "2026-06-13") + run_script("render_architecture_maintainability.py", ROOT, "--generated-at", "2026-06-13") + run_script("render_context_reports.py") + run_script( + "render_adoption_drift_report.py", + ROOT, + "--events-jsonl", + tmp_root / "telemetry_events.jsonl", + "--record-event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--timestamp", + "2026-06-13T10:00:00Z", + ) + run_script( + "build_skill_atlas.py", + "--workspace-root", + ROOT, + "--output-dir", + ROOT / "skill_atlas", + "--report-html", + ROOT / "reports" / "skill_atlas.html", + "--report-json", + ROOT / "reports" / "skill_atlas.json", + "--today", + "2026-06-13", + ) + run_script( + "upgrade_check.py", + ROOT, + "--previous-package-json", + ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json", + "--current-package-json", + ROOT / "reports" / "registry_audit.json", + "--output-json", + tmp_root / "upgrade_check.json", + "--output-md", + tmp_root / "upgrade_check.md", + "--generated-at", + "2026-06-13", + ) + copy_tmp_report(tmp_root, "upgrade_check") + + run_script("render_review_waivers.py", ROOT, "--generated-at", "2026-06-13") + run_script( + "render_review_annotations.py", + ROOT, + "--annotations-json", + tmp_root / "empty_review_annotations_input.json", + ) + run_script("probe_runtime_permissions.py", ROOT, "--package-dir", package_dir) + + for script_name in [ + "render_skill_os2_audit.py", + "render_world_class_evidence_plan.py", + "render_world_class_evidence_ledger.py", + "render_world_class_evidence_intake.py", + "render_world_class_preflight.py", + "render_world_class_submission_review.py", + "render_world_class_operator_runbook.py", + "render_world_class_claim_guard.py", + "render_skill_os2_coverage.py", + "render_benchmark_reproducibility.py", + ]: + allowed_returncodes = (0, 2) if script_name == "render_world_class_submission_review.py" else (0,) + run_script(script_name, ROOT, "--generated-at", "2026-06-13", allowed_returncodes=allowed_returncodes) + + +def render_review_studio_fixture(tmp_root: Path) -> tuple[Path, Path, subprocess.CompletedProcess[str]]: + prepare_review_studio_inputs(tmp_root) + output_html = tmp_root / "review-studio.html" + output_json = tmp_root / "review-studio.json" + proc = run( + [ + sys.executable, + SCRIPT, + ROOT, + "--output-html", + output_html, + "--output-json", + output_json, + ] + ) + return output_html, output_json, proc diff --git a/tests/review_studio_world_class_assertions.py b/tests/review_studio_world_class_assertions.py new file mode 100644 index 00000000..c7f84be0 --- /dev/null +++ b/tests/review_studio_world_class_assertions.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Focused assertions for Review Studio world-class evidence actions.""" + + +def assert_world_class_action(full_payload: dict) -> None: + world_class_action = next( + item for item in full_payload["review_actions"] if item["gate_key"] == "world-class-evidence" + ) + assert {item["path"] for item in world_class_action["source_refs"]} >= { + "reports/world_class_evidence_ledger.md", + "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_intake.md", + "reports/world_class_evidence_preflight.md", + "reports/world_class_evidence_preflight.html", + "reports/world_class_submission_review.md", + "reports/world_class_claim_guard.md", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/skill_os2_audit.md", + }, world_class_action + assert all(item["exists"] for item in world_class_action["source_refs"]), world_class_action + assert all("matched_pattern" in item for item in world_class_action["source_refs"]), world_class_action + assert all("excerpt" in item for item in world_class_action["source_refs"]), world_class_action + assert all(isinstance(item["line"], int) and item["line"] >= 1 for item in world_class_action["source_refs"]), world_class_action + assert any( + item["path"] == "reports/world_class_evidence_ledger.md" + and item["matched_pattern"] == "# World-Class Evidence Ledger" + and "World-Class Evidence Ledger" in item["excerpt"] + for item in world_class_action["source_refs"] + ), world_class_action + assert "world-class-runbook" in world_class_action["verification_command"], world_class_action + assert "--submissions-dir evidence/world_class/submissions" in world_class_action["verification_command"], world_class_action + assert "reports/world_class_operator_runbook.html" in world_class_action["source_fix"], world_class_action + assert {item["key"] for item in world_class_action["evidence_steps"]} == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, world_class_action + + provider_action_step = next( + item for item in world_class_action["evidence_steps"] if item["key"] == "provider-holdout" + ) + assert provider_action_step["status"] == "pending", provider_action_step + provider_readiness = provider_action_step["readiness"] + assert provider_readiness in {"fix-submission", "awaiting-submission"}, provider_action_step + assert provider_action_step["submission_path"] == "evidence/world_class/submissions/provider-holdout.json", provider_action_step + assert provider_action_step["template_path"] == "evidence/world_class/templates/provider-holdout.intake.json", provider_action_step + assert provider_action_step["source_blocked_count"] == 0, provider_action_step + assert provider_action_step["repair_blocked_count"] == 1, provider_action_step + assert provider_action_step["repair_counts_as_completion"] is False, provider_action_step + assert provider_action_step["phase_queue_blocked_count"] == 1, provider_action_step + assert provider_action_step["phase_queue_counts_as_completion"] is False, provider_action_step + provider_phase_queue = {item["phase"]: item for item in provider_action_step["phase_queue"]} + assert set(provider_phase_queue) == {"unblock-access"}, provider_phase_queue + assert provider_phase_queue["unblock-access"]["next_action_id"] == "provider-holdout-precheck-provider-api-key", provider_phase_queue + assert provider_phase_queue["unblock-access"]["row_count"] == 1, provider_phase_queue + assert "operator with provider credentials" in provider_phase_queue["unblock-access"]["owners"], provider_phase_queue + assert provider_action_step["blocked_checks"] == [], provider_action_step + provider_repair_rows = {item["target"]: item for item in provider_action_step["repair_rows"]} + assert set(provider_repair_rows) == {"provider-api-key"}, provider_repair_rows + assert provider_repair_rows["provider-api-key"]["action_id"] == "provider-holdout-precheck-provider-api-key", provider_repair_rows + assert provider_repair_rows["provider-api-key"]["repair_type"] == "precheck", provider_repair_rows + assert provider_repair_rows["provider-api-key"]["phase"] == "unblock-access", provider_repair_rows + assert provider_repair_rows["provider-api-key"]["priority"] == 20, provider_repair_rows + assert provider_repair_rows["provider-api-key"]["owner"] == "operator with provider credentials", provider_repair_rows + assert all(item["counts_as_completion"] is False for item in provider_action_step["repair_rows"]), provider_action_step + assert any("world-class-intake" in item["command"] for item in provider_action_step["commands"]), provider_action_step + assert any("world-class-ledger" in item["command"] for item in provider_action_step["commands"]), provider_action_step + assert any("output-exec --provider-runner openai" in item for item in provider_action_step["runbook"]), provider_action_step + assert not any("" in item or "OPENAI_API_KEY=" in item for item in provider_action_step["runbook"]), provider_action_step + assert "provider-backed model run" in provider_action_step["provenance_requirements"], provider_action_step + assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider_action_step["success_checks"], provider_action_step + assert "reports/output_execution_runs.json" in provider_action_step["evidence_artifacts"], provider_action_step + provider_role_contract = provider_action_step["artifact_role_contract"] + assert provider_role_contract["role_source"] == "world-class-submission-kit", provider_role_contract + assert provider_role_contract["counts_as_evidence"] is False, provider_role_contract + assert provider_role_contract["artifact_prefill_counts_as_evidence"] is False, provider_role_contract + assert provider_role_contract["submission_ref_total_count"] == 1, provider_role_contract + assert provider_role_contract["submission_ref_ready_count"] == 1, provider_role_contract + provider_role_rows = {item["role"]: item for item in provider_role_contract["roles"]} + assert provider_role_rows["submission-ref"]["copy_to_artifact_refs"] is True, provider_role_rows + assert provider_role_rows["supporting-evidence"]["copy_to_artifact_refs"] is False, provider_role_rows + assert any("provider credentials" in item for item in provider_action_step["privacy_contract"]), provider_action_step + + human_action_step = next( + item for item in world_class_action["evidence_steps"] if item["key"] == "human-adjudication" + ) + assert human_action_step["repair_blocked_count"] >= 2, human_action_step + assert human_action_step["phase_queue_blocked_count"] == 2, human_action_step + human_repair_rows = {item["target"]: item for item in human_action_step["repair_rows"]} + assert human_repair_rows["human-reviewer"]["repair_type"] == "precheck", human_repair_rows + assert human_repair_rows["human-reviewer"]["owner"] == "human reviewer", human_repair_rows + assert human_repair_rows["pending_count"]["repair_type"] == "source-check", human_repair_rows + assert "output-review" in human_repair_rows["pending_count"]["verification_command"], human_repair_rows + assert "prompt_sha256" in " ".join(human_action_step["success_checks"]), human_action_step + assert "prompt_sha256" in " ".join(human_action_step["privacy_contract"]), human_action_step + + expected_awaiting_count = 3 if provider_readiness == "fix-submission" else 4 + expected_invalid_count = 1 if provider_readiness == "fix-submission" else 0 + expected_intake_decision = "fix-intake" if provider_readiness == "fix-submission" else "awaiting-submissions" + expected_review_decision = "fix-submissions" if provider_readiness == "fix-submission" else "awaiting-submissions" + assert full_payload["data"]["world_class_evidence_ledger"]["summary"]["pending_count"] == 4 + assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == expected_intake_decision + submission_review = full_payload["data"]["world_class_submission_review"] + assert submission_review["summary"]["decision"] == expected_review_decision, submission_review + assert submission_review["summary"]["review_counts_submission_as_completion"] is False, submission_review + assert submission_review["summary"]["awaiting_submission_count"] == expected_awaiting_count, submission_review + assert submission_review["summary"]["invalid_submission_count"] == expected_invalid_count, submission_review + assert submission_review["summary"]["source_check_count"] >= 13, submission_review + assert submission_review["summary"]["source_blocked_count"] >= 6, submission_review + human_review_item = next(item for item in submission_review["items"] if item["evidence_key"] == "human-adjudication") + human_review_source = {item["field"]: item for item in human_review_item["source_checklist"]} + assert human_review_item["observed_state"]["raw_content_allowed"] is False, human_review_item + assert human_review_item["observed_state"]["raw_content_path_count"] == 0, human_review_item + assert human_review_source["raw_content_allowed"]["status"] == "pass", human_review_source + + runbook = full_payload["data"]["world_class_operator_runbook"] + assert runbook["summary"]["decision"] == "collect-evidence", runbook + assert runbook["summary"]["runbook_counts_as_completion"] is False, runbook + assert runbook["summary"]["awaiting_submission_count"] == expected_awaiting_count, runbook + assert runbook["summary"]["invalid_submission_count"] == expected_invalid_count, runbook + intake = full_payload["data"]["world_class_evidence_intake"] + assert intake["summary"]["template_pass_count"] == 4, intake + assert intake["summary"]["operator_checklist_count"] == 4, intake + assert intake["summary"]["operator_checklist_ready_count"] == 0, intake + assert intake["summary"]["ready_to_claim_world_class"] is False, intake + provider_checklist = next(item for item in intake["operator_checklist"] if item["evidence_key"] == "provider-holdout") + assert provider_checklist["readiness"] == provider_readiness, provider_checklist + if provider_readiness == "fix-submission": + assert provider_checklist["submission_status"] == "fail", provider_checklist + else: + assert provider_checklist["submission_status"] in {"missing", "not-found", "not-submitted"}, provider_checklist + assert provider_checklist["source_accepted"] is True, provider_checklist + assert provider_checklist["submission_path"] == "evidence/world_class/submissions/provider-holdout.json", provider_checklist + assert provider_checklist["commands"]["validate_intake"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", provider_checklist + assert provider_checklist["commands"]["submission_review"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", provider_checklist + assert provider_checklist["commands"]["refresh_ledger"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", provider_checklist + assert "provider-backed model run" in provider_checklist["must_collect"]["provenance_requirements"], provider_checklist + assert any( + "output-exec --provider-runner openai" in step for step in provider_checklist["must_collect"]["runbook"] + ), provider_checklist + assert full_payload["data"]["world_class_claim_guard"]["summary"]["decision"] == "claim-guard-pass-evidence-pending" + assert full_payload["data"]["world_class_claim_guard"]["summary"]["violation_count"] == 0 + assert full_payload["data"]["world_class_claim_guard"]["summary"]["ledger_pending_count"] == 4 diff --git a/tests/snapshots/vscode_adapter.json b/tests/snapshots/vscode_adapter.json new file mode 100644 index 00000000..a72867e5 --- /dev/null +++ b/tests/snapshots/vscode_adapter.json @@ -0,0 +1,26 @@ +{ + "platform": "vscode", + "canonical_metadata": "agents/interface.yaml", + "required_generated_file": "targets/vscode/README.md", + "required_fields": [ + "canonical_format", + "activation_mode", + "execution_context", + "shell", + "trust_level", + "remote_inline_execution", + "degradation_strategy", + "portability_profile", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", + "compiler", + "compiled_contract", + "permission_contract", + "target_permission_contract", + "target_native_contract", + "target_transform" + ] +} diff --git a/tests/verify_adaptation_safety.py b/tests/verify_adaptation_safety.py new file mode 100644 index 00000000..b6b337eb --- /dev/null +++ b/tests/verify_adaptation_safety.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +import hashlib +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +CLI = ROOT / "scripts" / "yao.py" +SCAN_SCRIPT = ROOT / "scripts" / "summarize_user_signals.py" +PROPOSE_SCRIPT = ROOT / "scripts" / "propose_adaptation.py" +APPLY_SCRIPT = ROOT / "scripts" / "apply_adaptation.py" +TMP = ROOT / "tests" / "tmp_adaptation_safety" + + +def run_script(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *args], + cwd=ROOT, + capture_output=True, + text=True, + ) + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def assert_report_contract_mirrors(payload: dict, contract_key: str) -> None: + contract = payload["report_contract"] + assert contract["top_level_mirrors_summary"] is True, payload + assert contract[f"top_level_mirrors_{contract_key}"] is True, payload + for field in contract["summary_fields"]: + assert payload[field] == payload["summary"][field], (field, payload.get(field), payload["summary"].get(field)) + for field in contract[f"{contract_key}_fields"]: + assert payload[field] == payload[contract_key][field], (field, payload.get(field), payload[contract_key].get(field)) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + skill_dir = TMP / "adaptive-demo-skill" + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + source = TMP / "curated-user-signals.jsonl" + source.write_text( + "\n".join( + [ + json.dumps({"text": "报告默认中文简体，同时右上角提供英文切换。"}, ensure_ascii=False), + json.dumps({"message": "新的 HTML 报告要双语，但默认中文简体。"}, ensure_ascii=False), + json.dumps({"content": "报告 UI 需要 Kami 白底排版、图表模块和清晰导航。"}, ensure_ascii=False), + json.dumps({"excerpt": "HTML 报告还是白底 Kami 风格，图表不要挤在一起。"}, ensure_ascii=False), + json.dumps({"note": "不要自动扫描私人日志；必须由用户提供明确路径。"}, ensure_ascii=False), + json.dumps({"body": "自适应升级需要先输出提案，授权后再修改，并能回滚。"}, ensure_ascii=False), + json.dumps({"text": "隐私证据里也要保护 token=abc123456789、sk-1234567890abcdef 和 /Users/laoyao/private/path。"}, ensure_ascii=False), + json.dumps({"text": "PDF 只提过一次，不能当作稳定偏好。"}, ensure_ascii=False), + ] + ) + + "\n", + encoding="utf-8", + ) + + scan_proc = run_script( + str(SCAN_SCRIPT), + str(skill_dir), + "--source", + str(source), + "--generated-at", + "2026-06-15T00:00:00Z", + ) + assert scan_proc.returncode == 0, scan_proc.stderr + scan_payload = json.loads(scan_proc.stdout) + assert scan_payload["ok"], scan_payload + assert scan_payload["privacy_contract"]["local_only"] is True, scan_payload + assert scan_payload["privacy_contract"]["implicit_private_log_scan"] is False, scan_payload + assert scan_payload["privacy_contract"]["raw_content_stored"] is False, scan_payload + assert scan_payload["privacy_contract"]["writes_repository_files"] is False, scan_payload + assert scan_payload["source"]["path"].startswith("[external-explicit-source]/"), scan_payload + pattern_ids = {item["pattern_id"] for item in scan_payload["patterns"]} + assert {"language_default", "report_ui", "approval_safety"} <= pattern_ids, scan_payload + serialized = json.dumps(scan_payload, ensure_ascii=False) + assert "sk-1234567890abcdef" not in serialized, serialized + assert "token=abc123456789" not in serialized, serialized + assert "/Users/laoyao/private" not in serialized, serialized + assert "[REDACTED_SECRET]" in serialized, serialized + assert "[LOCAL_PATH]" in serialized, serialized + assert (reports_dir / "user_patterns.json").exists(), reports_dir + assert (reports_dir / "user_patterns.md").exists(), reports_dir + + propose_proc = run_script( + str(PROPOSE_SCRIPT), + str(skill_dir), + "--generated-at", + "2026-06-15T00:00:00Z", + ) + assert propose_proc.returncode == 0, propose_proc.stderr + proposal_payload = json.loads(propose_proc.stdout) + assert proposal_payload["ok"], proposal_payload + assert_report_contract_mirrors(proposal_payload, "proposal_contract") + assert proposal_payload["summary"]["apply_supported"] is True, proposal_payload + assert proposal_payload["proposal_count"] == proposal_payload["summary"]["proposal_count"], proposal_payload + assert proposal_payload["approval_draft_supported"] is True, proposal_payload + assert proposal_payload["proposal_contract"]["proposal_only"] is True, proposal_payload + assert proposal_payload["proposal_contract"]["writes_repository_files"] is False, proposal_payload + assert proposal_payload["proposal_contract"]["apply_command_available"] is True, proposal_payload + assert proposal_payload["proposal_contract"]["target_file_sha256_required_for_apply"] is True, proposal_payload + assert proposal_payload["proposal_contract"]["approval_draft_supported"] is True, proposal_payload + assert proposal_payload["summary"]["proposal_count"] >= 3, proposal_payload + assert all(item["status"] == "proposal-only" for item in proposal_payload["proposals"]), proposal_payload + assert all(item["requires_approval"] is True for item in proposal_payload["proposals"]), proposal_payload + assert all(item["write_allowed_without_approval"] is False for item in proposal_payload["proposals"]), proposal_payload + assert any( + any("tests/verify_adaptation_safety.py" in command for command in item["verification_commands"]) + for item in proposal_payload["proposals"] + ), proposal_payload + assert (reports_dir / "adaptation_proposals.json").exists(), reports_dir + assert (reports_dir / "adaptation_proposals.md").exists(), reports_dir + + template_proc = run_script(str(APPLY_SCRIPT), str(skill_dir), "--write-template", "--generated-at", "2026-06-15T00:00:00Z") + assert template_proc.returncode == 0, template_proc.stderr + template_payload = json.loads(template_proc.stdout) + assert_report_contract_mirrors(template_payload, "apply_contract") + assert template_payload["summary"]["apply_supported"] is True, template_payload + assert template_payload["apply_supported"] is True, template_payload + assert template_payload["summary"]["attempt_count"] == 0, template_payload + assert template_payload["apply_contract"]["target_file_sha256_required"] is True, template_payload + assert template_payload["apply_contract"]["approval_draft_supported"] is True, template_payload + assert (reports_dir / "adaptation_approval_ledger.json").exists(), reports_dir + assert (reports_dir / "adaptation_regression_report.json").exists(), reports_dir + template_ledger = json.loads((reports_dir / "adaptation_approval_ledger.json").read_text(encoding="utf-8")) + assert_report_contract_mirrors(template_ledger, "approval_contract") + assert template_ledger["pending_review_count"] == 0, template_ledger + + missing_source_proc = run_script(str(SCAN_SCRIPT), str(skill_dir)) + assert missing_source_proc.returncode != 0, missing_source_proc + + history_source = TMP / ".zsh_history" + history_source.write_text("报告默认中文\n报告默认中文\n", encoding="utf-8") + history_proc = run_script(str(SCAN_SCRIPT), str(skill_dir), "--source", str(history_source)) + assert history_proc.returncode == 2, history_proc.stdout + history_payload = json.loads(history_proc.stdout) + assert history_payload["ok"] is False, history_payload + assert any("Refusing private history source" in item for item in history_payload["failures"]), history_payload + + cli_skill_dir = TMP / "cli-adaptive-demo-skill" + cli_skill_dir.mkdir(parents=True, exist_ok=True) + cli_source = TMP / "cli-user-signals.jsonl" + cli_source.write_text( + "\n".join( + [ + json.dumps({"text": "报告默认中文简体，同时提供英文切换。"}, ensure_ascii=False), + json.dumps({"text": "新的 HTML 报告还是默认中文简体，并保留英文版。"}, ensure_ascii=False), + json.dumps({"text": "报告 UI 要保持 Kami 白底排版和图表模块。"}, ensure_ascii=False), + json.dumps({"text": "HTML 报告的图表和白底 Kami 排版都要清晰。"}, ensure_ascii=False), + json.dumps({"text": "自适应升级必须先生成提案，授权后再修改。"}, ensure_ascii=False), + json.dumps({"text": "不要默认扫描私人日志，要由用户提供明确路径。"}, ensure_ascii=False), + ] + ) + + "\n", + encoding="utf-8", + ) + cli_scan = run_script( + str(CLI), + "adapt-scan", + str(cli_skill_dir), + "--source", + str(cli_source), + "--generated-at", + "2026-06-15T00:00:00Z", + ) + assert cli_scan.returncode == 0, cli_scan.stderr + cli_scan_payload = json.loads(cli_scan.stdout) + assert cli_scan_payload["summary"]["pattern_count"] >= 3, cli_scan_payload + assert cli_scan_payload["privacy_contract"]["writes_repository_files"] is False, cli_scan_payload + assert (cli_skill_dir / "reports" / "user_patterns.json").exists(), cli_skill_dir + cli_propose = run_script( + str(CLI), + "adapt-propose", + str(cli_skill_dir), + "--generated-at", + "2026-06-15T00:00:00Z", + ) + assert cli_propose.returncode == 0, cli_propose.stderr + cli_proposal_payload = json.loads(cli_propose.stdout) + assert_report_contract_mirrors(cli_proposal_payload, "proposal_contract") + assert cli_proposal_payload["summary"]["proposal_count"] >= 3, cli_proposal_payload + assert cli_proposal_payload["proposal_count"] == cli_proposal_payload["summary"]["proposal_count"], cli_proposal_payload + assert cli_proposal_payload["summary"]["apply_supported"] is True, cli_proposal_payload + assert cli_proposal_payload["proposal_contract"]["proposal_only"] is True, cli_proposal_payload + assert cli_proposal_payload["proposal_contract"]["writes_repository_files"] is False, cli_proposal_payload + assert cli_proposal_payload["proposal_contract"]["apply_command_available"] is True, cli_proposal_payload + assert cli_proposal_payload["proposal_contract"]["target_file_sha256_required_for_apply"] is True, cli_proposal_payload + assert cli_proposal_payload["proposal_contract"]["approval_draft_supported"] is True, cli_proposal_payload + + policy_path = cli_skill_dir / "references" / "user-memory-policy.md" + policy_path.parent.mkdir(parents=True, exist_ok=True) + policy_path.write_text("old policy\n", encoding="utf-8") + verifier = cli_skill_dir / "tests" / "check_policy.py" + verifier.parent.mkdir(parents=True, exist_ok=True) + verifier.write_text( + "from pathlib import Path\n" + "assert 'approved adaptive note' in Path('references/user-memory-policy.md').read_text(encoding='utf-8')\n", + encoding="utf-8", + ) + approval_proposal = next( + item for item in cli_proposal_payload["proposals"] if item["pattern_id"] == "approval_safety" + ) + patch_text = ( + "diff --git a/references/user-memory-policy.md b/references/user-memory-policy.md\n" + "--- a/references/user-memory-policy.md\n" + "+++ b/references/user-memory-policy.md\n" + "@@ -1 +1,2 @@\n" + " old policy\n" + "+approved adaptive note\n" + ) + patch_path = TMP / "approved-adaptation.patch" + patch_path.write_text(patch_text, encoding="utf-8") + ledger_path = cli_skill_dir / "reports" / "adaptation_approval_ledger.json" + prepare_proc = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(patch_path), + "--generated-at", + "2026-06-15T00:00:00Z", + "--today", + "2026-06-15", + "--prepare-approval", + ) + assert prepare_proc.returncode == 0, prepare_proc.stdout + prepare_payload = json.loads(prepare_proc.stdout) + assert_report_contract_mirrors(prepare_payload, "apply_contract") + assert prepare_payload["summary"]["approval_draft_count"] == 1, prepare_payload + assert prepare_payload["approval_draft_count"] == 1, prepare_payload + assert prepare_payload["approval_draft"]["decision"] == "pending-review", prepare_payload + assert prepare_payload["approval_draft"]["patch_sha256"] == sha256_text(patch_text), prepare_payload + assert prepare_payload["approval_draft"]["target_file_sha256"] == { + "references/user-memory-policy.md": sha256_text("old policy\n"), + }, prepare_payload + approval_ledger = json.loads(ledger_path.read_text(encoding="utf-8")) + assert_report_contract_mirrors(approval_ledger, "approval_contract") + assert approval_ledger["summary"]["pending_review_count"] == 1, approval_ledger + assert approval_ledger["pending_review_count"] == 1, approval_ledger + approval_ledger["entries"][0].update( + { + "decision": "approved", + "reviewer": "qa-reviewer", + "reason": "Fixture validates approval-gated adaptive apply.", + "approved_at": "2026-06-15", + "expires_at": "2026-12-31", + "verification_commands": ["python3 tests/check_policy.py"], + "rollback_plan": "git apply -R approved-adaptation.patch", + } + ) + ledger_path.write_text(json.dumps(approval_ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + policy_path.write_text("changed outside approval\n", encoding="utf-8") + stale_proc = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(patch_path), + "--output-json", + str(cli_skill_dir / "reports" / "stale_baseline_regression.json"), + "--output-md", + str(cli_skill_dir / "reports" / "stale_baseline_regression.md"), + "--generated-at", + "2026-06-15T00:00:00Z", + "--today", + "2026-06-15", + ) + assert stale_proc.returncode == 2, stale_proc.stdout + stale_payload = json.loads(stale_proc.stdout) + assert any("baseline sha256" in item for item in stale_payload["failures"]), stale_payload + assert "approved adaptive note" not in policy_path.read_text(encoding="utf-8"), policy_path + policy_path.write_text("old policy\n", encoding="utf-8") + dry_run = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(patch_path), + "--generated-at", + "2026-06-15T00:00:00Z", + "--today", + "2026-06-15", + ) + assert dry_run.returncode == 0, dry_run.stderr + dry_payload = json.loads(dry_run.stdout) + assert_report_contract_mirrors(dry_payload, "apply_contract") + assert dry_payload["summary"]["dry_run_count"] == 1, dry_payload + assert dry_payload["dry_run_count"] == 1, dry_payload + assert dry_payload["summary"]["applied_count"] == 0, dry_payload + refreshed_ledger = json.loads(ledger_path.read_text(encoding="utf-8")) + assert_report_contract_mirrors(refreshed_ledger, "approval_contract") + assert refreshed_ledger["summary"]["approval_count"] == 1, refreshed_ledger + assert refreshed_ledger["approval_count"] == 1, refreshed_ledger + assert refreshed_ledger["summary"]["pending_review_count"] == 0, refreshed_ledger + assert "approved adaptive note" not in policy_path.read_text(encoding="utf-8"), policy_path + + apply_proc = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(patch_path), + "--generated-at", + "2026-06-15T00:00:00Z", + "--today", + "2026-06-15", + "--apply", + "--run-verification", + ) + assert apply_proc.returncode == 0, apply_proc.stderr + apply_payload = json.loads(apply_proc.stdout) + assert_report_contract_mirrors(apply_payload, "apply_contract") + assert apply_payload["summary"]["applied_count"] == 1, apply_payload + assert apply_payload["applied_count"] == 1, apply_payload + assert apply_payload["summary"]["regression_run_count"] == 1, apply_payload + assert apply_payload["summary"]["regression_pass_count"] == 1, apply_payload + assert "approved adaptive note" in policy_path.read_text(encoding="utf-8"), policy_path + + policy_path.write_text("old policy\n", encoding="utf-8") + failing_verifier = cli_skill_dir / "tests" / "failing_policy.py" + failing_verifier.write_text( + "from pathlib import Path\n" + "assert 'impossible verifier token' in Path('references/user-memory-policy.md').read_text(encoding='utf-8')\n", + encoding="utf-8", + ) + rollback_patch = ( + "diff --git a/references/user-memory-policy.md b/references/user-memory-policy.md\n" + "--- a/references/user-memory-policy.md\n" + "+++ b/references/user-memory-policy.md\n" + "@@ -1 +1,2 @@\n" + " old policy\n" + "+rolled back adaptive note\n" + ) + rollback_patch_path = TMP / "rollback-adaptation.patch" + rollback_patch_path.write_text(rollback_patch, encoding="utf-8") + approval_ledger["entries"][0]["patch_sha256"] = sha256_text(rollback_patch) + approval_ledger["entries"][0]["verification_commands"] = ["python3 tests/failing_policy.py"] + ledger_path.write_text(json.dumps(approval_ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + rollback_proc = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(rollback_patch_path), + "--output-json", + str(cli_skill_dir / "reports" / "rollback_adaptation_regression.json"), + "--output-md", + str(cli_skill_dir / "reports" / "rollback_adaptation_regression.md"), + "--today", + "2026-06-15", + "--apply", + "--run-verification", + ) + assert rollback_proc.returncode == 2, rollback_proc.stdout + rollback_payload = json.loads(rollback_proc.stdout) + assert_report_contract_mirrors(rollback_payload, "apply_contract") + assert rollback_payload["summary"]["applied_count"] == 0, rollback_payload + assert rollback_payload["rollback_count"] == 1, rollback_payload + assert rollback_payload["summary"]["rollback_count"] == 1, rollback_payload + assert rollback_payload["attempts"][0]["status"] == "failed-rolled-back", rollback_payload + assert rollback_payload["attempts"][0]["rollback_result"]["ok"] is True, rollback_payload + assert "rolled back adaptive note" not in policy_path.read_text(encoding="utf-8"), policy_path + assert policy_path.read_text(encoding="utf-8") == "old policy\n", policy_path + + unsafe_patch = ( + "diff --git a/README.md b/README.md\n" + "--- a/README.md\n" + "+++ b/README.md\n" + "@@ -1 +1,2 @@\n" + " old\n" + "+unsafe\n" + ) + unsafe_patch_path = TMP / "unsafe-adaptation.patch" + unsafe_patch_path.write_text(unsafe_patch, encoding="utf-8") + approval_ledger["entries"][0]["patch_sha256"] = sha256_text(unsafe_patch) + ledger_path.write_text(json.dumps(approval_ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + unsafe_proc = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(unsafe_patch_path), + "--output-json", + str(cli_skill_dir / "reports" / "unsafe_adaptation_regression.json"), + "--output-md", + str(cli_skill_dir / "reports" / "unsafe_adaptation_regression.md"), + "--today", + "2026-06-15", + ) + assert unsafe_proc.returncode == 2, unsafe_proc.stdout + unsafe_payload = json.loads(unsafe_proc.stdout) + assert_report_contract_mirrors(unsafe_payload, "apply_contract") + assert any("outside approval target_files" in item for item in unsafe_payload["failures"]), unsafe_payload + + proposal_report_path = cli_skill_dir / "reports" / "adaptation_proposals.json" + proposal_report = json.loads(proposal_report_path.read_text(encoding="utf-8")) + for proposal in proposal_report["proposals"]: + if proposal["proposal_id"] == approval_proposal["proposal_id"]: + proposal["target_files"].append("references/new-user-memory-policy.md") + proposal_report_path.write_text(json.dumps(proposal_report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + new_file_patch = ( + "diff --git a/references/new-user-memory-policy.md b/references/new-user-memory-policy.md\n" + "new file mode 100644\n" + "--- /dev/null\n" + "+++ b/references/new-user-memory-policy.md\n" + "@@ -0,0 +1 @@\n" + "+new approved policy\n" + ) + new_file_patch_path = TMP / "new-file-adaptation.patch" + new_file_patch_path.write_text(new_file_patch, encoding="utf-8") + new_file_prepare = run_script( + str(CLI), + "adapt-apply", + str(cli_skill_dir), + "--proposal-id", + approval_proposal["proposal_id"], + "--patch-file", + str(new_file_patch_path), + "--approval-ledger", + str(cli_skill_dir / "reports" / "new_file_approval_ledger.json"), + "--output-json", + str(cli_skill_dir / "reports" / "new_file_prepare_regression.json"), + "--output-md", + str(cli_skill_dir / "reports" / "new_file_prepare_regression.md"), + "--generated-at", + "2026-06-15T00:00:00Z", + "--today", + "2026-06-15", + "--prepare-approval", + ) + assert new_file_prepare.returncode == 0, new_file_prepare.stdout + new_file_payload = json.loads(new_file_prepare.stdout) + assert_report_contract_mirrors(new_file_payload, "apply_contract") + assert new_file_payload["approval_draft"]["target_file_sha256"] == { + "references/new-user-memory-policy.md": "__absent__", + }, new_file_payload + assert not (cli_skill_dir / "references" / "new-user-memory-policy.md").exists(), cli_skill_dir + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_adapter_snapshots.py b/tests/verify_adapter_snapshots.py index b390ba72..22f38d48 100644 --- a/tests/verify_adapter_snapshots.py +++ b/tests/verify_adapter_snapshots.py @@ -39,6 +39,8 @@ def main() -> None: "claude", "--platform", "generic", + "--platform", + "vscode", "--expectations", str(EXPECTATIONS), "--output-dir", @@ -54,7 +56,7 @@ def main() -> None: raise SystemExit(proc.returncode) failures = [] - for name in ("openai", "claude", "generic"): + for name in ("openai", "claude", "generic", "vscode"): snapshot = json.loads((SNAPSHOTS / f"{name}_adapter.json").read_text(encoding="utf-8")) adapter = json.loads((TMP / "targets" / name / "adapter.json").read_text(encoding="utf-8")) if adapter.get("platform") != snapshot["platform"]: @@ -152,6 +154,10 @@ def main() -> None: readme = (TMP / "targets" / "claude" / "README.md").read_text(encoding="utf-8") if "Native surface:" not in readme or "Activation:" not in readme: failures.append(f"{name}: generated README does not describe native behavior") + if name == "vscode": + readme = (TMP / "targets" / "vscode" / "README.md").read_text(encoding="utf-8") + if "VS Code / Copilot Agent Skills Package" not in readme or "workspace trust" not in readme: + failures.append(f"{name}: generated README does not describe VS Code install and trust behavior") report = {"ok": not failures, "failures": failures} print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/tests/verify_adoption_drift.py b/tests/verify_adoption_drift.py index 6010be02..03bdf2f8 100644 --- a/tests/verify_adoption_drift.py +++ b/tests/verify_adoption_drift.py @@ -103,22 +103,30 @@ def main() -> None: "failed", "--failure-type", "script_error", + "--source", + "yao_cli", + "--command", + "validate", "--timestamp", "2026-06-13T10:04:00Z", ) assert final["ok"], final summary = final["payload"]["summary"] assert summary["event_count"] == 5, summary - assert summary["adoption_rate"] == 40.0, summary + assert summary["adoption_sample_count"] == 4, summary + assert summary["adoption_rate"] == 50.0, summary assert summary["missed_trigger_count"] == 2, summary assert summary["bad_output_count"] == 1, summary assert summary["script_error_count"] == 1, summary assert summary["risk_band"] == "high", summary + assert summary["source_types"]["yao_cli"] == 1, summary + assert summary["command_counts"]["validate"] == 1, summary assert final["payload"]["privacy_contract"]["raw_content_allowed"] is False, final assert final["payload"]["next_iteration_candidates"], final markdown = (skill_dir / "reports" / "adoption_drift_report.md").read_text(encoding="utf-8") assert "metadata-only telemetry" in markdown, markdown assert "Raw user prompts" in markdown, markdown + assert "source=`yao_cli` command=`validate`" in markdown, markdown unsafe_events = TMP / "unsafe_events.jsonl" unsafe_events.write_text( diff --git a/tests/verify_architecture_maintainability.py b/tests/verify_architecture_maintainability.py new file mode 100644 index 00000000..a8f96f25 --- /dev/null +++ b/tests/verify_architecture_maintainability.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_architecture_maintainability.py" + + +def main() -> None: + tmp = ROOT / "tests" / "tmp_architecture_maintainability" + if tmp.exists(): + shutil.rmtree(tmp) + tmp.mkdir(parents=True, exist_ok=True) + + output_json = tmp / "architecture_maintainability.json" + output_md = tmp / "architecture_maintainability.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-14", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["summary"]["decision"] == "pass", payload["summary"] + assert payload["summary"]["early_watch_line_threshold"] == 600, payload["summary"] + assert payload["summary"]["watch_line_threshold"] == 720, payload["summary"] + assert payload["summary"]["early_watchlist_count"] >= 2, payload["summary"] + assert payload["summary"]["watchlist_count"] == 0, payload["summary"] + assert payload["summary"]["hotspot_count"] == 0, payload["summary"] + assert payload["summary"]["blocker_count"] == 0, payload["summary"] + assert payload["summary"]["command_handler_count"] >= 60, payload["summary"] + assert payload["summary"]["entrypoint_command_handler_count"] < 30, payload["summary"] + assert payload["summary"]["command_module_count"] >= 5, payload["summary"] + assert payload["summary"]["largest_file_lines"] < 900, payload["summary"] + assert all(item["severity"] == "pass" for item in payload["largest_files"]), payload["largest_files"] + assert payload["watchlist"] == [], payload["watchlist"] + assert payload["early_watchlist"], payload["early_watchlist"] + assert all(item["early_watch"] is True for item in payload["early_watchlist"]), payload["early_watchlist"] + early_watch_paths = {item["path"] for item in payload["early_watchlist"]} + assert "scripts/render_review_viewer.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/render_skill_os2_coverage.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/render_reference_synthesis.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/build_skill_atlas.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/skill_report_model.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/render_review_studio.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/review_studio_gates.py" not in early_watch_paths, payload["early_watchlist"] + assert "scripts/cross_packager.py" not in early_watch_paths, payload["early_watchlist"] + renderer_lines = len((ROOT / "scripts" / "render_review_studio.py").read_text(encoding="utf-8").splitlines()) + action_module = (ROOT / "scripts" / "review_studio_actions.py").read_text(encoding="utf-8") + action_lines = len(action_module.splitlines()) + assert renderer_lines < 600, renderer_lines + assert action_lines < 450, action_lines + assert 'SCRIPT_INTERFACE = "internal-module"' in action_module, action_module[:400] + hotspot_paths = {item["path"] for item in payload["hotspots"]} + assert "scripts/yao.py" not in hotspot_paths, hotspot_paths + assert "scripts/render_review_studio.py" not in hotspot_paths, hotspot_paths + assert "scripts/review_studio_actions.py" not in hotspot_paths, hotspot_paths + assert "scripts/render_review_viewer.py" not in hotspot_paths, hotspot_paths + assert output_json.exists(), output_json + markdown = output_md.read_text(encoding="utf-8") + assert "# Architecture Maintainability" in markdown, markdown + assert "No file-size hotspots found." in markdown, markdown + assert "No near-threshold files found." in markdown, markdown + assert "## Early Watchlist" in markdown, markdown + assert f"- early watchlist: `{payload['summary']['early_watchlist_count']}`" in markdown, markdown + early_watch_markdown = markdown.split("## Early Watchlist", 1)[1].split("## Largest Files", 1)[0] + assert "scripts/render_review_viewer.py" not in markdown, markdown + assert "scripts/render_skill_os2_coverage.py" not in markdown, markdown + assert "scripts/render_reference_synthesis.py" not in markdown, markdown + assert "scripts/build_skill_atlas.py" not in early_watch_markdown, early_watch_markdown + assert "scripts/skill_report_model.py" not in early_watch_markdown, early_watch_markdown + assert "scripts/render_review_studio.py" not in early_watch_markdown, early_watch_markdown + assert "scripts/review_studio_gates.py" not in early_watch_markdown, early_watch_markdown + assert "scripts/cross_packager.py" not in early_watch_markdown, early_watch_markdown + assert "Do not split a file only for line count" in markdown, markdown + + blocker_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(tmp / "architecture_blocker.json"), + "--output-md", + str(tmp / "architecture_blocker.md"), + "--warn-lines", + "10", + "--block-lines", + "100", + "--generated-at", + "2026-06-14", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert blocker_proc.returncode == 2, blocker_proc.stdout + blocker_payload = json.loads(blocker_proc.stdout) + assert blocker_payload["ok"] is False, blocker_payload + assert blocker_payload["summary"]["blocker_count"] > 0, blocker_payload["summary"] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_benchmark_reproducibility.py b/tests/verify_benchmark_reproducibility.py new file mode 100644 index 00000000..ce97343a --- /dev/null +++ b/tests/verify_benchmark_reproducibility.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_benchmark_reproducibility.py" +TMP = ROOT / "tests" / "tmp_benchmark_reproducibility" +sys.path.insert(0, str(ROOT / "scripts")) +from evidence_consistency_core import beta_public_claim_split_values # noqa: E402 +from render_benchmark_reproducibility import beta_deferred_evidence # noqa: E402 + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_evidence_ledger.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_evidence_intake.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_preflight.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + submission_review_proc = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_submission_review.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + assert submission_review_proc.returncode in {0, 2}, submission_review_proc.stderr + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_operator_runbook.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_claim_guard.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "python_compat_check.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + output_json = TMP / "benchmark_reproducibility.json" + output_md = TMP / "benchmark_reproducibility.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["reproducibility_ready"] is True, payload + assert payload["summary"]["release_lock_ready"] == (payload["git_status"]["source_dirty"] is False), payload + assert payload["summary"]["methodology_complete"] is True, payload + assert payload["summary"]["missing_artifact_count"] == 0, payload + assert len(payload["summary"]["evidence_bundle_sha256"]) == 64, payload + assert payload["summary"]["evidence_bundle_sha256"] == payload["evidence_bundle"]["sha256"], payload + assert payload["summary"]["source_contract_sha256"], payload + assert payload["summary"]["archive_sha256"], payload + assert payload["summary"]["output_case_count"] >= 5, payload + assert payload["summary"]["failure_disclosure_count"] >= 1, payload + assert payload["summary"]["command_count"] >= 10, payload + assert "working_tree_dirty" in payload["summary"], payload + assert "source_tree_dirty" in payload["summary"], payload + assert "generated_tree_dirty" in payload["summary"], payload + assert payload["summary"]["source_changed_file_count"] == payload["git_status"]["source_changed_file_count"], payload + assert payload["summary"]["generated_changed_file_count"] == payload["git_status"][ + "generated_changed_file_count" + ], payload + assert payload["git_status"]["available"] is True, payload + assert payload["git_status"]["scope"] == "generation-time status before this report is written", payload + assert isinstance(payload["git_status"]["generated_dirty_prefixes"], list), payload + assert payload["release_lock"]["status_scope"] == "generation-time status before this report is written", payload + assert payload["release_lock"]["commit"] == payload["commit"], payload + assert payload["release_lock"]["source_changed_file_count"] == payload["git_status"][ + "source_changed_file_count" + ], payload + assert payload["release_lock"]["generated_changed_file_count"] == payload["git_status"][ + "generated_changed_file_count" + ], payload + assert payload["evidence_bundle"]["algorithm"] == "sha256(path,label,exists,artifact_sha256)", payload + assert payload["evidence_bundle"]["artifact_count"] == payload["summary"]["required_artifact_count"], payload + assert payload["evidence_bundle"]["existing_count"] == payload["summary"]["required_artifact_count"], payload + assert payload["evidence_bundle"]["missing_count"] == 0, payload + assert payload["evidence_bundle"]["missing_paths"] == [], payload + assert payload["summary"]["provider_evidence_complete"] is True, payload + assert payload["summary"]["human_review_complete"] is False, payload + assert payload["summary"]["world_class_ready"] is False, payload + assert payload["summary"]["world_class_source_check_count"] >= 13, payload + assert payload["summary"]["world_class_source_pass_count"] >= 6, payload + assert payload["summary"]["world_class_source_blocked_count"] >= 6, payload + assert ( + payload["summary"]["world_class_source_pass_count"] + + payload["summary"]["world_class_source_blocked_count"] + == payload["summary"]["world_class_source_check_count"] + ), payload + expected_beta_ready = ( + payload["summary"]["reproducibility_ready"] + and payload["summary"]["release_lock_ready"] + and payload["summary"]["provider_evidence_complete"] + ) + assert payload["summary"]["beta_test_ready"] is expected_beta_ready, payload + assert payload["summary"]["beta_test_blocker_count"] == len(payload["beta_test_release"]["blockers"]), payload + assert payload["summary"]["beta_test_deferred_evidence_count"] == len( + payload["beta_test_release"]["allowed_deferred_evidence"] + ), payload + assert payload["beta_test_release"]["ready"] is expected_beta_ready, payload["beta_test_release"] + assert "beta/public test release" in payload["beta_test_release"]["scope"], payload["beta_test_release"] + assert "Human blind-review" in payload["beta_test_release"]["policy"], payload["beta_test_release"] + assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[ + "beta_test_release" + ] + ledger_payload = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8")) + pending_ledger_keys = { + item["key"] for item in ledger_payload["entries"] if item.get("status") == "pending" + } + deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]} + assert deferred_keys == pending_ledger_keys, payload["beta_test_release"] + assert "human-adjudication" in deferred_keys, payload["beta_test_release"] + assert "provider-holdout" in deferred_keys, payload["beta_test_release"] + assert "provider-ledger-review" not in deferred_keys, payload["beta_test_release"] + assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[ + "beta_test_release" + ] + synthetic_deferred = beta_deferred_evidence( + { + "entries": [ + {"key": "provider-holdout", "label": "Provider Holdout", "status": "accepted"}, + {"key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "pending"}, + ] + } + ) + assert {item["key"] for item in synthetic_deferred} == {"native-client-telemetry"}, synthetic_deferred + expected_beta_split, actual_beta_split = beta_public_claim_split_values( + { + "summary": { + "reproducibility_ready": True, + "release_lock_ready": True, + "provider_evidence_complete": True, + "beta_test_ready": True, + "public_claim_ready": False, + "human_review_complete": True, + }, + "beta_test_release": { + "ready": True, + "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "allowed_deferred_evidence": [{"key": "native-client-telemetry", "reason": "still pending"}], + }, + }, + { + "summary": {"ready_to_claim_world_class": False}, + "entries": [ + {"key": "human-adjudication", "status": "accepted"}, + {"key": "native-client-telemetry", "status": "pending"}, + ], + }, + {"data": {"output_review_adjudication": {"summary": {"pair_count": 5, "pending_count": 0}}}}, + ) + assert expected_beta_split == actual_beta_split, (expected_beta_split, actual_beta_split) + assert expected_beta_split["human_review_complete"] is True, expected_beta_split + assert expected_beta_split["deferred_human_review"] is False, expected_beta_split + assert payload["summary"]["public_claim_ready"] is False, payload + minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4 + assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload + assert payload["public_claim"]["ready"] is False, payload["public_claim"] + if payload["summary"]["release_lock_ready"]: + assert not any("release lock" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] + else: + assert any("release lock" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] + assert not any("provider-backed model holdout evidence is incomplete" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] + assert any("human blind-review" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] + assert any("world-class evidence" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] + assert any("world-class source checks" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] + assert "complete source checks" in payload["public_claim"]["policy"], payload["public_claim"] + headings = {item["heading"]: item["exists"] for item in payload["methodology"]["sections"]} + assert headings["## Benchmark Types"], headings + assert headings["## Failure Disclosure"], headings + artifacts = {item["path"]: item for item in payload["artifacts_checked"]} + assert artifacts["reports/benchmark_methodology.md"]["exists"], artifacts + assert artifacts["evals/failure-cases.md"]["exists"], artifacts + assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts + assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts + assert artifacts["reports/world_class_evidence_intake.json"]["exists"], artifacts + assert artifacts["reports/world_class_evidence_preflight.json"]["exists"], artifacts + assert artifacts["reports/world_class_submission_review.json"]["exists"], artifacts + assert artifacts["reports/world_class_operator_runbook.json"]["exists"], artifacts + assert artifacts["reports/world_class_operator_runbook.md"]["exists"], artifacts + assert artifacts["reports/world_class_operator_runbook.html"]["exists"], artifacts + assert artifacts["reports/world_class_claim_guard.json"]["exists"], artifacts + assert artifacts["reports/python_compatibility.json"]["exists"], artifacts + assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-claim-guard ." for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py python-compat ." for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py evidence-consistency ." for command in payload["reproduction_commands"]), payload + assert any("Provider-backed model holdout source evidence is complete" in item for item in payload["limitations"]), payload["limitations"] + markdown = output_md.read_text(encoding="utf-8") + assert "Benchmark Reproducibility" in markdown, markdown + assert "Evidence bundle SHA256" in markdown, markdown + assert "release lock ready" in markdown, markdown + assert "Source tree dirty at generation" in markdown, markdown + assert "Generated evidence dirty at generation" in markdown, markdown + assert "source changed files at generation" in markdown, markdown + assert "generated changed files at generation" in markdown, markdown + assert "world-class source checks" in markdown, markdown + assert "beta test ready" in markdown, markdown + assert "## Beta Test Boundary" in markdown, markdown + assert "human-adjudication" in markdown, markdown + assert "do not claim world-class" in markdown, markdown + assert "public claim ready: `false`" in markdown, markdown + assert "## Public Claim Boundary" in markdown, markdown + assert "provider-backed model holdout evidence is incomplete" not in markdown, markdown + assert "Provider-backed model holdout source evidence is complete" in markdown, markdown + assert "world-class source checks are not all accepted" in markdown, markdown + assert "## Release Lock" in markdown, markdown + assert "## Evidence Bundle" in markdown, markdown + assert "reports/benchmark_methodology.md" in markdown, markdown + assert "reports/world_class_evidence_preflight.json" in markdown, markdown + assert "reports/world_class_operator_runbook.html" in markdown, markdown + assert "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" in markdown, markdown + assert "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" in markdown, markdown + assert "python3 scripts/yao.py evidence-consistency ." in markdown, markdown + assert "make ci-test" in markdown, markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_compile_skill.py b/tests/verify_compile_skill.py index be5d6c50..6c3a0c25 100644 --- a/tests/verify_compile_skill.py +++ b/tests/verify_compile_skill.py @@ -44,6 +44,8 @@ def main() -> None: "claude", "--target", "generic", + "--target", + "vscode", "--output-json", str(output_json), "--output-md", @@ -54,13 +56,13 @@ def main() -> None: payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["schema_version"] == "1.0", payload - assert payload["summary"]["target_count"] == 3, payload - assert payload["summary"]["pass_count"] == 3, payload + assert payload["summary"]["target_count"] == 4, payload + assert payload["summary"]["pass_count"] == 4, payload assert output_json.exists(), output_json assert output_md.exists(), output_md assert "Compiled Targets" in output_md.read_text(encoding="utf-8") by_target = {item["target"]: item for item in payload["targets"]} - for target in ("openai", "claude", "generic"): + for target in ("openai", "claude", "generic", "vscode"): item = by_target[target] assert item["compiler"]["name"] == "yao-skill-ir-compiler", item assert item["compiler"]["source"] == "skill-ir", item diff --git a/tests/verify_conformance_suite.py b/tests/verify_conformance_suite.py index 5039ee5a..b180b162 100644 --- a/tests/verify_conformance_suite.py +++ b/tests/verify_conformance_suite.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import json +import shutil import subprocess import sys from pathlib import Path @@ -52,6 +53,35 @@ def main() -> None: assert output_md.exists(), output_md assert "Runtime Conformance Matrix" in output_md.read_text(encoding="utf-8") + renamed_workspace = tmp_root / "renamed-conformance-root" + shutil.rmtree(renamed_workspace, ignore_errors=True) + renamed_root = renamed_workspace / "checkout-alias" + shutil.copytree( + ROOT, + renamed_root, + ignore=shutil.ignore_patterns(".git", ".previews", "dist", "__pycache__", ".pytest_cache", "tmp*"), + ) + renamed_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(renamed_root), + "--output-json", + str(tmp_root / "renamed_conformance_matrix.json"), + "--output-md", + str(tmp_root / "renamed_conformance_matrix.md"), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + renamed_payload = json.loads(renamed_proc.stdout) + assert renamed_payload["ok"], renamed_payload + assert renamed_payload["summary"]["pass_count"] == 5, renamed_payload + warnings = [warning for target in renamed_payload["targets"] for warning in target["warnings"]] + assert any("source checkout directory differs" in warning for warning in warnings), renamed_payload + broken = tmp_root / "broken-skill" (broken / "agents").mkdir(parents=True, exist_ok=True) (broken / "reports").mkdir(parents=True, exist_ok=True) diff --git a/tests/verify_daily_skillops.py b/tests/verify_daily_skillops.py new file mode 100644 index 00000000..8c39851a --- /dev/null +++ b/tests/verify_daily_skillops.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_daily_skillops_report.py" +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_daily_skillops" + + +def run_command(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *args], + cwd=ROOT, + capture_output=True, + text=True, + ) + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def seed_release_reports(skill_dir: Path) -> None: + reports = skill_dir / "reports" + write_json( + reports / "skill_os2_coverage.json", + { + "ok": True, + "summary": { + "local_blueprint_ready": True, + "public_world_class_ready": False, + "world_class_evidence_pending_count": 4, + }, + }, + ) + write_json( + reports / "world_class_evidence_ledger.json", + { + "ok": True, + "summary": { + "pending_count": 4, + "accepted_count": 0, + "ready_to_claim_world_class": False, + }, + }, + ) + write_json(reports / "evidence_consistency.json", {"ok": True, "checks": [{"status": "pass"}]}) + write_json(reports / "benchmark_reproducibility.json", {"ok": True, "summary": {"release_lock_ready": True}}) + write_json( + reports / "adaptation_approval_ledger.json", + { + "ok": True, + "summary": { + "approval_count": 1, + "pending_review_count": 1, + "applied_count": 0, + "rollback_count": 0, + }, + }, + ) + write_json( + reports / "adaptation_regression_report.json", + { + "ok": True, + "summary": { + "attempt_count": 1, + "dry_run_count": 1, + "applied_count": 0, + "rollback_count": 0, + }, + }, + ) + + +def write_signal_source(path: Path) -> None: + path.write_text( + "\n".join( + [ + json.dumps({"text": "报告默认中文简体，同时右上角提供英文切换；敏感内容 token=abc123456789、sk-1234567890abcdef 和 /Users/laoyao/private/path 必须脱敏。"}, ensure_ascii=False), + json.dumps({"message": "新的 HTML 报告要双语，但默认中文简体。"}, ensure_ascii=False), + json.dumps({"content": "报告 UI 需要 Kami 白底排版、图表模块和清晰导航。"}, ensure_ascii=False), + json.dumps({"excerpt": "HTML 报告还是白底 Kami 风格，图表不要挤在一起。"}, ensure_ascii=False), + json.dumps({"note": "不要自动扫描私人日志；必须由用户提供明确路径。"}, ensure_ascii=False), + json.dumps({"body": "自适应升级需要先输出提案，授权后再修改，并能回滚。"}, ensure_ascii=False), + json.dumps({"text": "每次升级都要有测试、证据报告和 GitHub push。"}, ensure_ascii=False), + json.dumps({"text": "敏感内容也不能直接进入公开日报。"}, ensure_ascii=False), + ] + ) + + "\n", + encoding="utf-8", + ) + + +def assert_daily_contract(payload: dict) -> None: + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["report_contract"]["contract"] == "daily-skillops-report", payload + assert payload["report_contract"]["top_level_mirrors_summary"] is True, payload + for field in payload["report_contract"]["summary_fields"]: + assert payload[field] == payload["summary"][field], (field, payload.get(field), payload["summary"].get(field)) + contract = payload["operations_contract"] + assert contract["explicit_source_required_for_scan"] is True, payload + assert contract["implicit_private_log_scan"] is False, payload + assert contract["raw_content_stored"] is False, payload + assert contract["redacted_excerpts_only"] is True, payload + assert contract["proposal_only"] is True, payload + assert contract["approval_required_for_writes"] is True, payload + assert contract["writes_source_files"] is False, payload + assert contract["auto_patch_enabled"] is False, payload + assert contract["daily_report_counts_as_world_class_evidence"] is False, payload + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + skill_dir = TMP / "daily-skillops-demo" + skill_dir.mkdir(parents=True) + seed_release_reports(skill_dir) + source = TMP / "curated-signals.jsonl" + write_signal_source(source) + source_before = source.read_text(encoding="utf-8") + + output_json = skill_dir / "reports" / "skillops" / "daily" / "2026-06-16.json" + output_md = skill_dir / "reports" / "skillops" / "daily" / "2026-06-16.md" + proc = run_command( + str(SCRIPT), + str(skill_dir), + "--source", + str(source), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-16T08:00:00Z", + ) + assert proc.returncode == 0, proc.stderr + payload = json.loads(proc.stdout) + assert_daily_contract(payload) + assert payload["summary"]["source_supplied"] is True, payload + assert payload["summary"]["pattern_count"] >= 4, payload + assert payload["summary"]["proposal_count"] >= 4, payload + assert payload["summary"]["pending_review_count"] == 1, payload + assert payload["summary"]["world_class_pending_count"] == 4, payload + assert payload["summary"]["public_world_class_ready"] is False, payload + assert payload["summary"]["release_lock_ready"] is True, payload + assert payload["summary"]["evidence_consistency_ok"] is True, payload + assert payload["opportunity_summary"]["opportunity_count"] >= 4, payload + assert payload["opportunity_summary"]["top_score"] >= 70, payload + assert payload["decision_policy"]["writes_source_files"] is False, payload + assert payload["decision_policy"]["auto_patch_enabled"] is False, payload + action_types = {item["action_type"] for item in payload["opportunities"]} + assert {"patch_existing_skill", "agents_update"} <= action_types, payload["opportunities"] + assert action_types <= set(payload["decision_policy"]["action_types"]), payload["opportunities"] + assert all(item["write_allowed_without_approval"] is False for item in payload["opportunities"]), payload[ + "opportunities" + ] + action_keys = {item["key"] for item in payload["actions"]} + assert {"review-adaptation-proposals", "resolve-pending-approvals", "close-world-class-evidence"} <= action_keys, payload + assert output_json.exists(), output_json + assert output_md.exists(), output_md + assert (skill_dir / "reports" / "user_patterns.json").exists(), skill_dir + assert (skill_dir / "reports" / "adaptation_proposals.json").exists(), skill_dir + assert source.read_text(encoding="utf-8") == source_before, source + + serialized = json.dumps(payload, ensure_ascii=False) + assert "sk-1234567890abcdef" not in serialized, serialized + assert "token=abc123456789" not in serialized, serialized + assert "/Users/laoyao/private" not in serialized, serialized + assert "[REDACTED_SECRET]" in serialized, serialized + assert "[LOCAL_PATH]" in serialized, serialized + markdown = output_md.read_text(encoding="utf-8") + assert "Daily SkillOps Report" in markdown, markdown + assert "does not scan private logs" in markdown, markdown + assert "daily_report_counts_as_world_class_evidence: `false`" in markdown, markdown + assert "## Opportunities" in markdown, markdown + assert "ready for approval review" in markdown, markdown + + cli_dir = TMP / "cli-daily-skillops-demo" + cli_dir.mkdir(parents=True) + seed_release_reports(cli_dir) + cli_output_json = cli_dir / "reports" / "skillops" / "daily" / "cli.json" + cli_output_md = cli_dir / "reports" / "skillops" / "daily" / "cli.md" + cli_proc = run_command( + str(CLI), + "daily-skillops", + str(cli_dir), + "--source", + str(source), + "--output-json", + str(cli_output_json), + "--output-md", + str(cli_output_md), + "--generated-at", + "2026-06-16T08:00:00Z", + ) + assert cli_proc.returncode == 0, cli_proc.stderr + cli_payload = json.loads(cli_proc.stdout) + assert_daily_contract(cli_payload) + assert cli_payload["summary"]["proposal_count"] >= 4, cli_payload + assert cli_output_json.exists(), cli_output_json + assert cli_output_md.exists(), cli_output_md + + history_source = TMP / ".zsh_history" + history_source.write_text("报告默认中文\n报告默认中文\n", encoding="utf-8") + history_proc = run_command(str(SCRIPT), str(skill_dir), "--source", str(history_source)) + assert history_proc.returncode == 2, history_proc.stdout + history_payload = json.loads(history_proc.stdout) + assert history_payload["ok"] is False, history_payload + assert any("Refusing private history source" in item for item in history_payload["failures"]), history_payload + + no_refresh_dir = TMP / "no-refresh-daily-skillops-demo" + no_refresh_dir.mkdir(parents=True) + seed_release_reports(no_refresh_dir) + write_json( + no_refresh_dir / "reports" / "user_patterns.json", + {"ok": True, "summary": {"pattern_count": 0}, "patterns": []}, + ) + no_refresh_output_json = no_refresh_dir / "reports" / "skillops" / "daily" / "no-refresh.json" + no_refresh_proc = run_command( + str(SCRIPT), + str(no_refresh_dir), + "--source", + str(source), + "--no-refresh-source-reports", + "--output-json", + str(no_refresh_output_json), + "--generated-at", + "2026-06-16T08:00:00Z", + ) + assert no_refresh_proc.returncode == 0, no_refresh_proc.stderr + no_refresh_payload = json.loads(no_refresh_proc.stdout) + assert_daily_contract(no_refresh_payload) + assert no_refresh_payload["summary"]["source_supplied"] is True, no_refresh_payload + assert no_refresh_payload["summary"]["pattern_count"] >= 4, no_refresh_payload + assert no_refresh_payload["summary"]["proposal_count"] >= 4, no_refresh_payload + assert no_refresh_payload["opportunity_summary"]["opportunity_count"] >= 4, no_refresh_payload + assert not (no_refresh_dir / "reports" / "adaptation_proposals.json").exists(), no_refresh_dir + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py new file mode 100644 index 00000000..d4209600 --- /dev/null +++ b/tests/verify_evidence_consistency.py @@ -0,0 +1,719 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) +from evidence_consistency_release import CLEAN_LOCK_REPORT_COMMANDS, SOURCE_REFRESH_REPORT_COMMANDS + +SCRIPT = ROOT / "scripts" / "render_evidence_consistency.py" +TMP = ROOT / "tests" / "tmp_evidence_consistency" +REPORT_FILES = [ + "AGENTS.md", + "reports/benchmark_reproducibility.json", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/skill-interpretation.json", + "reports/skill-interpretation.html", + "reports/adoption_drift_report.json", + "reports/world_class_evidence_ledger.json", + "reports/world_class_evidence_plan.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_preflight.json", + "reports/world_class_evidence_preflight.html", + "reports/world_class_submission_review.json", + "reports/world_class_operator_runbook.json", + "reports/skill_os2_coverage.json", + "reports/review-studio.json", + "reports/package_verification.json", + "reports/install_simulation.json", + "reports/security_trust_report.json", + "reports/context_budget.json", + "reports/world_class_claim_guard.json", + "reports/skill-os-2-review.md", + "skill-ir/examples/yao-meta-skill.json", + "scripts/ci_test.py", +] + + +def run(cmd: list[str], *, check: bool = False) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=check) + + +def copy_reports(dst: Path) -> None: + for relative in REPORT_FILES: + source = ROOT / relative + target = dst / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + + +def refresh_embedded_reports() -> None: + script_names = [ + "render_context_reports.py", + "render_skill_os2_coverage.py", + "render_world_class_evidence_plan.py", + "render_world_class_evidence_ledger.py", + "render_world_class_evidence_intake.py", + "render_world_class_preflight.py", + "render_world_class_submission_review.py", + "render_world_class_operator_runbook.py", + "render_world_class_claim_guard.py", + "render_benchmark_reproducibility.py", + "render_skill_overview.py", + "render_skill_interpretation.py", + "render_review_viewer.py", + "render_review_studio.py", + ] + allow_nonzero = { + "render_world_class_evidence_intake.py", + "render_world_class_submission_review.py", + } + for script_name in script_names: + command = [sys.executable, str(ROOT / "scripts" / script_name)] + if script_name != "render_context_reports.py": + command.append(str(ROOT)) + proc = subprocess.run( + command, + cwd=ROOT, + capture_output=True, + text=True, + ) + if proc.returncode != 0 and script_name not in allow_nonzero: + raise subprocess.CalledProcessError(proc.returncode, command, proc.stdout, proc.stderr) + + +def assert_world_class_roadmap_matches_ledger() -> None: + ledger = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8")) + summary = ledger["summary"] + pending_count = int(summary["pending_count"]) + external_pending_count = int(summary["external_pending_count"]) + human_pending_count = int(summary["human_pending_count"]) + expected_total = f"{pending_count} 项待补证据" + expected_breakdown = f"外部 {external_pending_count} 项、人工 {human_pending_count} 项" + + for report_name in ["skill-overview", "skill-interpretation"]: + report_path = ROOT / "reports" / f"{report_name}.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + serialized = json.dumps(report, ensure_ascii=False) + assert "继续补齐剩余 2 项外部/人工证据" not in serialized, report_name + assert report["world_class_readiness"]["pending_count"] == pending_count, report["world_class_readiness"] + assert report["world_class_readiness"]["external_pending_count"] == external_pending_count, report[ + "world_class_readiness" + ] + assert report["world_class_readiness"]["human_pending_count"] == human_pending_count, report[ + "world_class_readiness" + ] + actions = "\n".join(report["iteration_roadmap"]["items"][0]["actions"]) + if pending_count > 2: + assert expected_total in actions, actions + assert expected_breakdown in actions, actions + + +def assert_release_evidence_instructions_cover_first_class_reports() -> None: + agents_text = (ROOT / "AGENTS.md").read_text(encoding="utf-8") + source_refresh_header = "After source changes that affect scripts" + clean_lock_header = "For final release evidence" + assert source_refresh_header in agents_text, agents_text + assert clean_lock_header in agents_text, agents_text + + source_refresh = agents_text.split(source_refresh_header, 1)[1].split(clean_lock_header, 1)[0] + clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0] + for command in SOURCE_REFRESH_REPORT_COMMANDS: + assert command in source_refresh, command + for command in CLEAN_LOCK_REPORT_COMMANDS: + assert command in clean_lock, command + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + refresh_embedded_reports() + assert_world_class_roadmap_matches_ledger() + assert_release_evidence_instructions_cover_first_class_reports() + output_json = TMP / "evidence_consistency.json" + output_md = TMP / "evidence_consistency.md" + proc = run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-15", + ], + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["decision"] == "consistent", payload + assert payload["summary"]["fail_count"] == 0, payload + assert payload["summary"]["check_count"] >= 31, payload + checks = {item["key"]: item for item in payload["checks"]} + assert checks["release-evidence-flow-covers-first-class-reports"]["status"] == "pass", checks[ + "release-evidence-flow-covers-first-class-reports" + ] + assert checks["review-studio-context-budget-mirror"]["status"] == "pass", checks[ + "review-studio-context-budget-mirror" + ] + assert checks["skill-ir-evidence-path-contract"]["status"] == "pass", checks[ + "skill-ir-evidence-path-contract" + ] + assert checks["skill-ir-evidence-path-contract"]["actual"]["review_studio_evidence_path"] == ( + "skill-ir/examples/yao-meta-skill.json" + ), checks["skill-ir-evidence-path-contract"] + assert checks["review-studio-gate-action-mirror"]["status"] == "pass", checks[ + "review-studio-gate-action-mirror" + ] + gate_action_mirror = checks["review-studio-gate-action-mirror"]["actual"] + assert gate_action_mirror["non_pass_gate_keys"] == [ + "output-lab", + "review-waivers", + "world-class-evidence", + ], gate_action_mirror + assert gate_action_mirror["action_gate_keys"] == gate_action_mirror["non_pass_gate_keys"], gate_action_mirror + assert gate_action_mirror["pass_gate_action_ids_empty"] is True, gate_action_mirror + assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["source_ref_present"] is True, ( + gate_action_mirror + ) + assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["title_present"] is True, ( + gate_action_mirror + ) + assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["next_step_present"] is True, ( + gate_action_mirror + ) + assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["reason_present"] is True, ( + gate_action_mirror + ) + assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"] + assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] + assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"] + assert checks["benchmark-beta-public-claim-split"]["status"] == "pass", checks[ + "benchmark-beta-public-claim-split" + ] + beta_split = checks["benchmark-beta-public-claim-split"]["actual"] + assert beta_split["public_claim_ready"] is False, beta_split + assert beta_split["human_review_complete"] is False, beta_split + assert beta_split["deferred_evidence_keys"] == ["human-adjudication", "native-client-telemetry", "native-permission-enforcement", "provider-holdout"], beta_split + assert beta_split["deferred_human_review"] is True, beta_split + assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"] + assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[ + "preflight-submission-kit-handoff" + ] + preflight_handoff = checks["preflight-submission-kit-handoff"]["actual"] + assert preflight_handoff["artifact_role_source"] == "world-class-submission-kit", preflight_handoff + assert preflight_handoff["artifact_role_counts_as_evidence"] is False, preflight_handoff + assert preflight_handoff["submission_ref_role_present"] is True, preflight_handoff + assert preflight_handoff["supporting_evidence_role_present"] is True, preflight_handoff + assert preflight_handoff["submission_ref_copy_to_artifact_refs"] is True, preflight_handoff + assert preflight_handoff["supporting_evidence_copy_to_artifact_refs"] is False, preflight_handoff + assert checks["review-studio-preflight-artifact-role-handoff"]["status"] == "pass", checks[ + "review-studio-preflight-artifact-role-handoff" + ] + assert checks["world-class-phase-queue-consistency"]["status"] == "pass", checks[ + "world-class-phase-queue-consistency" + ] + phase_queue_actual = checks["world-class-phase-queue-consistency"]["actual"] + assert phase_queue_actual["summary"]["phase_queue_count"] == 2, phase_queue_actual + assert phase_queue_actual["summary"]["phase_queue_row_count"] >= 11, phase_queue_actual + assert phase_queue_actual["summary"] == phase_queue_actual["operator_runbook_summary"], phase_queue_actual + assert phase_queue_actual["top_level_phase_queue"] == phase_queue_actual[ + "operator_runbook_top_level_phase_queue" + ], phase_queue_actual + assert phase_queue_actual["phase_queue_counts_as_completion"] is False, phase_queue_actual + assert set(phase_queue_actual["item_phase_queues"]) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, phase_queue_actual + assert phase_queue_actual["operator_runbook_phase_queues"] == phase_queue_actual["item_phase_queues"], ( + phase_queue_actual + ) + assert phase_queue_actual["review_studio_phase_queues"] == phase_queue_actual["item_phase_queues"], ( + phase_queue_actual + ) + role_handoff = checks["review-studio-preflight-artifact-role-handoff"]["actual"] + assert role_handoff["provider-holdout"]["role_source"] == "world-class-submission-kit", role_handoff + assert role_handoff["provider-holdout"]["submission_ref_total_count"] == 1, role_handoff + assert role_handoff["provider-holdout"]["submission_ref_copy_to_artifact_refs"] is True, role_handoff + assert role_handoff["provider-holdout"]["supporting_evidence_copy_to_artifact_refs"] is False, role_handoff + assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"] + assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[ + "claim-guard-package-runtime-surface" + ] + assert checks["world-class-evidence-workflow-coverage"]["status"] == "pass", checks[ + "world-class-evidence-workflow-coverage" + ] + workflow_actual = checks["world-class-evidence-workflow-coverage"]["actual"] + assert workflow_actual["operator_coordination_keys"] == [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout", + ], workflow_actual + assert workflow_actual["operator_coordination_counts_as_completion"] is False, workflow_actual + assert workflow_actual["operator_release_gate_ready"] is False, workflow_actual + assert workflow_actual["operator_release_gate_counts_as_completion"] is False, workflow_actual + assert checks["skill-os-2-review-current-evidence"]["status"] == "pass", checks[ + "skill-os-2-review-current-evidence" + ] + markdown = output_md.read_text(encoding="utf-8") + assert "Evidence Consistency" in markdown, markdown + assert "decision: `consistent`" in markdown, markdown + assert "does not create provider, human, native-client, or permission-enforcement evidence" in markdown, markdown + + drift_root = TMP / "drift-skill" + copy_reports(drift_root) + overview_path = drift_root / "reports" / "skill-overview.json" + overview = json.loads(overview_path.read_text(encoding="utf-8")) + overview["adoption_drift"]["summary"]["adoption_sample_count"] += 1 + overview_path.write_text(json.dumps(overview, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(drift_root), + "--output-json", + str(TMP / "drift.json"), + "--output-md", + str(TMP / "drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert drift_proc.returncode == 2, drift_proc.stdout + drift_payload = json.loads(drift_proc.stdout) + drift_checks = {item["key"]: item for item in drift_payload["checks"]} + assert drift_payload["ok"] is False, drift_payload + assert drift_payload["summary"]["decision"] == "evidence-drift-detected", drift_payload + assert drift_checks["overview-adoption-summary"]["status"] == "fail", drift_checks["overview-adoption-summary"] + assert drift_checks["interpretation-adoption-summary"]["status"] == "pass", drift_checks["interpretation-adoption-summary"] + + skill_ir_drift_root = TMP / "skill-ir-drift-skill" + copy_reports(skill_ir_drift_root) + studio_path = skill_ir_drift_root / "reports" / "review-studio.json" + studio = json.loads(studio_path.read_text(encoding="utf-8")) + studio["evidence_paths"]["skill_ir"] = "reports/skill-ir.json" + studio_path.write_text(json.dumps(studio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + skill_ir_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(skill_ir_drift_root), + "--output-json", + str(TMP / "skill_ir_drift.json"), + "--output-md", + str(TMP / "skill_ir_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert skill_ir_drift_proc.returncode == 2, skill_ir_drift_proc.stdout + skill_ir_drift_payload = json.loads(skill_ir_drift_proc.stdout) + skill_ir_drift_checks = {item["key"]: item for item in skill_ir_drift_payload["checks"]} + assert skill_ir_drift_checks["skill-ir-evidence-path-contract"]["status"] == "fail", ( + skill_ir_drift_checks["skill-ir-evidence-path-contract"] + ) + + claim_guard_drift_root = TMP / "claim-guard-drift-skill" + copy_reports(claim_guard_drift_root) + claim_guard_path = claim_guard_drift_root / "reports" / "world_class_claim_guard.json" + claim_guard = json.loads(claim_guard_path.read_text(encoding="utf-8")) + claim_guard["summary"]["package_claim_surface_count"] = 0 + claim_guard["scanned_surfaces"] = [ + item for item in claim_guard["scanned_surfaces"] if item.get("path") != "dist/manifest.json" + ] + claim_guard_path.write_text(json.dumps(claim_guard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + claim_guard_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(claim_guard_drift_root), + "--output-json", + str(TMP / "claim_guard_drift.json"), + "--output-md", + str(TMP / "claim_guard_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert claim_guard_drift_proc.returncode == 2, claim_guard_drift_proc.stdout + claim_guard_drift_payload = json.loads(claim_guard_drift_proc.stdout) + claim_guard_drift_checks = {item["key"]: item for item in claim_guard_drift_payload["checks"]} + assert claim_guard_drift_checks["claim-guard-package-runtime-surface"]["status"] == "fail", ( + claim_guard_drift_checks["claim-guard-package-runtime-surface"] + ) + + preflight_drift_root = TMP / "preflight-drift-skill" + copy_reports(preflight_drift_root) + preflight_path = preflight_drift_root / "reports" / "world_class_evidence_preflight.json" + preflight = json.loads(preflight_path.read_text(encoding="utf-8")) + preflight["summary"]["credential_value_exposed"] = True + preflight_path.write_text(json.dumps(preflight, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + preflight_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(preflight_drift_root), + "--output-json", + str(TMP / "preflight_drift.json"), + "--output-md", + str(TMP / "preflight_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert preflight_drift_proc.returncode == 2, preflight_drift_proc.stdout + preflight_drift_payload = json.loads(preflight_drift_proc.stdout) + preflight_drift_checks = {item["key"]: item for item in preflight_drift_payload["checks"]} + assert preflight_drift_checks["preflight-world-class-boundary"]["status"] == "fail", ( + preflight_drift_checks["preflight-world-class-boundary"] + ) + + preflight_handoff_drift_root = TMP / "preflight-handoff-drift-skill" + copy_reports(preflight_handoff_drift_root) + preflight_handoff_path = preflight_handoff_drift_root / "reports" / "world_class_evidence_preflight.json" + preflight_handoff = json.loads(preflight_handoff_path.read_text(encoding="utf-8")) + preflight_handoff["submissions"]["drafts_count_as_evidence"] = True + preflight_handoff["submissions"]["commands"].pop("prepare_submission", None) + preflight_handoff_path.write_text(json.dumps(preflight_handoff, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + preflight_handoff_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(preflight_handoff_drift_root), + "--output-json", + str(TMP / "preflight_handoff_drift.json"), + "--output-md", + str(TMP / "preflight_handoff_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert preflight_handoff_drift_proc.returncode == 2, preflight_handoff_drift_proc.stdout + preflight_handoff_drift_payload = json.loads(preflight_handoff_drift_proc.stdout) + preflight_handoff_drift_checks = {item["key"]: item for item in preflight_handoff_drift_payload["checks"]} + assert preflight_handoff_drift_checks["preflight-submission-kit-handoff"]["status"] == "fail", ( + preflight_handoff_drift_checks["preflight-submission-kit-handoff"] + ) + + phase_queue_drift_root = TMP / "phase-queue-drift-skill" + copy_reports(phase_queue_drift_root) + phase_queue_path = phase_queue_drift_root / "reports" / "world_class_evidence_preflight.json" + phase_queue_payload = json.loads(phase_queue_path.read_text(encoding="utf-8")) + phase_queue_payload["summary"]["phase_queue_row_count"] = 1 + phase_queue_payload["phase_queue"][0]["counts_as_completion"] = True + phase_queue_payload["items"][0]["phase_queue"][0]["row_count"] = 999 + phase_queue_path.write_text(json.dumps(phase_queue_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + operator_phase_queue_path = phase_queue_drift_root / "reports" / "world_class_operator_runbook.json" + operator_phase_queue_payload = json.loads(operator_phase_queue_path.read_text(encoding="utf-8")) + operator_phase_queue_payload["summary"]["phase_queue_count"] = 99 + operator_phase_queue_payload["phase_queue"][0]["row_count"] = 999 + operator_phase_queue_payload["items"][0]["phase_queue"][0]["row_count"] = 999 + operator_phase_queue_path.write_text( + json.dumps(operator_phase_queue_payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + phase_queue_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(phase_queue_drift_root), + "--output-json", + str(TMP / "phase_queue_drift.json"), + "--output-md", + str(TMP / "phase_queue_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert phase_queue_drift_proc.returncode == 2, phase_queue_drift_proc.stdout + phase_queue_drift_payload = json.loads(phase_queue_drift_proc.stdout) + phase_queue_drift_checks = {item["key"]: item for item in phase_queue_drift_payload["checks"]} + assert phase_queue_drift_checks["world-class-phase-queue-consistency"]["status"] == "fail", ( + phase_queue_drift_checks["world-class-phase-queue-consistency"] + ) + + gate_action_drift_root = TMP / "gate-action-drift-skill" + copy_reports(gate_action_drift_root) + gate_action_path = gate_action_drift_root / "reports" / "review-studio.json" + gate_action_payload = json.loads(gate_action_path.read_text(encoding="utf-8")) + for gate in gate_action_payload["gates"]: + if gate["key"] == "world-class-evidence": + gate["review_action_id"] = "" + gate["review_action_title"] = "" + gate["review_action_next_step"] = "" + gate["review_action_reason"] = "" + gate["review_action_source_ref_count"] = 0 + for action in gate_action_payload["review_actions"]: + if action["gate_key"] == "world-class-evidence": + action["title"] = "" + action["next_step"] = "" + action["reason"] = "" + action["source_refs"] = [] + gate_action_path.write_text(json.dumps(gate_action_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + gate_action_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(gate_action_drift_root), + "--output-json", + str(TMP / "gate_action_drift.json"), + "--output-md", + str(TMP / "gate_action_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert gate_action_drift_proc.returncode == 2, gate_action_drift_proc.stdout + gate_action_drift_payload = json.loads(gate_action_drift_proc.stdout) + gate_action_drift_checks = {item["key"]: item for item in gate_action_drift_payload["checks"]} + assert gate_action_drift_checks["review-studio-gate-action-mirror"]["status"] == "fail", ( + gate_action_drift_checks["review-studio-gate-action-mirror"] + ) + + release_flow_drift_root = TMP / "release-flow-drift-skill" + copy_reports(release_flow_drift_root) + agents_path = release_flow_drift_root / "AGENTS.md" + agents_text = agents_path.read_text(encoding="utf-8") + agents_path.write_text( + agents_text.replace("python3 scripts/render_skill_interpretation.py .\n", "", 1), + encoding="utf-8", + ) + release_flow_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(release_flow_drift_root), + "--output-json", + str(TMP / "release_flow_drift.json"), + "--output-md", + str(TMP / "release_flow_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert release_flow_drift_proc.returncode == 2, release_flow_drift_proc.stdout + release_flow_drift_payload = json.loads(release_flow_drift_proc.stdout) + release_flow_drift_checks = {item["key"]: item for item in release_flow_drift_payload["checks"]} + assert release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]["status"] == "fail", ( + release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"] + ) + + context_drift_root = TMP / "context-drift-skill" + copy_reports(context_drift_root) + context_path = context_drift_root / "reports" / "context_budget.json" + context_payload = json.loads(context_path.read_text(encoding="utf-8")) + context_payload["warnings"] = ["Deferred resource footprint is high: stale test warning."] + context_payload["stats"]["deferred_resource_tokens"] = 1 + context_payload["stats"]["deferred_resource_governance"]["status"] = "pass" + context_path.write_text(json.dumps(context_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + context_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(context_drift_root), + "--output-json", + str(TMP / "context_drift.json"), + "--output-md", + str(TMP / "context_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert context_drift_proc.returncode == 2, context_drift_proc.stdout + context_drift_payload = json.loads(context_drift_proc.stdout) + context_drift_checks = {item["key"]: item for item in context_drift_payload["checks"]} + assert context_drift_checks["review-studio-context-budget-mirror"]["status"] == "fail", ( + context_drift_checks["review-studio-context-budget-mirror"] + ) + + benchmark_flow_drift_root = TMP / "benchmark-flow-drift-skill" + copy_reports(benchmark_flow_drift_root) + agents_path = benchmark_flow_drift_root / "AGENTS.md" + agents_text = agents_path.read_text(encoding="utf-8") + clean_lock_header = "For final release evidence" + benchmark_command = 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"\n' + prefix, suffix = agents_text.split(clean_lock_header, 1) + agents_path.write_text( + prefix + clean_lock_header + suffix.replace(benchmark_command, "", 1), + encoding="utf-8", + ) + benchmark_flow_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(benchmark_flow_drift_root), + "--output-json", + str(TMP / "benchmark_flow_drift.json"), + "--output-md", + str(TMP / "benchmark_flow_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert benchmark_flow_drift_proc.returncode == 2, benchmark_flow_drift_proc.stdout + benchmark_flow_drift_payload = json.loads(benchmark_flow_drift_proc.stdout) + benchmark_flow_drift_checks = {item["key"]: item for item in benchmark_flow_drift_payload["checks"]} + assert benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]["status"] == "fail", ( + benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"] + ) + + stale_clean_lock_root = TMP / "stale-clean-lock-skill" + copy_reports(stale_clean_lock_root) + benchmark_path = stale_clean_lock_root / "reports" / "benchmark_reproducibility.json" + benchmark = json.loads(benchmark_path.read_text(encoding="utf-8")) + benchmark["git_status"]["dirty"] = True + benchmark["git_status"]["changed_file_count"] = 3 + benchmark["git_status"]["source_dirty"] = True + benchmark["git_status"]["source_changed_file_count"] = 1 + benchmark["git_status"]["generated_dirty"] = True + benchmark["git_status"]["generated_changed_file_count"] = 2 + benchmark["summary"]["release_lock_ready"] = False + benchmark["summary"]["working_tree_dirty"] = True + benchmark["summary"]["changed_file_count"] = 3 + benchmark["summary"]["source_tree_dirty"] = True + benchmark["summary"]["source_changed_file_count"] = 1 + benchmark["summary"]["generated_tree_dirty"] = True + benchmark["summary"]["generated_changed_file_count"] = 2 + benchmark["release_lock"]["ready"] = False + benchmark["release_lock"]["reason"] = "source files were dirty at generation time" + benchmark_path.write_text(json.dumps(benchmark, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + for report_name in ["skill-overview", "skill-interpretation"]: + report_path = stale_clean_lock_root / "reports" / f"{report_name}.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + report["benchmark_reproducibility"]["summary"]["release_lock_ready"] = False + report["benchmark_reproducibility"]["summary"]["working_tree_dirty"] = True + report["benchmark_reproducibility"]["summary"]["changed_file_count"] = 3 + report["benchmark_reproducibility"]["summary"]["source_tree_dirty"] = True + report["benchmark_reproducibility"]["summary"]["source_changed_file_count"] = 1 + report["benchmark_reproducibility"]["summary"]["generated_tree_dirty"] = True + report["benchmark_reproducibility"]["summary"]["generated_changed_file_count"] = 2 + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + subprocess.run(["git", "init"], cwd=stale_clean_lock_root, capture_output=True, text=True, check=True) + subprocess.run(["git", "add", "."], cwd=stale_clean_lock_root, capture_output=True, text=True, check=True) + subprocess.run( + ["git", "-c", "user.name=Yao Test", "-c", "user.email=yao-test@example.com", "commit", "-m", "seed"], + cwd=stale_clean_lock_root, + capture_output=True, + text=True, + check=True, + ) + stale_clean_lock_proc = run( + [ + sys.executable, + str(SCRIPT), + str(stale_clean_lock_root), + "--output-json", + str(TMP / "stale_clean_lock.json"), + "--output-md", + str(TMP / "stale_clean_lock.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert stale_clean_lock_proc.returncode == 2, stale_clean_lock_proc.stdout + stale_clean_lock_payload = json.loads(stale_clean_lock_proc.stdout) + stale_clean_lock_checks = {item["key"]: item for item in stale_clean_lock_payload["checks"]} + assert stale_clean_lock_checks["benchmark-release-lock-self-consistency"]["status"] == "pass", ( + stale_clean_lock_checks["benchmark-release-lock-self-consistency"] + ) + assert stale_clean_lock_checks["benchmark-clean-worktree-release-lock"]["status"] == "fail", ( + stale_clean_lock_checks["benchmark-clean-worktree-release-lock"] + ) + + generated_only_lock_root = TMP / "generated-only-lock-skill" + copy_reports(generated_only_lock_root) + generated_benchmark_path = generated_only_lock_root / "reports" / "benchmark_reproducibility.json" + generated_benchmark = json.loads(generated_benchmark_path.read_text(encoding="utf-8")) + generated_benchmark["git_status"]["dirty"] = True + generated_benchmark["git_status"]["changed_file_count"] = 4 + generated_benchmark["git_status"]["source_dirty"] = False + generated_benchmark["git_status"]["source_changed_file_count"] = 0 + generated_benchmark["git_status"]["generated_dirty"] = True + generated_benchmark["git_status"]["generated_changed_file_count"] = 4 + generated_benchmark["summary"]["release_lock_ready"] = True + generated_benchmark["summary"]["working_tree_dirty"] = True + generated_benchmark["summary"]["changed_file_count"] = 4 + generated_benchmark["summary"]["source_tree_dirty"] = False + generated_benchmark["summary"]["source_changed_file_count"] = 0 + generated_benchmark["summary"]["generated_tree_dirty"] = True + generated_benchmark["summary"]["generated_changed_file_count"] = 4 + generated_benchmark["release_lock"]["ready"] = True + generated_benchmark["release_lock"]["reason"] = "only generated evidence artifacts were dirty at generation time" + generated_benchmark_path.write_text( + json.dumps(generated_benchmark, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + generated_only_proc = run( + [ + sys.executable, + str(SCRIPT), + str(generated_only_lock_root), + "--output-json", + str(TMP / "generated_only_lock.json"), + "--output-md", + str(TMP / "generated_only_lock.md"), + "--generated-at", + "2026-06-15", + ] + ) + generated_only_payload = json.loads(generated_only_proc.stdout) + generated_only_checks = {item["key"]: item for item in generated_only_payload["checks"]} + assert generated_only_checks["benchmark-release-lock-self-consistency"]["status"] == "pass", ( + generated_only_checks["benchmark-release-lock-self-consistency"] + ) + + workflow_drift_root = TMP / "workflow-drift-skill" + copy_reports(workflow_drift_root) + studio_path = workflow_drift_root / "reports" / "review-studio.json" + studio = json.loads(studio_path.read_text(encoding="utf-8")) + for action in studio["review_actions"]: + if action.get("gate_key") == "world-class-evidence": + action["evidence_steps"] = [ + item for item in action["evidence_steps"] if item.get("key") != "native-client-telemetry" + ] + studio_path.write_text(json.dumps(studio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + operator_runbook_path = workflow_drift_root / "reports" / "world_class_operator_runbook.json" + operator_runbook = json.loads(operator_runbook_path.read_text(encoding="utf-8")) + operator_runbook["coordination_plan"] = [ + item + for item in operator_runbook["coordination_plan"] + if item.get("evidence_key") != "native-client-telemetry" + ] + operator_runbook["summary"]["coordination_counts_as_completion"] = True + operator_runbook["release_gate"]["counts_as_completion"] = True + operator_runbook_path.write_text(json.dumps(operator_runbook, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + workflow_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(workflow_drift_root), + "--output-json", + str(TMP / "workflow_drift.json"), + "--output-md", + str(TMP / "workflow_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert workflow_drift_proc.returncode == 2, workflow_drift_proc.stdout + workflow_drift_payload = json.loads(workflow_drift_proc.stdout) + workflow_drift_checks = {item["key"]: item for item in workflow_drift_payload["checks"]} + assert workflow_drift_checks["world-class-evidence-workflow-coverage"]["status"] == "fail", ( + workflow_drift_checks["world-class-evidence-workflow-coverage"] + ) + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_html_rendering.py b/tests/verify_html_rendering.py new file mode 100644 index 00000000..32fb436e --- /dev/null +++ b/tests/verify_html_rendering.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from html_rendering import html_text # noqa: E402 + + +def main() -> None: + assert html_text(None) == "" + assert html_text(0) == "0" + assert html_text(False) == "False" + assert html_text("") == "" + assert html_text("&") == "<tag data-x="1">&</tag>" + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_install_simulation.py b/tests/verify_install_simulation.py index 60c79de3..f141c992 100644 --- a/tests/verify_install_simulation.py +++ b/tests/verify_install_simulation.py @@ -3,6 +3,7 @@ import json import shutil import subprocess import sys +import tempfile import zipfile from pathlib import Path @@ -28,18 +29,20 @@ def run(cmd: list[str]) -> dict: } -def build_package(out_dir: Path) -> dict: +def build_package(out_dir: Path, skill_root: Path = ROOT) -> dict: return run( [ sys.executable, str(PACKAGER), - str(ROOT), + str(skill_root), "--platform", "openai", "--platform", "claude", "--platform", "generic", + "--platform", + "vscode", "--expectations", str(EXPECTATIONS), "--output-dir", @@ -49,12 +52,12 @@ def build_package(out_dir: Path) -> dict: ) -def simulate(package_dir: Path, output_json: Path, output_md: Path) -> dict: +def simulate(package_dir: Path, output_json: Path, output_md: Path, skill_root: Path = ROOT) -> dict: return run( [ sys.executable, str(SIMULATOR), - str(ROOT), + str(skill_root), "--package-dir", str(package_dir), "--install-root", @@ -69,6 +72,23 @@ def simulate(package_dir: Path, output_json: Path, output_md: Path) -> dict: ) +def rewrite_archive_json(package_dir: Path, relative_path: str, transform) -> None: + archive_path = package_dir / "yao-meta-skill.zip" + rewritten_path = package_dir / "yao-meta-skill.rewritten.zip" + archive_member = f"yao-meta-skill/{relative_path}" + replaced = False + with zipfile.ZipFile(archive_path) as archive_in, zipfile.ZipFile(rewritten_path, "w", compression=zipfile.ZIP_DEFLATED) as archive_out: + for info in archive_in.infolist(): + data = archive_in.read(info.filename) + if info.filename == archive_member: + payload = json.loads(data.decode("utf-8")) + data = (json.dumps(transform(payload), ensure_ascii=False, indent=2) + "\n").encode("utf-8") + replaced = True + archive_out.writestr(info, data) + assert replaced, archive_member + rewritten_path.replace(archive_path) + + def main() -> None: if TMP.exists(): shutil.rmtree(TMP) @@ -85,9 +105,47 @@ def main() -> None: assert payload["summary"]["entrypoint_loaded"], payload assert payload["summary"]["manifest_loaded"], payload assert payload["summary"]["interface_loaded"], payload - assert payload["summary"]["adapter_count"] == 3, payload + assert payload["summary"]["adapter_count"] == 4, payload + assert payload["summary"]["installer_permission_enforced_count"] == 12, payload + assert payload["summary"]["installer_permission_failure_count"] == 0, payload + assert payload["summary"]["permission_target_count"] == 4, payload + assert payload["summary"]["permission_capability_count"] == 3, payload assert not payload["failures"], payload - assert "Install Simulation" in (TMP / "install_simulation.md").read_text(encoding="utf-8") + valid_markdown = (TMP / "install_simulation.md").read_text(encoding="utf-8") + assert "Install Simulation" in valid_markdown + assert "Installer permissions enforced" in valid_markdown + + with tempfile.TemporaryDirectory(prefix="renamed-install-root-") as temp_root: + renamed_root = Path(temp_root) / "checkout-alias" + shutil.copytree( + ROOT, + renamed_root, + ignore=shutil.ignore_patterns(".git", ".previews", "dist", "__pycache__", ".pytest_cache", "tmp*"), + ) + renamed_dir = TMP / "renamed-dist" + renamed_build = build_package(renamed_dir, renamed_root) + assert renamed_build["ok"], renamed_build + assert (renamed_dir / "yao-meta-skill.zip").exists(), renamed_build + renamed_valid = simulate(renamed_dir, TMP / "renamed_install_simulation.json", TMP / "renamed_install_simulation.md", renamed_root) + assert renamed_valid["ok"], renamed_valid + assert renamed_valid["payload"]["summary"]["archive_extracted"], renamed_valid + assert renamed_valid["payload"]["installed_skill_dir"].endswith("simulate-yao-meta-skill/yao-meta-skill"), renamed_valid + + policy_gap_dir = TMP / "policy-gap-dist" + shutil.copytree(valid_dir, policy_gap_dir) + + def remove_vscode_network_enforcement(payload: dict) -> dict: + payload["capabilities"]["network"]["target_enforcement"].pop("vscode", None) + return payload + + rewrite_archive_json(policy_gap_dir, "security/permission_policy.json", remove_vscode_network_enforcement) + policy_gap = simulate(policy_gap_dir, TMP / "policy_gap.json", TMP / "policy_gap.md") + policy_gap_payload = policy_gap["payload"] + assert policy_gap["returncode"] == 2, policy_gap + assert not policy_gap_payload["ok"], policy_gap_payload + assert policy_gap_payload["summary"]["installer_permission_enforced_count"] == 11, policy_gap_payload + assert policy_gap_payload["summary"]["installer_permission_failure_count"] >= 1, policy_gap_payload + assert any("vscode capability network has target enforcement note" in item for item in policy_gap_payload["failures"]), policy_gap_payload unsafe_dir = TMP / "unsafe-dist" shutil.copytree(valid_dir, unsafe_dir) diff --git a/tests/verify_local_install_sync.py b/tests/verify_local_install_sync.py index 1e74afca..d9b9d738 100644 --- a/tests/verify_local_install_sync.py +++ b/tests/verify_local_install_sync.py @@ -3,21 +3,73 @@ import json import shutil import subprocess import sys +import zipfile from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SYNC_SCRIPT = ROOT / "scripts" / "sync_local_install.py" TMP = ROOT / "tests" / "tmp" / "local_install_sync" +PACKAGER = ROOT / "scripts" / "cross_packager.py" +EXPECTATIONS = ROOT / "evals" / "packaging_expectations.json" -def run_sync(install_dir: Path) -> dict: +def build_package(out_dir: Path) -> dict: + proc = subprocess.run( + [ + sys.executable, + str(PACKAGER), + str(ROOT), + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--platform", + "vscode", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(out_dir), + "--zip", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return {"ok": proc.returncode == 0, "payload": payload, "stderr": proc.stderr} + + +def rewrite_archive_json(package_dir: Path, relative_path: str, transform) -> None: + archive_path = package_dir / "yao-meta-skill.zip" + rewritten_path = package_dir / "yao-meta-skill.rewritten.zip" + archive_member = f"yao-meta-skill/{relative_path}" + replaced = False + with zipfile.ZipFile(archive_path) as archive_in, zipfile.ZipFile(rewritten_path, "w", compression=zipfile.ZIP_DEFLATED) as archive_out: + for info in archive_in.infolist(): + data = archive_in.read(info.filename) + if info.filename == archive_member: + payload = json.loads(data.decode("utf-8")) + data = (json.dumps(transform(payload), ensure_ascii=False, indent=2) + "\n").encode("utf-8") + replaced = True + archive_out.writestr(info, data) + assert replaced, archive_member + rewritten_path.replace(archive_path) + + +def run_sync(install_dir: Path, package_dir: Path) -> dict: proc = subprocess.run( [ sys.executable, str(SYNC_SCRIPT), "--install-dir", str(install_dir), + "--package-dir", + str(package_dir), + "--generated-at", + "2026-06-13", ], cwd=ROOT, capture_output=True, @@ -33,13 +85,17 @@ def run_sync(install_dir: Path) -> dict: } -def run_sync_raw(install_dir: Path) -> subprocess.CompletedProcess[str]: +def run_sync_raw(install_dir: Path, package_dir: Path) -> subprocess.CompletedProcess[str]: return subprocess.run( [ sys.executable, str(SYNC_SCRIPT), "--install-dir", str(install_dir), + "--package-dir", + str(package_dir), + "--generated-at", + "2026-06-13", ], cwd=ROOT, capture_output=True, @@ -51,6 +107,9 @@ def main() -> None: if TMP.exists(): shutil.rmtree(TMP) TMP.mkdir(parents=True, exist_ok=True) + package_dir = TMP / "dist" + package = build_package(package_dir) + assert package["ok"], package install_dir = TMP / "installed-skill" install_dir.mkdir(parents=True) @@ -67,15 +126,33 @@ def main() -> None: untracked_file = ROOT / "sync-local-untracked.tmp" untracked_file.write_text("do not copy me\n", encoding="utf-8") try: - result = run_sync(install_dir) + result = run_sync(install_dir, package_dir) finally: untracked_file.unlink(missing_ok=True) + policy_gap_dir = TMP / "policy-gap-dist" + shutil.copytree(package_dir, policy_gap_dir) + + def remove_vscode_network_enforcement(payload: dict) -> dict: + payload["capabilities"]["network"]["target_enforcement"].pop("vscode", None) + return payload + + rewrite_archive_json(policy_gap_dir, "security/permission_policy.json", remove_vscode_network_enforcement) + refused_install_dir = TMP / "preflight-refused-install" + refused_install_dir.mkdir(parents=True) + (refused_install_dir / "SKILL.md").write_text( + "---\nname: yao-meta-skill\ndescription: local install fixture\n---\n", + encoding="utf-8", + ) + refused_stale = refused_install_dir / "stale.txt" + refused_stale.write_text("must not be touched after preflight failure\n", encoding="utf-8") + preflight_refused = run_sync_raw(refused_install_dir, policy_gap_dir) + ordinary_dir = TMP / "ordinary-folder" ordinary_dir.mkdir(parents=True) ordinary_file = ordinary_dir / "keep.txt" ordinary_file.write_text("must survive a refused sync\n", encoding="utf-8") - refused = run_sync_raw(ordinary_dir) + refused = run_sync_raw(ordinary_dir, package_dir) makefile_text = (ROOT / "Makefile").read_text(encoding="utf-8") readme_text = (ROOT / "README.md").read_text(encoding="utf-8") @@ -89,6 +166,10 @@ def main() -> None: "stale_file_removed": not stale_file.exists(), "install_git_metadata_preserved": git_config.exists(), "install_sentinel_written": (install_dir / ".yao-local-install.json").exists(), + "install_preflight_enforced": result["payload"]["install_preflight"]["installer_permission_enforced_count"] == 12, + "install_preflight_permission_failures_zero": result["payload"]["install_preflight"]["installer_permission_failure_count"] == 0, + "install_preflight_blocks_sync": preflight_refused.returncode != 0, + "install_preflight_failure_preserves_files": refused_stale.exists(), "ordinary_dir_refused": refused.returncode != 0, "ordinary_dir_preserved": ordinary_file.exists(), "makefile_target_present": "sync-local-install" in makefile_text, diff --git a/tests/verify_output_execution_runs.py b/tests/verify_output_execution_runs.py index a488185d..371700c9 100644 --- a/tests/verify_output_execution_runs.py +++ b/tests/verify_output_execution_runs.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 import json +import os import shutil import subprocess import sys +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "run_output_execution.py" LOCAL_RUNNER = ROOT / "scripts" / "local_output_eval_runner.py" +PROVIDER_RUNNER = ROOT / "scripts" / "provider_output_eval_runner.py" def main() -> None: @@ -125,6 +129,353 @@ def main() -> None: assert all(item["duration_ms"] is not None for item in command["runs"]), command assert all(item["model"] == "fixture-model" for item in command["runs"]), command + metadata_only_runner = tmp_root / "metadata_only_runner.py" + metadata_only_runner.write_text( + "\n".join( + [ + "import json, sys", + "request = json.loads(sys.stdin.read())", + "print(json.dumps({", + " 'output': request['fixture_output'],", + " 'provider': 'metadata-only-provider',", + " 'model': 'metadata-only-model',", + " 'usage': {'input_tokens': 11, 'output_tokens': 17, 'total_tokens': 28}", + "}))", + ] + ) + + "\n", + encoding="utf-8", + ) + metadata_only_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(tmp_root / "metadata-only.json"), + "--output-md", + str(tmp_root / "metadata-only.md"), + "--runner-command", + json.dumps([sys.executable, str(metadata_only_runner)]), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + metadata_only = json.loads(metadata_only_proc.stdout) + assert metadata_only["ok"], metadata_only + assert metadata_only["summary"]["command_executed_count"] == 10, metadata_only + assert metadata_only["summary"]["model_executed_count"] == 0, metadata_only + assert metadata_only["summary"]["token_observed_count"] == 10, metadata_only + assert all(item["execution_mode"] == "command" for item in metadata_only["runs"]), metadata_only + + provider_request = { + "case_id": "provider-contract", + "variant": "with_skill", + "prompt": "Create a reusable skill package.", + "input_files": [], + "metadata": {}, + } + missing_env = os.environ.copy() + missing_env.pop("OPENAI_API_KEY", None) + missing_env.pop("YAO_OUTPUT_EVAL_MODEL", None) + missing_provider_proc = subprocess.run( + [sys.executable, str(PROVIDER_RUNNER), "--model", "fixture-model"], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=missing_env, + ) + assert missing_provider_proc.returncode == 2, missing_provider_proc.stdout + assert "missing API key env" in missing_provider_proc.stderr, missing_provider_proc.stderr + missing_deepseek_proc = subprocess.run( + [sys.executable, str(PROVIDER_RUNNER), "--provider", "deepseek", "--model", "fixture-model"], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=missing_env, + ) + assert missing_deepseek_proc.returncode == 2, missing_deepseek_proc.stdout + assert "missing API key env: DEEPSEEK_API_KEY" in missing_deepseek_proc.stderr, missing_deepseek_proc.stderr + + custom_env = os.environ.copy() + custom_env["OPENAI_API_KEY"] = "test-key" + custom_env["DEEPSEEK_API_KEY"] = "test-key" + custom_env["YAO_OUTPUT_EVAL_MODEL"] = "fixture-model" + custom_host_proc = subprocess.run( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--base-url", + "https://example.com/v1/responses", + ], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=custom_env, + ) + assert custom_host_proc.returncode == 2, custom_host_proc.stdout + assert "custom provider host requires --allow-custom-base-url" in custom_host_proc.stderr, custom_host_proc.stderr + + wrong_path_proc = subprocess.run( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--base-url", + "https://api.openai.com/v1/chat/completions", + ], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=custom_env, + ) + assert wrong_path_proc.returncode == 2, wrong_path_proc.stdout + assert "provider endpoint path must start with /v1/responses" in wrong_path_proc.stderr, wrong_path_proc.stderr + empty_path_proc = subprocess.run( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--base-url", + "https://api.openai.com", + ], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=custom_env, + ) + assert empty_path_proc.returncode == 2, empty_path_proc.stdout + assert "provider endpoint path must start with /v1/responses" in empty_path_proc.stderr, empty_path_proc.stderr + + class ProviderHandler(BaseHTTPRequestHandler): + requests: list[dict] = [] + + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length).decode("utf-8") + ProviderHandler.requests.append( + { + "path": self.path, + "authorization": self.headers.get("Authorization", ""), + "body": json.loads(body), + } + ) + if self.path.endswith("/chat/completions"): + payload = { + "id": "chat_fixture", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Create SKILL.md and reports/skill-overview.html for this reusable package.", + } + } + ], + "usage": {"prompt_tokens": 21, "completion_tokens": 9, "total_tokens": 30}, + } + else: + payload = { + "id": "resp_fixture", + "output": [ + { + "content": [ + { + "type": "output_text", + "text": "Create SKILL.md and reports/skill-overview.html for this reusable package.", + } + ] + } + ], + "usage": {"input_tokens": 21, "output_tokens": 9, "total_tokens": 30}, + } + encoded = json.dumps(payload).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, format: str, *args: object) -> None: + return + + server = HTTPServer(("127.0.0.1", 0), ProviderHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + provider_url = f"http://127.0.0.1:{server.server_port}/v1/responses" + provider_env = os.environ.copy() + provider_env["OPENAI_API_KEY"] = "test-key" + provider_env["DEEPSEEK_API_KEY"] = "test-key" + provider_env["YAO_OUTPUT_EVAL_MODEL"] = "fixture-model" + try: + direct_provider_proc = subprocess.run( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--base-url", + provider_url, + "--allow-insecure-localhost", + ], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=provider_env, + check=True, + ) + direct_provider = json.loads(direct_provider_proc.stdout) + assert direct_provider["execution_kind"] == "model", direct_provider + assert direct_provider["provider"] == "openai", direct_provider + assert direct_provider["model"] == "fixture-model", direct_provider + assert direct_provider["usage"]["estimated"] is False, direct_provider + assert ProviderHandler.requests[-1]["path"] == "/v1/responses", ProviderHandler.requests[-1] + assert ProviderHandler.requests[-1]["authorization"] == "Bearer test-key", ProviderHandler.requests[-1] + assert ProviderHandler.requests[-1]["body"]["model"] == "fixture-model", ProviderHandler.requests[-1] + assert ProviderHandler.requests[-1]["body"]["temperature"] == 0.0, ProviderHandler.requests[-1] + assert "Create a reusable skill package." in ProviderHandler.requests[-1]["body"]["input"], ProviderHandler.requests[-1] + assert "fixture_output" not in ProviderHandler.requests[-1]["body"]["input"], ProviderHandler.requests[-1] + + governed_provider_request = { + "case_id": "file-backed-governed-package", + "variant": "with_skill", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": ["fixtures/release-brief-source.md"], + "metadata": {"case_type": "boundary", "tier": "governed"}, + } + governed_provider_proc = subprocess.run( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--base-url", + provider_url, + "--allow-insecure-localhost", + "--model", + "fixture-model", + ], + cwd=ROOT, + input=json.dumps(governed_provider_request), + capture_output=True, + text=True, + env=provider_env, + check=True, + ) + governed_provider = json.loads(governed_provider_proc.stdout) + assert governed_provider["execution_kind"] == "model", governed_provider + governed_input = ProviderHandler.requests[-1]["body"]["input"] + assert "Release Brief Source Fixture" in governed_input, governed_input + assert "file-backed fixture" in governed_input, governed_input + assert "evidence" in governed_input, governed_input + assert "input_files" in governed_input, governed_input + assert "output contract" in governed_input, governed_input + assert "rollback boundary" in governed_input, governed_input + assert "trust report" in governed_input, governed_input + assert "reports/output_quality_scorecard.md" in governed_input, governed_input + assert "missing evidence" in governed_input, governed_input + + chat_provider_url = f"http://127.0.0.1:{server.server_port}/chat/completions" + direct_chat_provider_proc = subprocess.run( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--provider", + "deepseek", + "--api-format", + "chat-completions", + "--thinking", + "disabled", + "--base-url", + chat_provider_url, + "--allow-insecure-localhost", + "--model", + "fixture-model", + ], + cwd=ROOT, + input=json.dumps(provider_request), + capture_output=True, + text=True, + env=provider_env, + check=True, + ) + direct_chat_provider = json.loads(direct_chat_provider_proc.stdout) + assert direct_chat_provider["execution_kind"] == "model", direct_chat_provider + assert direct_chat_provider["provider"] == "deepseek", direct_chat_provider + assert direct_chat_provider["model"] == "fixture-model", direct_chat_provider + assert direct_chat_provider["usage"]["estimated"] is False, direct_chat_provider + assert ProviderHandler.requests[-1]["path"] == "/chat/completions", ProviderHandler.requests[-1] + assert ProviderHandler.requests[-1]["body"]["model"] == "fixture-model", ProviderHandler.requests[-1] + assert ProviderHandler.requests[-1]["body"]["thinking"]["type"] == "disabled", ProviderHandler.requests[-1] + assert ProviderHandler.requests[-1]["body"]["temperature"] == 0.0, ProviderHandler.requests[-1] + assert "Create a reusable skill package." in ProviderHandler.requests[-1]["body"]["messages"][0]["content"], ( + ProviderHandler.requests[-1] + ) + + provider_cases = tmp_root / "provider_cases.jsonl" + provider_cases.write_text( + json.dumps( + { + "id": "provider-contract", + "prompt": "Create a reusable skill package.", + "baseline_output": "", + "with_skill_output": "", + "assertions": [ + { + "id": "has-skill-entry", + "description": "Provider output includes the skill entrypoint.", + "required": ["SKILL.md"], + } + ], + }, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + provider_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(provider_cases), + "--output-json", + str(tmp_root / "provider.json"), + "--output-md", + str(tmp_root / "provider.md"), + "--runner-command", + json.dumps( + [ + sys.executable, + str(PROVIDER_RUNNER), + "--base-url", + provider_url, + "--allow-insecure-localhost", + "--model", + "fixture-model", + ] + ), + ], + cwd=ROOT, + capture_output=True, + text=True, + env=provider_env, + check=True, + ) + provider = json.loads(provider_proc.stdout) + assert provider["ok"], provider + assert provider["summary"]["variant_run_count"] == 2, provider + assert provider["summary"]["model_executed_count"] == 2, provider + assert provider["summary"]["token_observed_count"] == 2, provider + assert provider["summary"]["token_estimated_count"] == 0, provider + assert all(item["provider"] == "openai" for item in provider["runs"]), provider + assert all(item["model"] == "fixture-model" for item in provider["runs"]), provider + finally: + server.shutdown() + thread.join(timeout=2) + bad_runner = tmp_root / "bad_runner.py" bad_runner.write_text("import sys\nsys.exit(3)\n", encoding="utf-8") bad_proc = subprocess.run( diff --git a/tests/verify_output_review_adjudication.py b/tests/verify_output_review_adjudication.py index 93ac4fa8..45f35f68 100644 --- a/tests/verify_output_review_adjudication.py +++ b/tests/verify_output_review_adjudication.py @@ -7,8 +7,14 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from output_review_privacy import BLOCKED_DECISION_FIELDS # noqa: E402 + OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py" ADJUDICATOR = ROOT / "scripts" / "adjudicate_output_review.py" +IMPORTER = ROOT / "scripts" / "import_output_review_decisions.py" +CLI = ROOT / "scripts" / "yao.py" def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: @@ -21,7 +27,41 @@ def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str] ) +def write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def adjudicate_payload( + tmp_root: Path, + blind_pack_json: Path, + answer_key_json: Path, + name: str, + decisions: dict, +) -> tuple[subprocess.CompletedProcess[str], dict]: + decisions_path = tmp_root / f"{name}_decisions.json" + write_json(decisions_path, decisions) + proc = run( + [ + str(ADJUDICATOR), + "--blind-pack", + str(blind_pack_json), + "--answer-key", + str(answer_key_json), + "--decisions", + str(decisions_path), + "--output-json", + str(tmp_root / f"{name}_adjudication.json"), + "--output-md", + str(tmp_root / f"{name}_adjudication.md"), + ], + check=False, + ) + return proc, json.loads(proc.stdout) + + def main() -> None: + assert {"api_key", "raw_provider_prompt", "expected_winner_role"} <= BLOCKED_DECISION_FIELDS + tmp_root = ROOT / "tests" / "tmp_output_review_adjudication" if tmp_root.exists(): shutil.rmtree(tmp_root) @@ -75,7 +115,38 @@ def main() -> None: assert pending_payload["summary"]["pending_count"] == 5, pending_payload assert pending_payload["summary"]["agreement_rate"] is None, pending_payload assert pending_payload["summary"]["needs_review"], pending_payload + assert pending_payload["summary"]["answer_revealed_count"] == 0, pending_payload + assert pending_payload["summary"]["pending_answer_hidden_count"] == 5, pending_payload + assert pending_payload["summary"]["reviewer_checklist_count"] == 5, pending_payload + assert pending_payload["summary"]["reviewer_checklist_pending_count"] == 5, pending_payload + assert pending_payload["summary"]["reviewer_checklist_ready_count"] == 0, pending_payload + assert pending_payload["summary"]["reviewer_checklist_invalid_count"] == 0, pending_payload + assert pending_payload["summary"]["reviewer_metadata_present"] is False, pending_payload + assert pending_payload["summary"]["blind_review_attested"] is False, pending_payload + assert pending_payload["summary"]["raw_content_excluded_attested"] is False, pending_payload + assert pending_payload["summary"]["reason_required"] is True, pending_payload + assert pending_payload["summary"]["ready_for_human_evidence"] is False, pending_payload + assert all(not item["expected_winner_variant"] and not item["expected_revealed"] for item in pending_payload["pairs"]), pending_payload + assert all("prompt" not in item and len(item["prompt_sha256"]) == 64 for item in pending_payload["pairs"]), pending_payload + checklist = {item["case_id"]: item for item in pending_payload["reviewer_checklist"]} + assert checklist["skill-package-contract"]["readiness"] == "awaiting-decision", checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["answer_key_visible"] is False, checklist["skill-package-contract"] + assert "prompt" not in checklist["skill-package-contract"] and len(checklist["skill-package-contract"]["prompt_sha256"]) == 64, checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["decisions_path"].endswith("tests/tmp_output_review_adjudication/missing_decisions.json"), checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["commands"]["write_template"] == "python3 scripts/adjudicate_output_review.py --write-template", checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["commands"]["import_decisions"].startswith("python3 scripts/yao.py output-review-import"), checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["required_fields"]["winner_variant"].startswith("A or B"), checklist["skill-package-contract"] + assert checklist["skill-package-contract"]["required_fields"]["reason"].startswith("Required rationale"), checklist["skill-package-contract"] + assert "reviewer" in checklist["skill-package-contract"]["required_fields"], checklist["skill-package-contract"] + assert "reviewed_at" in checklist["skill-package-contract"]["required_fields"], checklist["skill-package-contract"] assert "No reviewer decisions recorded yet" in pending_md.read_text(encoding="utf-8"), pending_md + pending_text = pending_md.read_text(encoding="utf-8") + assert "| skill-package-contract | pending | hidden | pending |" in pending_text, pending_text + assert "| skill-package-contract | pending | A | pending |" not in pending_text, pending_text + assert "Reviewer Checklist" in pending_text, pending_text + assert "Reviewer checklist: `0` ready / `5` total" in pending_text, pending_text + assert "answer key visible: `false`" in pending_text, pending_text + assert "Ready for human evidence: `false`" in pending_text, pending_text template_path = tmp_root / "output_review_decisions.json" template_proc = run( @@ -100,12 +171,23 @@ def main() -> None: template = json.loads(template_path.read_text(encoding="utf-8")) assert len(template["decisions"]) == 5, template assert all(item["winner_variant"] == "" for item in template["decisions"]), template + assert len(template["review_integrity"]["blind_pack_sha256"]) == 64, template + assert template["review_integrity"]["case_count"] == 5, template + assert template["reviewer_attestation"]["blind_review_completed_before_answer_key"] is False, template + assert template["reviewer_attestation"]["answer_key_not_opened_before_decisions"] is False, template answer_key = json.loads(answer_key_json.read_text(encoding="utf-8")) + reviewer_attestation = { + "blind_review_completed_before_answer_key": True, + "answer_key_not_opened_before_decisions": True, + "raw_content_excluded": True, + "reviewer_reason_required": True, + } decisions = { "schema_version": "1.0", "reviewer": "Yao QA", "reviewed_at": "2026-06-13", + "reviewer_attestation": reviewer_attestation, "decisions": [ { "case_id": item["case_id"], @@ -139,7 +221,287 @@ def main() -> None: assert filled_payload["summary"]["pending_count"] == 0, filled_payload assert filled_payload["summary"]["agreement_count"] == 5, filled_payload assert filled_payload["summary"]["agreement_rate"] == 100.0, filled_payload + assert filled_payload["summary"]["answer_revealed_count"] == 5, filled_payload + assert filled_payload["summary"]["pending_answer_hidden_count"] == 0, filled_payload + assert filled_payload["summary"]["reviewer_checklist_ready_count"] == 5, filled_payload + assert filled_payload["summary"]["reviewer_checklist_pending_count"] == 0, filled_payload + assert filled_payload["summary"]["reviewer_metadata_present"] is True, filled_payload + assert filled_payload["summary"]["blind_review_attested"] is True, filled_payload + assert filled_payload["summary"]["raw_content_excluded_attested"] is True, filled_payload + assert filled_payload["summary"]["ready_for_human_evidence"] is True, filled_payload + assert len(filled_payload["review_integrity"]["blind_pack_sha256"]) == 64, filled_payload assert all(item["status"] == "match" for item in filled_payload["pairs"]), filled_payload + assert all(item["expected_winner_variant"] in {"A", "B"} and item["expected_revealed"] for item in filled_payload["pairs"]), filled_payload + assert all("prompt" not in item and len(item["prompt_sha256"]) == 64 for item in filled_payload["pairs"]), filled_payload + filled_checklist = {item["case_id"]: item for item in filled_payload["reviewer_checklist"]} + assert all(item["readiness"] == "adjudicated" and item["answer_key_visible"] and "prompt" not in item and len(item["prompt_sha256"]) == 64 for item in filled_checklist.values()), filled_checklist + + import_source = tmp_root / "reviewer_source.json" + import_source.write_text(json.dumps(decisions, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + imported_path = tmp_root / "imported_decisions.json" + imported_proc = run( + [ + str(IMPORTER), + "--input", + str(import_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(imported_path), + "--run-adjudication", + "--answer-key", + str(answer_key_json), + "--adjudication-json", + str(tmp_root / "imported_adjudication.json"), + "--adjudication-md", + str(tmp_root / "imported_adjudication.md"), + ] + ) + imported_payload = json.loads(imported_proc.stdout) + assert imported_payload["ok"], imported_payload + assert imported_payload["summary"]["canonical_written"] is True, imported_payload + assert imported_payload["summary"]["adjudication_run"] is True, imported_payload + assert imported_payload["summary"]["adjudication_pending_count"] == 0, imported_payload + imported_decisions = json.loads(imported_path.read_text(encoding="utf-8")) + assert imported_decisions["import_contract"]["raw_content_allowed"] is False, imported_decisions + assert imported_decisions["import_contract"]["answer_key_fields_allowed"] is False, imported_decisions + assert imported_decisions["import_contract"]["answer_key_opened_by_importer"] is False, imported_decisions + assert imported_decisions["import_contract"]["blind_review_attested"] is True, imported_decisions + assert imported_decisions["reviewer_attestation"]["blind_review_completed_before_answer_key"] is True, imported_decisions + assert len(imported_decisions["review_integrity"]["blind_pack_sha256"]) == 64, imported_decisions + assert imported_decisions["reviewer"] == "Yao QA", imported_decisions + + no_reason_source = tmp_root / "no_reason_source.json" + write_json( + no_reason_source, + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": answer_key["answers"][0]["expected_winner_variant"], + "confidence": 0.8, + } + ], + }, + ) + no_reason_import_proc = run( + [ + str(IMPORTER), + "--input", + str(no_reason_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "no_reason_decisions.json"), + ], + check=False, + ) + no_reason_import_payload = json.loads(no_reason_import_proc.stdout) + assert no_reason_import_proc.returncode == 2, no_reason_import_payload + assert any("reason is required for imported human decisions" in failure for failure in no_reason_import_payload["failures"]), no_reason_import_payload + assert not (tmp_root / "no_reason_decisions.json").exists(), no_reason_import_payload + + cli_import_proc = run( + [ + str(CLI), + "output-review-import", + "--input", + str(import_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "cli_imported_decisions.json"), + "--run-adjudication", + "--answer-key", + str(answer_key_json), + "--adjudication-json", + str(tmp_root / "cli_imported_adjudication.json"), + "--adjudication-md", + str(tmp_root / "cli_imported_adjudication.md"), + ] + ) + cli_import_payload = json.loads(cli_import_proc.stdout) + assert cli_import_payload["ok"], cli_import_payload + assert cli_import_payload["summary"]["canonical_written"] is True, cli_import_payload + assert cli_import_payload["summary"]["adjudication_pending_count"] == 0, cli_import_payload + + jsonl_source = tmp_root / "reviewer_source.jsonl" + jsonl_source.write_text("\n".join(json.dumps(item, ensure_ascii=False) for item in decisions["decisions"]) + "\n", encoding="utf-8") + jsonl_proc = run([ + str(IMPORTER), "--input", str(jsonl_source), "--blind-pack", str(blind_pack_json), + "--output-json", str(tmp_root / "jsonl_decisions.json"), "--reviewer", "Yao QA", + "--reviewed-at", "2026-06-13", "--blind-review-attested", + ]) + jsonl_payload = json.loads(jsonl_proc.stdout) + assert jsonl_payload["ok"], jsonl_payload + assert jsonl_payload["summary"]["completed_decision_count"] == 5, jsonl_payload + jsonl_decisions = json.loads((tmp_root / "jsonl_decisions.json").read_text(encoding="utf-8")) + assert jsonl_decisions["reviewer_attestation"]["blind_review_completed_before_answer_key"] is True, jsonl_decisions + + csv_source = tmp_root / "reviewer_source.csv" + csv_source.write_text( + "case_id,winner_variant,confidence,reason\n" + + "\n".join( + f"{item['case_id']},{item['expected_winner_variant']},0.8,CSV reviewer choice" + for item in answer_key["answers"] + ) + + "\n", + encoding="utf-8", + ) + csv_proc = run([ + str(IMPORTER), "--input", str(csv_source), "--blind-pack", str(blind_pack_json), + "--output-json", str(tmp_root / "csv_decisions.json"), "--reviewer", "Yao QA", + "--reviewed-at", "2026-06-13", "--blind-review-attested", + ]) + csv_payload = json.loads(csv_proc.stdout) + assert csv_payload["ok"], csv_payload + assert csv_payload["summary"]["completed_decision_count"] == 5, csv_payload + + private_source = tmp_root / "private_source.json" + private_source.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": "A", + "prompt": "raw prompt must not be imported", + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + private_proc = run( + [ + str(IMPORTER), + "--input", + str(private_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "private_decisions.json"), + ], + check=False, + ) + private_payload = json.loads(private_proc.stdout) + assert private_proc.returncode == 2, private_payload + assert private_payload["ok"] is False, private_payload + assert any("forbidden raw or answer-key fields" in failure for failure in private_payload["failures"]), private_payload + assert not (tmp_root / "private_decisions.json").exists(), private_payload + + nested_private_source = tmp_root / "nested_private_source.json" + nested_private_source.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": "A", + "metadata": {"Raw_Output": "nested raw output must not be imported"}, + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + nested_private_proc = run( + [ + str(IMPORTER), + "--input", + str(nested_private_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "nested_private_decisions.json"), + ], + check=False, + ) + nested_private_payload = json.loads(nested_private_proc.stdout) + assert nested_private_proc.returncode == 2, nested_private_payload + assert nested_private_payload["ok"] is False, nested_private_payload + assert any("decision #1.metadata.Raw_Output" in failure for failure in nested_private_payload["failures"]), nested_private_payload + assert not (tmp_root / "nested_private_decisions.json").exists(), nested_private_payload + + credential_source = tmp_root / "credential_source.json" + credential_source.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": "A", + "metadata": {"API_KEY": "credential material must not be imported"}, + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + credential_proc = run( + [ + str(IMPORTER), + "--input", + str(credential_source), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "credential_decisions.json"), + ], + check=False, + ) + credential_payload = json.loads(credential_proc.stdout) + assert credential_proc.returncode == 2, credential_payload + assert credential_payload["ok"] is False, credential_payload + assert any("decision #1.metadata.API_KEY" in failure for failure in credential_payload["failures"]), credential_payload + assert not (tmp_root / "credential_decisions.json").exists(), credential_payload + + unknown_case = tmp_root / "unknown_case.json" + unknown_case.write_text( + json.dumps( + { + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [{"case_id": "unknown", "winner_variant": "A"}], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + unknown_proc = run( + [ + str(IMPORTER), + "--input", + str(unknown_case), + "--blind-pack", + str(blind_pack_json), + "--output-json", + str(tmp_root / "unknown_decisions.json"), + ], + check=False, + ) + unknown_payload = json.loads(unknown_proc.stdout) + assert unknown_proc.returncode == 2, unknown_payload + assert any("unknown case_id" in failure for failure in unknown_payload["failures"]), unknown_payload invalid = { "schema_version": "1.0", @@ -176,8 +538,60 @@ def main() -> None: invalid_payload = json.loads(invalid_proc.stdout) assert not invalid_payload["ok"], invalid_payload assert invalid_payload["summary"]["invalid_decision_count"] == 1, invalid_payload + assert invalid_payload["summary"]["answer_revealed_count"] == 0, invalid_payload + assert invalid_payload["summary"]["reviewer_checklist_invalid_count"] == 1, invalid_payload + assert invalid_payload["pairs"][0]["expected_winner_variant"] == "", invalid_payload + assert invalid_payload["pairs"][0]["expected_revealed"] is False, invalid_payload + invalid_checklist = {item["case_id"]: item for item in invalid_payload["reviewer_checklist"]} + assert invalid_checklist[answer_key["answers"][0]["case_id"]]["readiness"] == "fix-decision", invalid_checklist + assert invalid_checklist[answer_key["answers"][0]["case_id"]]["answer_key_visible"] is False, invalid_checklist assert invalid_payload["failures"], invalid_payload + no_reason_direct = { + "schema_version": "1.0", + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": answer_key["answers"][0]["expected_winner_variant"], + "confidence": 0.9, + "reason": "", + } + ], + } + no_reason_proc, no_reason_payload = adjudicate_payload( + tmp_root, blind_pack_json, answer_key_json, "no_reason", no_reason_direct + ) + assert no_reason_proc.returncode == 2, no_reason_proc.stdout + assert no_reason_payload["summary"]["invalid_decision_count"] == 1, no_reason_payload + assert no_reason_payload["summary"]["answer_revealed_count"] == 0, no_reason_payload + assert no_reason_payload["summary"]["ready_for_human_evidence"] is False, no_reason_payload + assert no_reason_payload["pairs"][0]["expected_winner_variant"] == "", no_reason_payload + assert no_reason_payload["pairs"][0]["expected_revealed"] is False, no_reason_payload + assert any("reason is required before answer key can be revealed" in failure for failure in no_reason_payload["failures"]), no_reason_payload + + missing_metadata = { + "schema_version": "1.0", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": answer_key["answers"][0]["expected_winner_variant"], + "confidence": 0.9, + "reason": "Visible rubric-based rationale.", + } + ], + } + missing_metadata_proc, missing_metadata_payload = adjudicate_payload( + tmp_root, blind_pack_json, answer_key_json, "missing_metadata", missing_metadata + ) + assert missing_metadata_proc.returncode == 2, missing_metadata_proc.stdout + assert missing_metadata_payload["summary"]["reviewer_metadata_present"] is False, missing_metadata_payload + assert missing_metadata_payload["summary"]["invalid_decision_count"] == 1, missing_metadata_payload + assert missing_metadata_payload["summary"]["answer_revealed_count"] == 0, missing_metadata_payload + assert missing_metadata_payload["summary"]["ready_for_human_evidence"] is False, missing_metadata_payload + assert any("reviewer and reviewed_at are required" in failure for failure in missing_metadata_payload["failures"]), missing_metadata_payload + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_output_review_kit.py b/tests/verify_output_review_kit.py new file mode 100644 index 00000000..e8d0cd3d --- /dev/null +++ b/tests/verify_output_review_kit.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py" +REVIEW_KIT = ROOT / "scripts" / "prepare_output_review_kit.py" +CLI = ROOT / "scripts" / "yao.py" + + +def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *args], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def assert_no_answer_key_leak(text: str) -> None: + forbidden = [ + "output_blind_answer_key", + "expected_winner", + "variant_a_role", + "variant_b_role", + "score_winner_role", + ] + for needle in forbidden: + assert needle not in text, f"review kit leaked hidden field {needle!r}" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_output_review_kit" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + + scorecard_json = tmp_root / "output_quality_scorecard.json" + scorecard_md = tmp_root / "output_quality_scorecard.md" + blind_pack_json = tmp_root / "output_blind_review_pack.json" + blind_pack_md = tmp_root / "output_blind_review_pack.md" + answer_key_json = tmp_root / "output_blind_answer_key.json" + + run( + [ + str(OUTPUT_EVAL), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(scorecard_json), + "--output-md", + str(scorecard_md), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--blind-answer-key-json", + str(answer_key_json), + ] + ) + + decisions_path = tmp_root / "output_review_decisions.json" + kit_json = tmp_root / "output_review_kit.json" + kit_md = tmp_root / "output_review_kit.md" + kit_html = tmp_root / "output_review_kit.html" + pending_proc = run( + [ + str(REVIEW_KIT), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(decisions_path), + "--output-json", + str(kit_json), + "--output-md", + str(kit_md), + "--output-html", + str(kit_html), + "--write-template", + ] + ) + pending_payload = json.loads(pending_proc.stdout) + assert pending_payload["ok"], pending_payload + assert pending_payload["template_written"], pending_payload + assert pending_payload["summary"]["case_count"] == 5, pending_payload + assert pending_payload["summary"]["pending_decision_count"] == 5, pending_payload + assert pending_payload["summary"]["ready_for_adjudication_count"] == 0, pending_payload + assert pending_payload["summary"]["answer_key_hidden"] is True, pending_payload + assert pending_payload["summary"]["answer_key_path_exposed"] is False, pending_payload + assert pending_payload["summary"]["ready_to_run_adjudication"] is False, pending_payload + assert pending_payload["review_contract"]["required_fields"]["reason"].startswith("Required rationale"), pending_payload + assert "answer_key" not in pending_payload["artifacts"], pending_payload["artifacts"] + assert pending_payload["artifacts"]["reviewer_kit_html"].endswith("tests/tmp_output_review_kit/output_review_kit.html"), pending_payload["artifacts"] + assert len(pending_payload["cases"]) == 5, pending_payload["cases"] + first_case = pending_payload["cases"][0] + assert first_case["variant_a"]["output"], first_case + assert first_case["variant_b"]["output"], first_case + assert first_case["decision_state"]["status"] == "awaiting-decision", first_case + assert decisions_path.exists(), decisions_path + + kit_json_text = kit_json.read_text(encoding="utf-8") + kit_md_text = kit_md.read_text(encoding="utf-8") + kit_html_text = kit_html.read_text(encoding="utf-8") + assert "# Output Review Kit" in kit_md_text, kit_md_text + assert "answer key path exposed: `false`" in kit_md_text, kit_md_text + assert "Variant A" in kit_md_text and "Variant B" in kit_md_text, kit_md_text + assert "python3 scripts/yao.py output-review" in kit_md_text, kit_md_text + assert "Output Review Kit" in kit_html_text, kit_html_text + assert "Reviewer cockpit for output quality decisions" in kit_html_text, kit_html_text + assert "Ready0" in kit_html_text, kit_html_text + assert "Invalid0" in kit_html_text, kit_html_text + assert "Variant A" in kit_html_text and "Variant B" in kit_html_text, kit_html_text + assert "Decision Template" in kit_html_text, kit_html_text + assert_no_answer_key_leak(kit_json_text) + assert_no_answer_key_leak(kit_md_text) + assert_no_answer_key_leak(kit_html_text) + + cli_proc = run( + [ + str(CLI), + "output-review-kit", + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(decisions_path), + "--output-json", + str(tmp_root / "cli_output_review_kit.json"), + "--output-md", + str(tmp_root / "cli_output_review_kit.md"), + "--output-html", + str(tmp_root / "cli_output_review_kit.html"), + ] + ) + cli_payload = json.loads(cli_proc.stdout) + assert cli_payload["ok"], cli_payload + assert cli_payload["summary"]["case_count"] == 5, cli_payload + assert cli_payload["summary"]["answer_key_hidden"] is True, cli_payload + assert "answer_key" not in cli_payload["artifacts"], cli_payload["artifacts"] + assert (tmp_root / "cli_output_review_kit.html").exists(), cli_payload + + template = json.loads(decisions_path.read_text(encoding="utf-8")) + filled = { + "schema_version": "1.0", + "reviewer": "Yao QA", + "reviewed_at": "2026-06-14", + "decisions": [ + { + "case_id": item["case_id"], + "winner_variant": "A", + "confidence": 0.8, + "reason": "Variant A is clearer against the visible rubric.", + } + for item in template["decisions"] + ], + } + decisions_path.write_text(json.dumps(filled, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + ready_proc = run( + [ + str(REVIEW_KIT), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(decisions_path), + "--output-json", + str(tmp_root / "ready_output_review_kit.json"), + "--output-md", + str(tmp_root / "ready_output_review_kit.md"), + "--output-html", + str(tmp_root / "ready_output_review_kit.html"), + ] + ) + ready_payload = json.loads(ready_proc.stdout) + assert ready_payload["ok"], ready_payload + assert ready_payload["summary"]["ready_for_adjudication_count"] == 5, ready_payload + assert ready_payload["summary"]["pending_decision_count"] == 0, ready_payload + assert ready_payload["summary"]["invalid_decision_count"] == 0, ready_payload + assert ready_payload["summary"]["reviewer_metadata_present"] is True, ready_payload + assert ready_payload["summary"]["ready_to_run_adjudication"] is True, ready_payload + assert all(case["decision_state"]["status"] == "ready-for-adjudication" for case in ready_payload["cases"]), ready_payload + + invalid = dict(filled) + invalid["decisions"] = [dict(filled["decisions"][0], winner_variant="C", reason="")] + invalid_path = tmp_root / "invalid_output_review_decisions.json" + invalid_path.write_text(json.dumps(invalid, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + invalid_proc = run( + [ + str(REVIEW_KIT), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--decisions", + str(invalid_path), + "--output-json", + str(tmp_root / "invalid_output_review_kit.json"), + "--output-md", + str(tmp_root / "invalid_output_review_kit.md"), + "--output-html", + str(tmp_root / "invalid_output_review_kit.html"), + ] + ) + invalid_payload = json.loads(invalid_proc.stdout) + assert invalid_payload["ok"], invalid_payload + assert invalid_payload["summary"]["invalid_decision_count"] == 1, invalid_payload + assert invalid_payload["summary"]["answer_key_hidden"] is True, invalid_payload + assert invalid_payload["cases"][0]["decision_state"]["status"] == "needs-fix", invalid_payload["cases"][0] + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_package_verification.py b/tests/verify_package_verification.py index 4d519076..b0f8138c 100644 --- a/tests/verify_package_verification.py +++ b/tests/verify_package_verification.py @@ -3,6 +3,7 @@ import json import shutil import subprocess import sys +import tempfile import zipfile from pathlib import Path @@ -28,18 +29,20 @@ def run(cmd: list[str]) -> dict: } -def build_package(out_dir: Path) -> dict: +def build_package(out_dir: Path, skill_root: Path = ROOT) -> dict: return run( [ sys.executable, str(PACKAGER), - str(ROOT), + str(skill_root), "--platform", "openai", "--platform", "claude", "--platform", "generic", + "--platform", + "vscode", "--expectations", str(EXPECTATIONS), "--output-dir", @@ -49,12 +52,12 @@ def build_package(out_dir: Path) -> dict: ) -def verify_package(out_dir: Path, output_json: Path, output_md: Path) -> dict: +def verify_package(out_dir: Path, output_json: Path, output_md: Path, skill_root: Path = ROOT) -> dict: return run( [ sys.executable, str(VERIFIER), - str(ROOT), + str(skill_root), "--package-dir", str(out_dir), "--expectations", @@ -84,13 +87,30 @@ def main() -> None: payload = valid["payload"] assert valid["ok"], valid assert payload["ok"], payload - assert payload["summary"]["target_count"] == 3, payload - assert payload["summary"]["adapter_count"] == 3, payload + assert payload["summary"]["target_count"] == 4, payload + assert payload["summary"]["adapter_count"] == 4, payload assert payload["summary"]["archive_present"], payload assert payload["summary"]["archive_sha256"], payload assert not payload["failures"], payload assert (TMP / "package_verification.md").exists(), TMP + with tempfile.TemporaryDirectory(prefix="renamed-package-root-") as temp_root: + renamed_root = Path(temp_root) / "checkout-alias" + shutil.copytree( + ROOT, + renamed_root, + ignore=shutil.ignore_patterns(".git", ".previews", "dist", "__pycache__", ".pytest_cache", "tmp*"), + ) + renamed_dir = TMP / "renamed-dist" + renamed_build = build_package(renamed_dir, renamed_root) + assert renamed_build["ok"], renamed_build + assert (renamed_dir / "yao-meta-skill.zip").exists(), renamed_build + with zipfile.ZipFile(renamed_dir / "yao-meta-skill.zip") as archive: + names = set(archive.namelist()) + assert "yao-meta-skill/SKILL.md" in names, sorted(list(names))[:10] + renamed_valid = verify_package(renamed_dir, TMP / "renamed_package_verification.json", TMP / "renamed_package_verification.md", renamed_root) + assert renamed_valid["ok"], renamed_valid + unsafe_dir = TMP / "unsafe-dist" shutil.copytree(valid_dir, unsafe_dir) with zipfile.ZipFile(unsafe_dir / "yao-meta-skill.zip", "a", compression=zipfile.ZIP_DEFLATED) as archive: diff --git a/tests/verify_portability_report.py b/tests/verify_portability_report.py index 7a73e351..3e1a7d1d 100644 --- a/tests/verify_portability_report.py +++ b/tests/verify_portability_report.py @@ -2,26 +2,51 @@ import json import subprocess import sys +import tempfile from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_portability_report.py" +sys.path.insert(0, str(ROOT / "scripts")) + +from render_portability_report import find_ir def main() -> None: - proc = subprocess.run( - [sys.executable, str(SCRIPT)], - cwd=ROOT, - capture_output=True, - text=True, - ) - if proc.returncode != 0: - print(proc.stdout) - print(proc.stderr) - raise SystemExit(proc.returncode) + with tempfile.TemporaryDirectory(prefix="renamed-skill-root-") as tmp: + tmp_root = Path(tmp) + ir_dir = tmp_root / "skill-ir" / "examples" + ir_dir.mkdir(parents=True) + ir_path = ir_dir / "yao-meta-skill.json" + ir_path.write_text( + json.dumps({"schema_version": "2.0.0", "job_to_be_done": "test renamed checkout"}), + encoding="utf-8", + ) + ir_payload, ir_source = find_ir(tmp_root) + assert ir_payload["schema_version"] == "2.0.0", ir_payload + assert ir_source == "skill-ir/examples/yao-meta-skill.json", ir_source - payload = json.loads(proc.stdout) + payload = None + for args in ([], [str(ROOT)]): + proc = subprocess.run( + [sys.executable, str(SCRIPT), *args], + cwd=ROOT, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(proc.stdout) + print(proc.stderr) + raise SystemExit(proc.returncode) + current_payload = json.loads(proc.stdout) + if payload is None: + payload = current_payload + elif current_payload != payload: + print(json.dumps({"default": payload, "explicit": current_payload}, ensure_ascii=False, indent=2)) + raise SystemExit(2) + + assert payload is not None failures = [] if payload.get("score", 0) < 95: failures.append(f"portability score too low: {payload.get('score')}") diff --git a/tests/verify_python_compat_check.py b/tests/verify_python_compat_check.py new file mode 100644 index 00000000..353e29e7 --- /dev/null +++ b/tests/verify_python_compat_check.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "python_compat_check.py" +TMP = ROOT / "tests" / "tmp_python_compat" + + +def run_check(*extra: str, check: bool = True, cwd: Path = ROOT) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra, + ], + cwd=cwd, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "python_compatibility.json" + output_md = TMP / "python_compatibility.md" + proc = run_check("--output-json", str(output_json), "--output-md", str(output_md)) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["target_python"] == "3.11", payload + assert payload["summary"]["file_count"] >= 50, payload["summary"] + assert payload["summary"]["issue_count"] == 0, payload["summary"] + assert payload["summary"]["fstring_311_violation_count"] == 0, payload["summary"] + markdown = output_md.read_text(encoding="utf-8") + assert "Python Compatibility" in markdown, markdown + assert "f-string 3.11 violations" in markdown, markdown + + safe = TMP / "safe.py" + safe.write_text( + "value = 'x|y'\n" + "escaped = value.replace('|', '\\\\|')\n" + "result = f'{escaped}'\n", + encoding="utf-8", + ) + safe_proc = run_check( + "--path", + safe.relative_to(ROOT).as_posix(), + "--output-json", + str(TMP / "safe_python_compatibility.json"), + "--output-md", + str(TMP / "safe_python_compatibility.md"), + check=True, + cwd=TMP, + ) + safe_payload = json.loads(safe_proc.stdout) + assert safe_payload["summary"]["file_count"] == 1, safe_payload + assert safe_payload["summary"]["issue_count"] == 0, safe_payload + + unsafe = TMP / "unsafe.py" + unsafe.write_text( + "value = 'x|y'\n" + "result = f\"{value.replace('|', '\\\\|')}\"\n", + encoding="utf-8", + ) + unsafe_proc = run_check( + "--path", + unsafe.relative_to(ROOT).as_posix(), + "--output-json", + str(TMP / "unsafe_python_compatibility.json"), + "--output-md", + str(TMP / "unsafe_python_compatibility.md"), + check=False, + cwd=TMP, + ) + assert unsafe_proc.returncode == 2, unsafe_proc.stdout + unsafe_payload = json.loads(unsafe_proc.stdout) + assert unsafe_payload["ok"] is False, unsafe_payload + assert unsafe_payload["summary"]["decision"] == "block-python-compat", unsafe_payload + assert unsafe_payload["summary"]["file_count"] == 1, unsafe_payload + assert unsafe_payload["summary"]["fstring_311_violation_count"] >= 1, unsafe_payload + assert any(item["rule"] == "fstring-expression-backslash" for item in unsafe_payload["issues"]), unsafe_payload["issues"] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_quality_checks.py b/tests/verify_quality_checks.py index d290b80c..e9c1cca8 100644 --- a/tests/verify_quality_checks.py +++ b/tests/verify_quality_checks.py @@ -3,6 +3,7 @@ import json import subprocess import sys from pathlib import Path +from tempfile import TemporaryDirectory ROOT = Path(__file__).resolve().parent.parent @@ -43,22 +44,162 @@ def require_context_targets(case: dict, max_initial: int, min_density: float) -> stats = case.get("payload", {}).get("stats", {}) initial = stats.get("estimated_initial_load_tokens") density = stats.get("quality_density") + deferred = stats.get("deferred_resource_tokens") + deferred_threshold = stats.get("deferred_resource_warn_threshold") + deferred_dirs = stats.get("deferred_resource_dirs", []) + deferred_governance = stats.get("deferred_resource_governance", {}) unused = stats.get("unused_resource_dirs", []) case["max_initial_load"] = max_initial case["observed_initial_load"] = initial case["minimum_quality_density"] = min_density case["observed_quality_density"] = density + case["observed_deferred_resource_tokens"] = deferred + case["deferred_resource_warn_threshold"] = deferred_threshold + case["deferred_resource_dir_count"] = len(deferred_dirs) if isinstance(deferred_dirs, list) else None + case["deferred_resource_governance_status"] = deferred_governance.get("status") if isinstance(deferred_governance, dict) else None case["unused_resource_dirs"] = unused case["passed"] = ( case["passed"] and initial is not None and density is not None + and deferred is not None + and deferred_threshold is not None + and isinstance(deferred_dirs, list) + and isinstance(deferred_governance, dict) + and deferred_governance.get("status") in {"governed", "not-required", "needs-review"} and initial <= max_initial and density >= min_density and not unused ) +def verify_private_report_ignored() -> dict: + with TemporaryDirectory(prefix="yao-private-report-ignore-") as temp_dir: + skill_dir = Path(temp_dir) / "skill" + (skill_dir / "reports").mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\n" + "name: report-ignore-fixture\n" + "description: Test fixture for ignored private reports.\n" + "---\n" + "Use this fixture to validate context sizing.\n", + encoding="utf-8", + ) + (skill_dir / "reports" / "kept.md").write_text("stable evidence\n", encoding="utf-8") + baseline = run( + "private_report_ignore_baseline", + [sys.executable, "scripts/context_sizer.py", str(skill_dir), "--json"], + ) + (skill_dir / "reports" / "client-research-plan.md").write_text( + "private notes\n" * 500, + encoding="utf-8", + ) + context_case = run( + "private_report_ignore_context_sizer", + [sys.executable, "scripts/context_sizer.py", str(skill_dir), "--json"], + ) + boundary_case = run( + "private_report_ignore_resource_boundary", + [sys.executable, "scripts/resource_boundary_check.py", str(skill_dir), "--max-initial-tokens", "5000"], + ) + + baseline_total = baseline.get("payload", {}).get("estimated_total_text_tokens") + context_total = context_case.get("payload", {}).get("estimated_total_text_tokens") + boundary_files = boundary_case.get("payload", {}).get("stats", {}).get("relevant_file_count") + passed = ( + baseline["passed"] + and context_case["passed"] + and boundary_case["passed"] + and baseline_total == context_total + and boundary_files == 2 + ) + return { + "name": "private_report_ignore", + "passed": passed, + "baseline_total_tokens": baseline_total, + "observed_total_tokens": context_total, + "resource_relevant_file_count": boundary_files, + "context_stdout": context_case["stdout"], + "boundary_stdout": boundary_case["stdout"], + } + + +def verify_volatile_report_outputs_ignored() -> dict: + with TemporaryDirectory(prefix="yao-volatile-report-ignore-") as temp_dir: + skill_dir = Path(temp_dir) / "skill" + (skill_dir / "reports").mkdir(parents=True) + (skill_dir / "SKILL.md").write_text( + "---\n" + "name: context-budget-ignore-fixture\n" + "description: Test fixture for self-generated context reports.\n" + "---\n" + "Use this fixture to validate context report idempotency.\n", + encoding="utf-8", + ) + (skill_dir / "reports" / "kept.md").write_text("stable evidence\n", encoding="utf-8") + baseline = run( + "volatile_report_ignore_baseline", + [sys.executable, "scripts/context_sizer.py", str(skill_dir), "--json"], + ) + volatile_reports = [ + "benchmark_reproducibility.json", + "benchmark_reproducibility.md", + "context_budget.json", + "context_budget_summary.json", + "context_budget.md", + "evidence_consistency.json", + "evidence_consistency.md", + "review-studio.html", + "review-studio.json", + "review-viewer.html", + "review-viewer.json", + "skill-interpretation.html", + "skill-interpretation.json", + "skill-overview.html", + "skill-overview.json", + "world_class_evidence_preflight.json", + "world_class_evidence_preflight.md", + "world_class_evidence_preflight.html", + ] + for report_name in volatile_reports: + report_path = skill_dir / "reports" / report_name + if report_path.suffix == ".json": + report_path.write_text( + json.dumps({"large": f"{report_name}\n" * 2000}, ensure_ascii=False), + encoding="utf-8", + ) + else: + report_path.write_text(f"{report_name}\n" * 2000, encoding="utf-8") + context_case = run( + "volatile_report_ignore_context_sizer", + [sys.executable, "scripts/context_sizer.py", str(skill_dir), "--json"], + ) + boundary_case = run( + "volatile_report_ignore_resource_boundary", + [sys.executable, "scripts/resource_boundary_check.py", str(skill_dir), "--max-initial-tokens", "5000"], + ) + + baseline_total = baseline.get("payload", {}).get("estimated_total_text_tokens") + context_total = context_case.get("payload", {}).get("estimated_total_text_tokens") + boundary_files = boundary_case.get("payload", {}).get("stats", {}).get("relevant_file_count") + passed = ( + baseline["passed"] + and context_case["passed"] + and boundary_case["passed"] + and baseline_total == context_total + and boundary_files == 2 + ) + return { + "name": "volatile_report_outputs_ignored", + "passed": passed, + "baseline_total_tokens": baseline_total, + "observed_total_tokens": context_total, + "resource_relevant_file_count": boundary_files, + "context_stdout": context_case["stdout"], + "boundary_stdout": boundary_case["stdout"], + } + + def main() -> None: python = sys.executable cases = [] @@ -75,6 +216,13 @@ def main() -> None: [python, "scripts/resource_boundary_check.py", str(ROOT)], ) require_context_targets(root_resource, 1000, 100.0) + root_payload = root_resource.get("payload", {}) + root_governance_status = root_resource.get("deferred_resource_governance_status") + root_resource["passed"] = ( + root_resource["passed"] + and root_governance_status == "governed" + and not any("Deferred resource footprint is high" in item for item in root_payload.get("warnings", [])) + ) cases.append(root_resource) complex_resource = run( @@ -115,6 +263,8 @@ def main() -> None: require_min_score(governed_example, 90) cases.insert(4, governed_example) cases.insert(5, governed_resource) + cases.append(verify_private_report_ignored()) + cases.append(verify_volatile_report_outputs_ignored()) report = {"ok": all(case["passed"] for case in cases), "cases": cases} print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/tests/verify_registry_audit.py b/tests/verify_registry_audit.py index a0f8d3db..7e769dfa 100644 --- a/tests/verify_registry_audit.py +++ b/tests/verify_registry_audit.py @@ -59,6 +59,7 @@ def main() -> None: assert package["compatibility"]["openai"] == "pass", package assert package["compatibility"]["claude"] == "pass", package assert package["compatibility"]["generic"] == "pass", package + assert package["compatibility"]["vscode"] == "pass", package assert (registry_dir / "index.json").exists(), registry_dir assert (registry_dir / "packages" / "yao-meta-skill.json").exists(), registry_dir assert output_json.exists(), output_json diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 23cab6fc..1cacb83a 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -1,270 +1,122 @@ #!/usr/bin/env python3 import json -import shutil -import subprocess +import re import sys -from pathlib import Path -ROOT = Path(__file__).resolve().parent.parent -SCRIPT = ROOT / "scripts" / "render_review_studio.py" +from review_studio_helpers import ROOT, prepare_tmp_root, render_review_studio_fixture +from review_studio_world_class_assertions import assert_world_class_action + sys.path.insert(0, str(ROOT / "scripts")) import render_review_studio as review_studio # noqa: E402 import review_studio_formatting as review_formatting # noqa: E402 +import review_studio_gates as review_gates # noqa: E402 import review_studio_layout as review_layout # noqa: E402 def main() -> None: - tmp_root = ROOT / "tests" / "tmp_review_studio" - if tmp_root.exists(): - shutil.rmtree(tmp_root) - tmp_root.mkdir(parents=True, exist_ok=True) - subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "run_output_execution.py"), - "--runner-command", - json.dumps(["python3", "scripts/local_output_eval_runner.py"]), - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "adjudicate_output_review.py")], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "compile_skill.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - package_dir = tmp_root / "dist" - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "cross_packager.py"), - str(ROOT), - "--platform", - "openai", - "--platform", - "claude", - "--platform", - "generic", - "--expectations", - str(ROOT / "evals" / "packaging_expectations.json"), - "--output-dir", - str(package_dir), - "--zip", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "simulate_install.py"), - str(ROOT), - "--package-dir", - str(package_dir), - "--install-root", - str(tmp_root / "install-root"), - "--output-json", - str(ROOT / "reports" / "install_simulation.json"), - "--output-md", - str(ROOT / "reports" / "install_simulation.md"), - "--generated-at", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "registry_audit.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "render_intent_confidence.py"), str(ROOT)], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "build_skill_atlas.py"), - "--workspace-root", - str(ROOT), - "--output-dir", - str(ROOT / "skill_atlas"), - "--report-html", - str(ROOT / "reports" / "skill_atlas.html"), - "--report-json", - str(ROOT / "reports" / "skill_atlas.json"), - "--today", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "upgrade_check.py"), - str(ROOT), - "--previous-package-json", - str(ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json"), - "--current-package-json", - str(ROOT / "reports" / "registry_audit.json"), - "--output-json", - str(tmp_root / "upgrade_check.json"), - "--output-md", - str(tmp_root / "upgrade_check.md"), - "--generated-at", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - (ROOT / "reports" / "upgrade_check.json").write_text( - (tmp_root / "upgrade_check.json").read_text(encoding="utf-8"), - encoding="utf-8", - ) - (ROOT / "reports" / "upgrade_check.md").write_text( - (tmp_root / "upgrade_check.md").read_text(encoding="utf-8"), - encoding="utf-8", - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "render_adoption_drift_report.py"), - str(ROOT), - "--events-jsonl", - str(tmp_root / "telemetry_events.jsonl"), - "--record-event", - "skill_activation", - "--activation-type", - "explicit", - "--outcome", - "accepted", - "--timestamp", - "2026-06-13T10:00:00Z", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "render_review_waivers.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "render_review_annotations.py"), - str(ROOT), - "--annotations-json", - str(tmp_root / "empty_review_annotations_input.json"), - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "probe_runtime_permissions.py"), - str(ROOT), - "--package-dir", - str(package_dir), - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - - output_html = tmp_root / "review-studio.html" - output_json = tmp_root / "review-studio.json" - proc = subprocess.run( - [ - sys.executable, - str(SCRIPT), - str(ROOT), - "--output-html", - str(output_html), - "--output-json", - str(output_json), - ], - cwd=ROOT, - capture_output=True, - text=True, - check=True, - ) + tmp_root = prepare_tmp_root() + output_html, output_json, proc = render_review_studio_fixture(tmp_root) payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["schema_version"] == "2.0", payload - assert payload["summary"]["decision"] == "ready", payload - assert payload["summary"]["gate_count"] == 13, payload - assert payload["summary"]["world_class_score"] == 100, payload - assert payload["summary"]["warning_count"] == 0, payload + assert payload["summary"]["decision"] == "review", payload + assert payload["summary"]["gate_count"] == 16, payload + assert payload["summary"]["gate_contract_ok"] is True, payload + assert payload["summary"]["world_class_score"] == 91, payload + assert payload["summary"]["warning_count"] == 3, payload assert payload["summary"]["blocker_count"] == 0, payload - assert payload["summary"]["action_count"] == 0, payload + assert payload["summary"]["action_count"] == 3, payload assert payload["summary"]["annotation_count"] == 0, payload assert payload["summary"]["open_annotation_blocker_count"] == 0, payload assert payload["summary"]["action_count"] == payload["summary"]["warning_count"] + payload["summary"]["blocker_count"], payload - assert payload["review_actions"] == [], payload + assert {item["gate_key"] for item in payload["review_actions"]} == { + "output-lab", + "review-waivers", + "world-class-evidence", + }, payload + action_by_gate = {item["gate_key"]: item for item in payload["review_actions"]} gate_keys = {item["key"] for item in payload["gates"]} - assert {"intent-canvas", "trigger-lab", "output-lab", "runtime-matrix", "trust-report", "permission-gates", "permission-runtime", "skill-atlas", "operations-loop", "review-waivers", "registry-audit", "release-notes"} <= gate_keys, payload + assert gate_keys == review_gates.REVIEW_STUDIO_GATE_KEYS, payload + for gate in payload["gates"]: + if gate["status"] == "pass": + assert gate["review_action_id"] == "", gate + assert gate["review_action_title"] == "", gate + assert gate["review_action_next_step"] == "", gate + assert gate["review_action_reason"] == "", gate + assert gate["review_action_source_ref_count"] == 0, gate + assert gate["review_action_verification_command"] == "", gate + continue + action = action_by_gate[gate["key"]] + assert gate["review_action_id"] == action["action_id"] == f"review-action-{gate['key']}", gate + assert gate["review_action_status"] == action["status"] == gate["status"], gate + assert gate["review_action_title"] == action["title"] == action["label"], gate + assert gate["review_action_next_step"] == action["next_step"] == action["summary"], gate + assert gate["review_action_reason"] == action["reason"] == action["why"], gate + assert gate["review_action_source_ref_count"] == len(action["source_refs"]) > 0, gate + assert gate["review_action_verification_command"] == action["verification_command"], gate + assert set(review_gates.GATE_WEIGHTS) == review_gates.REVIEW_STUDIO_GATE_KEYS, review_gates.GATE_WEIGHTS + gate_contract = payload["gate_contract"] + assert gate_contract["ok"] is True, gate_contract + assert gate_contract["expected_gate_count"] == 16, gate_contract + assert gate_contract["actual_gate_count"] == 16, gate_contract + assert set(gate_contract["expected_gate_keys"]) == review_gates.REVIEW_STUDIO_GATE_KEYS, gate_contract + assert set(gate_contract["rendered_gate_keys"]) == review_gates.REVIEW_STUDIO_GATE_KEYS, gate_contract + assert gate_contract["missing_gate_keys"] == [], gate_contract + assert gate_contract["unknown_gate_keys"] == [], gate_contract + assert gate_contract["duplicate_gate_keys"] == [], gate_contract + assert gate_contract["unweighted_gate_keys"] == [], gate_contract output_gate = next(item for item in payload["gates"] if item["key"] == "output-lab") - assert output_gate["status"] == "pass", output_gate + assert output_gate["status"] == "warn", output_gate assert "5/5 cases" in output_gate["detail"], output_gate assert "file-backed 1" in output_gate["detail"], output_gate assert "blind A/B 5" in output_gate["detail"], output_gate assert "exec 10" in output_gate["detail"], output_gate - assert "model 0" in output_gate["detail"], output_gate + assert "model 10" in output_gate["detail"], output_gate assert "reviewed 0/5" in output_gate["detail"], output_gate + assert "review pending 5" in output_gate["detail"], output_gate + context_gate = next(item for item in payload["gates"] if item["key"] == "context-budget") + assert context_gate["status"] == "pass", context_gate + initial_load = re.search(r"initial load (\d+)/1000", context_gate["detail"]) + assert initial_load, context_gate + assert int(initial_load.group(1)) <= 1000, context_gate + assert "deferred " in context_gate["detail"], context_gate + assert "/120000" in context_gate["detail"], context_gate + assert "top deferred scripts" in context_gate["detail"], context_gate + assert "resource governance governed" in context_gate["detail"], context_gate + assert "quality density" in context_gate["detail"], context_gate release_gate = next(item for item in payload["gates"] if item["key"] == "release-notes") assert "upgrade minor declared / minor recommended" in release_gate["detail"], release_gate assert "reports/upgrade_check.json" in release_gate["evidence"], release_gate registry_gate = next(item for item in payload["gates"] if item["key"] == "registry-audit") assert "install pass" in registry_gate["detail"], registry_gate + assert "installer permissions 12 enforced / 0 failures" in registry_gate["detail"], registry_gate assert "reports/install_simulation.json" in registry_gate["evidence"], registry_gate trust_gate = next(item for item in payload["gates"] if item["key"] == "trust-report") assert trust_gate["status"] == "pass", trust_gate - assert "2 network-capable scripts" in trust_gate["detail"], trust_gate + assert "3 network-capable scripts" in trust_gate["detail"], trust_gate assert "0 help smoke failures" in trust_gate["detail"], trust_gate + python_compat_gate = next(item for item in payload["gates"] if item["key"] == "python-compat") + assert python_compat_gate["status"] == "pass", python_compat_gate + assert "Python 3.11" in python_compat_gate["detail"], python_compat_gate + assert "0 compatibility issues" in python_compat_gate["detail"], python_compat_gate + assert "0 f-string 3.11 hazards" in python_compat_gate["detail"], python_compat_gate + assert python_compat_gate["evidence"] == "reports/python_compatibility.json", python_compat_gate + architecture_gate = next(item for item in payload["gates"] if item["key"] == "architecture-maintainability") + assert architecture_gate["status"] == "pass", architecture_gate + assert "0 hotspots" in architecture_gate["detail"], architecture_gate + assert re.search(r"\d+ watchlist files?", architecture_gate["detail"]), architecture_gate + assert re.search(r"\d+ early watch files?", architecture_gate["detail"]), architecture_gate + assert "0 blockers" in architecture_gate["detail"], architecture_gate + assert "CLI handlers" in architecture_gate["detail"], architecture_gate + assert architecture_gate["evidence"] == "reports/architecture_maintainability.json", architecture_gate permission_gate = next(item for item in payload["gates"] if item["key"] == "permission-gates") assert permission_gate["status"] == "pass", permission_gate assert "permissions approved" in permission_gate["detail"], permission_gate assert "gaps 0" in permission_gate["detail"], permission_gate permission_runtime_gate = next(item for item in payload["gates"] if item["key"] == "permission-runtime") assert permission_runtime_gate["status"] == "pass", permission_runtime_gate - assert "metadata fallback 3" in permission_runtime_gate["detail"], permission_runtime_gate + assert "metadata fallback 4" in permission_runtime_gate["detail"], permission_runtime_gate assert "native 0" in permission_runtime_gate["detail"], permission_runtime_gate intent_gate = next(item for item in payload["gates"] if item["key"] == "intent-canvas") assert intent_gate["status"] == "pass", intent_gate @@ -272,36 +124,208 @@ def main() -> None: atlas_gate = next(item for item in payload["gates"] if item["key"] == "skill-atlas") assert atlas_gate["status"] == "pass", atlas_gate assert "actionable route collisions" in atlas_gate["detail"], atlas_gate + assert "actionable drift" in atlas_gate["detail"], atlas_gate operations_gate = next(item for item in payload["gates"] if item["key"] == "operations-loop") assert operations_gate["status"] == "pass", operations_gate assert "metadata events" in operations_gate["detail"], operations_gate assert "risk low" in operations_gate["detail"], operations_gate + assert "daily proposals" in operations_gate["detail"], operations_gate + assert "weekly queue" in operations_gate["detail"], operations_gate assert "reports/adoption_drift_report.json" in operations_gate["evidence"], operations_gate + assert "reports/skillops/daily" in operations_gate["evidence"], operations_gate + assert "reports/skillops/weekly" in operations_gate["evidence"], operations_gate waivers_gate = next(item for item in payload["gates"] if item["key"] == "review-waivers") - assert waivers_gate["status"] == "pass", waivers_gate - assert "cover current warnings" in waivers_gate["detail"], waivers_gate + assert waivers_gate["status"] == "warn", waivers_gate + assert "1 warning gates still need reviewer decision" in waivers_gate["detail"], waivers_gate assert "reports/review_waivers.json" in waivers_gate["evidence"], waivers_gate + world_class_gate = next(item for item in payload["gates"] if item["key"] == "world-class-evidence") + world_class_summary = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8"))[ + "summary" + ] + assert world_class_gate["status"] == "warn", world_class_gate + assert "4 pending world-class evidence entries" in world_class_gate["detail"], world_class_gate + assert "1 human pending" in world_class_gate["detail"], world_class_gate + assert "3 external pending" in world_class_gate["detail"], world_class_gate + assert ( + f"source checks {world_class_summary['source_pass_count']}/{world_class_summary['source_check_count']} pass" + in world_class_gate["detail"] + ), world_class_gate + assert f"{world_class_summary['source_blocked_count']} blocked" in world_class_gate["detail"], world_class_gate + assert "overclaim guard true" in world_class_gate["detail"], world_class_gate + assert world_class_gate["evidence"] == "reports/world_class_evidence_ledger.json", world_class_gate assert output_html.exists(), output_html assert output_json.exists(), output_json full_payload = json.loads(output_json.read_text(encoding="utf-8")) + assert full_payload["evidence_paths"]["skill_ir"] == "skill-ir/examples/yao-meta-skill.json", full_payload[ + "evidence_paths" + ] assert full_payload["evidence_paths"]["compiled_targets"] == "reports/compiled_targets.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_execution"] == "reports/output_execution_runs.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_blind_review"] == "reports/output_blind_review_pack.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["output_review_kit_html"] == "reports/output_review_kit.html", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["output_review_decisions"] == "reports/output_review_decisions.json", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_review_adjudication"] == "reports/output_review_adjudication.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["python_compatibility"] == "reports/python_compatibility.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["architecture_maintainability"] == "reports/architecture_maintainability.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "benchmark_reproducibility.md").exists(): + assert full_payload["evidence_paths"]["benchmark_reproducibility"] == "reports/benchmark_reproducibility.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "skill_os2_coverage.md").exists(): + assert full_payload["evidence_paths"]["skill_os2_coverage"] == "reports/skill_os2_coverage.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["daily_skillops"].startswith("reports/skillops/daily/"), full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["weekly_curator"].startswith("reports/skillops/weekly/"), full_payload["evidence_paths"] assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["adaptation_proposals"] == "reports/adaptation_proposals.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["adaptation_approval_ledger"] == "reports/adaptation_approval_ledger.json", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["adaptation_regression"] == "reports/adaptation_regression_report.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_plan.md").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_plan"] == "reports/world_class_evidence_plan.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_ledger.md").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_ledger"] == "reports/world_class_evidence_ledger.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_intake.md").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_intake"] == "reports/world_class_evidence_intake.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_preflight.md").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_preflight"] == "reports/world_class_evidence_preflight.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_preflight.html").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_preflight_html"] == "reports/world_class_evidence_preflight.html", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_submission_review.md").exists(): + assert full_payload["evidence_paths"]["world_class_submission_review"] == "reports/world_class_submission_review.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_operator_runbook.md").exists(): + assert full_payload["evidence_paths"]["world_class_operator_runbook"] == "reports/world_class_operator_runbook.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_operator_runbook.html").exists(): + assert full_payload["evidence_paths"]["world_class_operator_runbook_html"] == "reports/world_class_operator_runbook.html", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_claim_guard.md").exists(): + assert full_payload["evidence_paths"]["world_class_claim_guard"] == "reports/world_class_claim_guard.md", full_payload["evidence_paths"] assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"] assert full_payload["data"]["output_execution"]["summary"]["command_executed_count"] == 10, full_payload["data"]["output_execution"] assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"] - assert full_payload["data"]["output_execution"]["summary"]["model_executed_count"] == 0, full_payload["data"]["output_execution"] + assert full_payload["data"]["output_execution"]["summary"]["model_executed_count"] == 10, full_payload["data"]["output_execution"] + assert full_payload["data"]["output_execution"]["summary"]["token_observed_count"] == 10, full_payload["data"]["output_execution"] + assert full_payload["data"]["output_execution"]["summary"]["token_estimated_count"] == 0, full_payload["data"]["output_execution"] assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["answer_revealed_count"] == 0, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_answer_hidden_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["reviewer_checklist_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["reviewer_checklist_pending_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["reviewer_checklist_ready_count"] == 0, full_payload["data"]["output_review_adjudication"] + assert all(not item["expected_revealed"] for item in full_payload["data"]["output_review_adjudication"]["pairs"]), full_payload["data"]["output_review_adjudication"] + benchmark_summary = full_payload["data"]["benchmark_reproducibility"]["summary"] + assert benchmark_summary["reproducibility_ready"] is True, benchmark_summary + assert benchmark_summary["release_lock_ready"] == (benchmark_summary["source_tree_dirty"] is False), benchmark_summary + assert "generated_tree_dirty" in benchmark_summary, benchmark_summary + expected_beta_ready = ( + benchmark_summary["reproducibility_ready"] + and benchmark_summary["release_lock_ready"] + and benchmark_summary["provider_evidence_complete"] + ) + assert benchmark_summary["beta_test_ready"] is expected_beta_ready, benchmark_summary + assert benchmark_summary["beta_test_deferred_evidence_count"] >= 1, benchmark_summary + beta_release = full_payload["data"]["benchmark_reproducibility"]["beta_test_release"] + assert beta_release["ready"] is expected_beta_ready, beta_release + assert not any("human blind-review" in item for item in beta_release["blockers"]), beta_release + pending_ledger_keys = { + item["key"] + for item in full_payload["data"]["world_class_evidence_ledger"]["entries"] + if item.get("status") == "pending" + } + deferred_keys = {item["key"] for item in beta_release["allowed_deferred_evidence"]} + assert deferred_keys == pending_ledger_keys, beta_release + assert any(item["key"] == "human-adjudication" for item in beta_release["allowed_deferred_evidence"]), ( + beta_release + ) + assert benchmark_summary["public_claim_ready"] is False, benchmark_summary + assert benchmark_summary["public_claim_blocker_count"] >= 3, benchmark_summary + public_claim = full_payload["data"]["benchmark_reproducibility"]["public_claim"] + assert public_claim["ready"] is False, public_claim + assert not any("provider-backed model holdout evidence is incomplete" in item for item in public_claim["blockers"]), public_claim + assert any("human blind-review adjudication is incomplete" in item for item in public_claim["blockers"]), public_claim + output_review_checklist = full_payload["data"]["output_review_adjudication"]["reviewer_checklist"] + assert len(output_review_checklist) == 5, output_review_checklist + assert all(not item["answer_key_visible"] for item in output_review_checklist), output_review_checklist + assert output_review_checklist[0]["commands"]["adjudicate"] == "python3 scripts/yao.py output-review", output_review_checklist[0] assert full_payload["data"]["review_annotations"]["summary"]["annotation_count"] == 0, full_payload["data"]["review_annotations"] + daily_skillops_summary = full_payload["data"]["daily_skillops"]["summary"] + assert daily_skillops_summary["writes_source_files"] is False, daily_skillops_summary + assert daily_skillops_summary["auto_patch_enabled"] is False, daily_skillops_summary + weekly_curator_summary = full_payload["data"]["weekly_curator"]["summary"] + assert weekly_curator_summary["unique_opportunity_count"] >= 1, weekly_curator_summary + assert weekly_curator_summary["writes_source_files"] is False, weekly_curator_summary + assert weekly_curator_summary["auto_patch_enabled"] is False, weekly_curator_summary + assert full_payload["data"]["adaptation_proposals"]["report_contract"]["contract"] == "adaptation-proposals", full_payload["data"]["adaptation_proposals"] + assert full_payload["data"]["adaptation_proposals"]["proposal_count"] == full_payload["data"]["adaptation_proposals"]["summary"]["proposal_count"], full_payload["data"]["adaptation_proposals"] + assert full_payload["data"]["adaptation_approval_ledger"]["report_contract"]["contract"] == "adaptation-approval-ledger", full_payload["data"]["adaptation_approval_ledger"] + assert full_payload["data"]["adaptation_approval_ledger"]["pending_review_count"] == 0, full_payload["data"]["adaptation_approval_ledger"] + assert full_payload["data"]["adaptation_regression"]["report_contract"]["contract"] == "adaptation-regression-report", full_payload["data"]["adaptation_regression"] + assert full_payload["data"]["adaptation_regression"]["approval_draft_supported"] is True, full_payload["data"]["adaptation_regression"] + waiver_summary = full_payload["data"]["review_waivers"]["summary"] + assert waiver_summary["waiver_candidate_count"] == 2, waiver_summary + assert waiver_summary["waiverable_open_count"] == 1, waiver_summary + assert waiver_summary["non_waivable_count"] == 1, waiver_summary + waiver_candidates = {item["gate_key"]: item for item in full_payload["data"]["review_waivers"]["waiver_candidates"]} + assert waiver_candidates["output-lab"]["waiver_allowed"] is True, waiver_candidates + assert waiver_candidates["output-lab"]["status"] == "needs-reviewer-decision", waiver_candidates + assert waiver_candidates["output-lab"]["risk_summary"] == "review pending 5; model-executed 10; output failures 0", waiver_candidates + assert "review-waivers . --add-waiver" in waiver_candidates["output-lab"]["suggested_command"], waiver_candidates + assert "Does not count as provider, human, or public world-class completion evidence" in waiver_candidates["output-lab"]["world_class_boundary"], waiver_candidates + assert waiver_candidates["world-class-evidence"]["waiver_allowed"] is False, waiver_candidates + assert waiver_candidates["world-class-evidence"]["status"] == "cannot-waive", waiver_candidates + assert waiver_candidates["world-class-evidence"]["risk_summary"] == "4 pending evidence entries; 1 human pending; 3 external pending", waiver_candidates + assert "Non-waivable completion boundary" in waiver_candidates["world-class-evidence"]["world_class_boundary"], waiver_candidates assert full_payload["data"]["compiled_targets"]["summary"]["target_count"] >= 4, full_payload["data"]["compiled_targets"] assert full_payload["data"]["compiled_targets"]["summary"]["block_count"] == 0, full_payload["data"]["compiled_targets"] - assert full_payload["data"]["runtime_permissions"]["summary"]["metadata_fallback_count"] == 3, full_payload["data"]["runtime_permissions"] + assert full_payload["data"]["runtime_permissions"]["summary"]["metadata_fallback_count"] == 4, full_payload["data"]["runtime_permissions"] assert full_payload["evidence_paths"]["runtime_permissions"] == "reports/runtime_permission_probes.md", full_payload["evidence_paths"] + assert full_payload["data"]["python_compatibility"]["summary"]["target_python"] == "3.11", full_payload["data"]["python_compatibility"] + assert full_payload["data"]["python_compatibility"]["summary"]["issue_count"] == 0, full_payload["data"]["python_compatibility"] + assert full_payload["data"]["architecture_maintainability"]["summary"]["hotspot_count"] == 0, full_payload["data"]["architecture_maintainability"] + assert full_payload["data"]["architecture_maintainability"]["summary"]["watchlist_count"] == 0, full_payload["data"]["architecture_maintainability"] + assert full_payload["data"]["architecture_maintainability"]["summary"]["early_watchlist_count"] >= 2, full_payload["data"]["architecture_maintainability"] + assert full_payload["data"]["architecture_maintainability"]["summary"]["blocker_count"] == 0, full_payload["data"]["architecture_maintainability"] + early_watch_paths = { + item["path"] for item in full_payload["data"]["architecture_maintainability"]["early_watchlist"] + } + assert "scripts/cross_packager.py" not in early_watch_paths, full_payload["data"]["architecture_maintainability"] action_keys = {item["gate_key"] for item in full_payload["review_actions"]} - assert action_keys == set(), full_payload["review_actions"] + assert action_keys == {"output-lab", "review-waivers", "world-class-evidence"}, full_payload["review_actions"] + assert_world_class_action(full_payload) + if full_payload["data"]["skill_os2_coverage"]: + assert full_payload["data"]["skill_os2_coverage"]["summary"]["local_blueprint_ready"] is True, full_payload["data"]["skill_os2_coverage"] + assert full_payload["data"]["skill_os2_coverage"]["summary"]["public_world_class_ready"] is False, full_payload["data"]["skill_os2_coverage"] + assert full_payload["data"]["skill_os2_coverage"]["summary"]["extension_track_count"] == 4, full_payload["data"]["skill_os2_coverage"] + assert full_payload["data"]["skill_os2_coverage"]["summary"]["extension_partial_count"] == 0, full_payload["data"]["skill_os2_coverage"] + assert full_payload["data"]["skill_os2_coverage"]["summary"]["extension_planned_count"] == 0, full_payload["data"]["skill_os2_coverage"] + assert full_payload["data"]["skill_os2_coverage"]["summary"]["extension_covered_count"] == 4, full_payload["data"]["skill_os2_coverage"] + assert full_payload["data"]["skill_os2_coverage"]["summary"]["adaptive_extension_ready"] is True, full_payload["data"]["skill_os2_coverage"] + extension_tracks = { + item["key"]: item + for item in full_payload["data"]["skill_os2_coverage"]["reference_extension_tracks"] + } + assert extension_tracks["skill-interpretation-report"]["status"] == "covered", extension_tracks + assert extension_tracks["adaptive-self-iteration"]["status"] == "covered", extension_tracks + assert extension_tracks["daily-skillops-report"]["status"] == "covered", extension_tracks + assert extension_tracks["weekly-curator-report"]["status"] == "covered", extension_tracks + world_class_entries = full_payload["data"]["world_class_evidence_ledger"]["entries"] + assert len(world_class_entries) == 4, world_class_entries + assert {item["key"] for item in world_class_entries} == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, world_class_entries + provider_entry = next(item for item in world_class_entries if item["key"] == "provider-holdout") + assert provider_entry["status"] == "pending", provider_entry + assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider_entry["success_checks"], provider_entry + assert any("output-exec --provider-runner openai" in step for step in provider_entry["runbook"]), provider_entry + assert provider_entry["observed_state"]["model_executed_count"] == 10, provider_entry + assert provider_entry["observed_state"]["token_observed_count"] == 10, provider_entry + provider_submission_status = provider_entry["submission_state"]["status"] + assert provider_submission_status in {"invalid-contract", "missing"}, provider_entry + if provider_submission_status == "invalid-contract": + assert provider_entry["submission_state"]["artifact_sha256_verified_count"] == 1, provider_entry + else: + assert provider_entry["submission_state"].get("artifact_sha256_verified_count", 0) == 0, provider_entry + assert provider_entry["submission_state"]["ledger_counts_as_completion"] is False, provider_entry assert full_payload["data"]["atlas"]["summary"]["actionable_route_collision_count"] == 0, full_payload["data"]["atlas"] + assert full_payload["data"]["atlas"]["summary"]["actionable_drift_signal_count"] == 0, full_payload["data"]["atlas"] assert full_payload["data"]["atlas"]["summary"]["non_actionable_issue_count"] >= 1, full_payload["data"]["atlas"] synthetic_actions = review_studio.build_review_actions( [ @@ -318,32 +342,144 @@ def main() -> None: output_html, ) assert len(synthetic_actions) == 1, synthetic_actions + assert synthetic_actions[0]["title"] == "输出实验", synthetic_actions + assert synthetic_actions[0]["next_step"] == synthetic_actions[0]["summary"], synthetic_actions + assert synthetic_actions[0]["reason"] == synthetic_actions[0]["why"], synthetic_actions assert synthetic_actions[0]["source_refs"], synthetic_actions assert {item["path"] for item in synthetic_actions[0]["source_refs"]} >= { "evals/output/cases.jsonl", "reports/output_quality_scorecard.md", "reports/output_execution_runs.md", "reports/output_blind_review_pack.md", + "reports/output_review_kit.html", "reports/output_review_adjudication.md", }, synthetic_actions assert all(item["exists"] for item in synthetic_actions[0]["source_refs"]), synthetic_actions assert all(isinstance(item["line"], int) and item["line"] >= 1 for item in synthetic_actions[0]["source_refs"]), synthetic_actions + assert all("excerpt" in item for item in synthetic_actions[0]["source_refs"]), synthetic_actions + assert any(item["path"] == "reports/output_review_kit.html" and item["matched_pattern"] == "Output Review Kit" and "Output Review Kit" in item["excerpt"] for item in synthetic_actions[0]["source_refs"]), synthetic_actions + assert any(item["path"] == "evals/output/cases.jsonl" and item["matched_pattern"] == '"id"' and '"id"' in item["excerpt"] for item in synthetic_actions[0]["source_refs"]), synthetic_actions synthetic_json = json.dumps(synthetic_actions, ensure_ascii=False) assert str(ROOT) not in synthetic_json, synthetic_json synthetic_html = review_studio.render_review_actions(synthetic_actions) assert "source-ref-list" in synthetic_html, synthetic_html assert "evals/output/cases.jsonl" in synthetic_html, synthetic_html + assert "
    " in synthetic_html, synthetic_html + assert "pattern: Output Review Kit" in synthetic_html, synthetic_html html = output_html.read_text(encoding="utf-8") assert "Review Studio 2.0" in html, html[:400] + assert "
    " in html, html + assert "pattern: # World-Class Evidence Ledger" in html, html assert "审查闸门" in html, html[:1200] assert "修复动作" in html, html[:3000] - assert "当前没有 blocker 或 warning" in html, html[:9000] + assert "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。" in html, html[:9000] + assert "resource governance governed" in html, html[:9000] + assert "reports/output_review_kit.html" in html, html[:9000] + assert "python3 scripts/adjudicate_output_review.py --write-template" in html, html[:9000] + assert "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。" in html, html[:9000] + assert "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据" in html, html + assert "action-card warn with-evidence" in html, html + assert "action-evidence-panel" in html, html + assert "证据采集" in html, html + assert "action-evidence-grid" in html, html + assert "action-evidence-checks" in html, html + assert "阶段队列" in html, html + assert "action-phase-list" in html, html + assert "action-phase-row blocked" in html, html + assert "#20 · unblock-access" in html, html + assert "#40 · collect-source" in html, html + assert "action-repair-list" in html, html + assert "action-repair-row blocked" in html, html + assert "provider-api-key" in html, html + assert "model_executed_count" in html, html + assert "#20 · unblock-access · precheck" in html, html + assert "#40 · collect-source · source-check" in html, html + assert "operator with provider credentials" in html, html + assert "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" in html, html + assert "action-command-list" in html, html + assert "action-runbook-list" in html, html + assert "采集契约" in html, html + assert "action-collection-grid" in html, html + assert "来源要求" in html, html + assert "通过条件" in html, html + assert "证据资产" in html, html + assert "隐私边界" in html, html + assert "世界证据" in html, html + assert "证据台账" in html, html + assert "证据入口" in html, html + assert "入口边界" in html, html + assert "声明守卫" in html, html + assert "声明边界" in html, html + assert "提交清单" in html, html + assert "world-intake-grid" in html, html + assert "操作命令" in html, html + assert "收集要求" in html, html + assert "通过条件" in html, html + assert "evidence/world_class/submissions/provider-holdout.json" in html, html + assert "evidence/world_class/templates/provider-holdout.intake.json" in html, html + assert "Native Client Telemetry" in html, html + assert "native-client-telemetry.json" in html, html + assert "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" in html, html + assert "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" in html, html + assert "资产角色" in html, html + assert "submission-ref" in html, html + assert "supporting-evidence" in html, html + assert "artifact_refs: true" in html, html + assert "intake 只校验证据包格式、来源、隐私和反过度声明" in html, html + assert "reports/world_class_evidence_intake.md" in html, html + assert "reports/world_class_operator_runbook.html" in html, html + assert "reports/world_class_claim_guard.md" in html, html + assert "日常运维" in html, html + assert "周度队列" in html, html + assert "Weekly Queue" in html, html + assert "英文完成断言、true 状态声明或中文完成态" in html, html + assert "world-evidence-grid" in html, html + assert "Provider Holdout" in html, html + assert "Native Permission Enforcement" in html, html + assert "提交态" in html, html + assert "status: missing" in html, html + assert "完成定义" in html, html + assert "证据来源" in html, html + assert "隐私约束" in html, html + assert "reports/output_execution_runs.json summary.model_executed_count > 0" in html, html + assert "计划、metadata fallback、待评审和本地命令不会被当成完成证据" in html, html + assert "执行步骤" in html, html + assert "output-exec --provider-runner openai" in html, html + assert "<redacted>" not in html and "" not in html, html + assert "world-runbook-panel" in html, html + assert "源证据检查" in html, html + assert "world-source-checks" in html, html + assert "Provider model run" in html, html + assert "model_executed_count: 10 / >0" in html, html + assert "Token usage observed" in html, html + assert "蓝图覆盖" in html, html + assert "Extension Track Count" in html, html + assert "Adaptive Extension Ready" in html, html + assert "
    Extension Partial Count
    0
    " in html, html + assert "
    Extension Covered Count
    4
    " in html, html + assert "
    Extension Planned Count
    0
    " in html, html + assert "Adaptive Extension Ready" in html, html + assert "本地蓝图" in html, html + assert "public world-class 仍以 world-class evidence ledger" in html, html + assert "公开声明" in html, html + assert "声明阻断" in html, html + assert "可公开声明" in html, html + assert "provider-backed model holdout evidence is incomplete" not in html, html + assert "human blind-review adjudication is incomplete" in html, html assert "审查批注" in html, html[:9000] assert "当前没有 reviewer 批注" in html, html[:9000] assert "输出实验" in html, html[:2000] assert "执行证据" in html, html assert "盲评包" in html, html[:5000] assert "审定报告" in html, html + assert "评审清单" in html, html + assert "output-review-grid" in html, html + assert "awaiting-decision" in html, html + assert "答案隐藏" in html, html + assert "winner_variant" in html, html + assert "python3 scripts/yao.py output-review" in html, html + assert "答案揭示" in html, html + assert "答案隐藏" in html, html assert "注册审计" in html, html[:3000] assert "包体验证" in html, html[:5000] assert "Upgrade" in html, html[:6000] @@ -353,8 +489,29 @@ def main() -> None: assert "Install" in html, html[:6500] assert "运营回路" in html, html[:7600] assert "人工批准" in html, html[:8200] + assert "warning 可以被有边界地接受" in html, html + assert "批准概况" in html, html + assert "批准候选" in html, html + assert "waiver-candidate-grid" in html, html + assert "可批准 · needs-reviewer-decision" in html, html + assert "不可批准 · cannot-waive" in html, html + assert "review pending 5; model-executed 10; output failures 0" in html, html + assert "4 pending evidence entries; 1 human pending; 3 external pending" in html, html + assert "Does not count as provider, human, or public world-class completion evidence" in html, html + assert "Non-waivable completion boundary" in html, html + assert "python3 scripts/yao.py review-waivers . --add-waiver" in html, html + assert "Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority" in html, html + assert "Do not use a waiver to claim public world-class readiness" in html, html assert "权限批准" in html, html[:9000] assert "权限探针" in html, html[:9500] + assert "Python 兼容" in html, html[:10000] + assert "解释器边界" in html, html[:10000] + assert "reports/python_compatibility.md" in html, html + assert "架构维护" in html, html + assert "Arch Debt" in html, html + assert "reports/architecture_maintainability.md" in html, html + assert "0 hotspots" in html, html + assert re.search(r"\d+ watchlist files?", html), html assert "kv-grid" in html, html assert "案例数" in html, html assert "命令执行" in html, html @@ -374,10 +531,24 @@ def main() -> None: assert "案例数" in formatted, formatted assert "abc123" in formatted, formatted assert review_formatting.value_text({"case_count": 5}) == "案例数: 5" - assert len(review_layout.REVIEW_STUDIO_NAV) == 15, review_layout.REVIEW_STUDIO_NAV + assert review_gates.min_output_cases("scaffold") == 1 + assert review_gates.min_output_cases("production") == 3 + assert review_gates.min_output_cases("governed") == 5 + assert review_gates.status_label("warn") == "关注" + assert review_gates.weighted_score([{"key": "output-lab", "status": "pass"}]) == 100 + assert review_gates.weighted_score([{"key": "output-lab", "status": "warn"}]) == 60 + assert not review_gates.gate_contract([{"key": "output-lab", "status": "pass"}])["ok"] + assert len(review_layout.REVIEW_STUDIO_NAV) == 16, review_layout.REVIEW_STUDIO_NAV + assert ("#world-class", "世界证据") in review_layout.REVIEW_STUDIO_NAV, review_layout.REVIEW_STUDIO_NAV assert "position: sticky" in review_layout.review_studio_css(), review_layout.review_studio_css()[:400] + assert review_layout.review_studio_css() == (ROOT / "assets" / "review-studio.css").read_text( + encoding="utf-8" + ).strip() assert "#overview" in review_layout.render_review_nav(), review_layout.render_review_nav() assert "审查总览" in review_layout.render_review_nav(), review_layout.render_review_nav() + assert "闸门契约" in html, html[:9000] + assert "期望 Gate" in html, html[:9000] + assert "实际 Gate" in html, html[:9000] assert review_layout.render_review_nav([]) == "" print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_review_viewer.py b/tests/verify_review_viewer.py index e60e441b..3d603f67 100644 --- a/tests/verify_review_viewer.py +++ b/tests/verify_review_viewer.py @@ -8,7 +8,26 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent +def assert_root_world_class_commands_are_current() -> None: + root_json = ROOT / "reports" / "review-viewer.json" + root_html = ROOT / "reports" / "review-viewer.html" + agent_docs = (ROOT / "AGENTS.md").read_text(encoding="utf-8") + assert root_json.exists(), root_json + assert root_html.exists(), root_html + json_text = root_json.read_text(encoding="utf-8") + assert 'python3 scripts/yao.py world-class-intake ."' not in json_text, root_json + assert "world-class-intake . --submissions-dir evidence/world_class/submissions" in json_text, root_json + for fragment in ( + 'python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT"', + 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"', + "python3 scripts/render_review_viewer.py .", + "release_lock_ready: false", + ): + assert fragment in agent_docs, fragment + + def main() -> None: + assert_root_world_class_commands_are_current() tmp_root = ROOT / "tests" / "tmp_review_viewer" if tmp_root.exists(): subprocess.run(["rm", "-rf", str(tmp_root)], check=True) @@ -43,6 +62,9 @@ def main() -> None: assert html_path.exists(), html_path assert json_path.exists(), json_path html_text = html_path.read_text(encoding="utf-8") + css_contract = (ROOT / "assets" / "review-viewer.css").read_text(encoding="utf-8").strip() + assert css_contract in html_text, html_text[:500] + assert 'rel="stylesheet"' not in html_text, html_text[:500] assert "Architecture at a glance" in html_text, html_text[:500] assert "Compare view" in html_text, html_text[:500] assert "Variant diff studio" in html_text, html_text[:900] diff --git a/tests/verify_review_waivers.py b/tests/verify_review_waivers.py index 25e1aeec..b7050a70 100644 --- a/tests/verify_review_waivers.py +++ b/tests/verify_review_waivers.py @@ -8,6 +8,10 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_review_waivers.py" +sys.path.insert(0, str(ROOT / "scripts")) + +import render_review_waivers as review_waivers # noqa: E402 +import review_studio_gates as review_gates # noqa: E402 def run(*args: str, check: bool = True) -> subprocess.CompletedProcess: @@ -21,6 +25,32 @@ def run(*args: str, check: bool = True) -> subprocess.CompletedProcess: def main() -> None: + expected_gate_keys = { + "intent-canvas", + "trigger-lab", + "output-lab", + "context-budget", + "runtime-matrix", + "trust-report", + "python-compat", + "architecture-maintainability", + "permission-gates", + "permission-runtime", + "skill-atlas", + "operations-loop", + "review-waivers", + "world-class-evidence", + "registry-audit", + "release-notes", + } + assert review_gates.REVIEW_STUDIO_GATE_KEYS == expected_gate_keys, review_gates.REVIEW_STUDIO_GATE_KEYS + assert set(review_gates.GATE_WEIGHTS) == expected_gate_keys, review_gates.GATE_WEIGHTS + assert review_waivers.REVIEW_STUDIO_GATE_KEYS == review_gates.REVIEW_STUDIO_GATE_KEYS + assert review_waivers.NON_WAIVABLE_GATE_KEYS == {"review-waivers", "world-class-evidence"} + assert "python-compat" in review_waivers.WAIVERABLE_GATE_KEYS + assert "architecture-maintainability" in review_waivers.WAIVERABLE_GATE_KEYS + assert "world-class-evidence" not in review_waivers.WAIVERABLE_GATE_KEYS + tmp_root = ROOT / "tests" / "tmp_review_waivers" if tmp_root.exists(): shutil.rmtree(tmp_root) @@ -32,6 +62,7 @@ def main() -> None: empty_payload = json.loads(empty.stdout) assert empty_payload["ok"], empty_payload assert empty_payload["summary"]["waiver_count"] == 0, empty_payload + assert empty_payload["summary"]["waiver_candidate_count"] == 0, empty_payload assert (skill / "reports" / "review_waivers.json").exists(), skill assert (skill / "reports" / "review_waivers.md").exists(), skill @@ -55,6 +86,64 @@ def main() -> None: assert added_payload["summary"]["covered_gate_keys"] == ["trust-report"], added_payload assert "trust-report" in (skill / "reports" / "review_waivers.md").read_text(encoding="utf-8") + python_compat = run( + str(skill), + "--generated-at", + "2026-06-13", + "--add-waiver", + "--gate-key", + "python-compat", + "--reviewer", + "Yao Team", + "--reason", + "Python compatibility warning is understood and bounded to this release.", + "--expires-at", + "2026-09-30", + ) + python_compat_payload = json.loads(python_compat.stdout) + assert python_compat_payload["ok"], python_compat_payload + assert python_compat_payload["summary"]["covered_gate_keys"] == ["python-compat", "trust-report"], python_compat_payload + + architecture = run( + str(skill), + "--generated-at", + "2026-06-13", + "--add-waiver", + "--gate-key", + "architecture-maintainability", + "--reviewer", + "Yao Team", + "--reason", + "Architecture warning is accepted only while the planned split remains in scope.", + "--expires-at", + "2026-09-30", + ) + architecture_payload = json.loads(architecture.stdout) + assert architecture_payload["ok"], architecture_payload + assert architecture_payload["summary"]["covered_gate_keys"] == [ + "architecture-maintainability", + "python-compat", + "trust-report", + ], architecture_payload + + non_waivable = run( + str(skill), + "--generated-at", + "2026-06-13", + "--add-waiver", + "--gate-key", + "world-class-evidence", + "--reviewer", + "Yao Team", + "--reason", + "World class evidence cannot be waived even with a reviewer.", + "--expires-at", + "2026-09-30", + check=False, + ) + assert non_waivable.returncode == 2, non_waivable.stdout + assert "invalid choice" in non_waivable.stderr, non_waivable.stderr + expired_source = tmp_root / "expired.json" expired_source.write_text( json.dumps( @@ -94,6 +183,39 @@ def main() -> None: assert invalid_payload["summary"]["invalid_count"] == 1, invalid_payload assert invalid_payload["failures"], invalid_payload + root_report_json = tmp_root / "root_review_waivers.json" + root_report_md = tmp_root / "root_review_waivers.md" + root = run( + str(ROOT), + "--output-json", + str(root_report_json), + "--output-md", + str(root_report_md), + "--generated-at", + "2026-06-14", + ) + root_payload = json.loads(root.stdout) + assert root_payload["ok"], root_payload + assert root_payload["summary"]["waiver_candidate_count"] == 2, root_payload + assert root_payload["summary"]["waiverable_open_count"] == 1, root_payload + assert root_payload["summary"]["non_waivable_count"] == 1, root_payload + candidates = {item["gate_key"]: item for item in root_payload["waiver_candidates"]} + assert candidates["output-lab"]["waiver_allowed"] is True, candidates + assert candidates["output-lab"]["status"] == "needs-reviewer-decision", candidates + assert "review pending" in candidates["output-lab"]["risk_summary"], candidates + assert "review-waivers . --add-waiver" in candidates["output-lab"]["suggested_command"], candidates + assert candidates["world-class-evidence"]["waiver_allowed"] is False, candidates + assert candidates["world-class-evidence"]["status"] == "cannot-waive", candidates + assert "Non-waivable completion boundary" in candidates["world-class-evidence"]["world_class_boundary"], candidates + assert "--submissions-dir evidence/world_class/submissions" in candidates["world-class-evidence"]["suggested_command"], candidates + root_markdown = root_report_md.read_text(encoding="utf-8") + assert "Candidate Actions" in root_markdown, root_markdown + assert "World-class evidence completion cannot be waived" in root_markdown, root_markdown + assert "Review Studio gates" in root_markdown, root_markdown + assert "Waiverable gates" in root_markdown, root_markdown + assert "Non-waivable gates" in root_markdown, root_markdown + assert "`world-class-evidence` | `cannot-waive` | `false`" in root_markdown, root_markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_runtime_permission_probes.py b/tests/verify_runtime_permission_probes.py index f5ef9ab5..da4615fd 100644 --- a/tests/verify_runtime_permission_probes.py +++ b/tests/verify_runtime_permission_probes.py @@ -9,6 +9,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent TMP = ROOT / "tests" / "tmp_runtime_permission" SCRIPT = ROOT / "scripts" / "probe_runtime_permissions.py" +SIMULATOR = ROOT / "scripts" / "simulate_install.py" def run(cmd: list[str]) -> dict: @@ -40,6 +41,8 @@ def main() -> None: "claude", "--platform", "generic", + "--platform", + "vscode", "--expectations", str(ROOT / "evals" / "packaging_expectations.json"), "--output-dir", @@ -48,6 +51,24 @@ def main() -> None: ] ) assert package["ok"], package + install = run( + [ + sys.executable, + str(SIMULATOR), + str(ROOT), + "--package-dir", + str(package_dir), + "--install-root", + str(TMP / "install-root"), + "--output-json", + str(TMP / "install_simulation.json"), + "--output-md", + str(TMP / "install_simulation.md"), + "--generated-at", + "2026-06-13", + ] + ) + assert install["ok"], install probe = run( [ @@ -60,19 +81,28 @@ def main() -> None: str(TMP / "runtime_permission_probes.json"), "--output-md", str(TMP / "runtime_permission_probes.md"), + "--install-simulation-json", + str(TMP / "install_simulation.json"), ] ) assert probe["ok"], probe payload = probe["payload"] summary = payload["summary"] - assert summary["target_count"] == 3, summary - assert summary["pass_count"] == 3, summary + assert summary["target_count"] == 4, summary + assert summary["pass_count"] == 4, summary assert summary["fail_count"] == 0, summary assert summary["native_enforcement_count"] == 0, summary - assert summary["metadata_fallback_count"] == 3, summary - assert summary["residual_risk_count"] == 3, summary + assert summary["metadata_fallback_count"] == 4, summary + assert summary["installer_enforcement_source_status"] == "present", summary + assert summary["installer_enforcement_pass_count"] == 4, summary + assert summary["installer_permission_enforced_count"] == 12, summary + assert summary["installer_permission_failure_count"] == 0, summary + assert summary["world_class_native_evidence_ready"] is False, summary + assert summary["installer_enforcement_ready"] is True, summary + assert summary["residual_risk_count"] == 4, summary assert payload["expected_capabilities"] == ["file_write", "network", "subprocess"], payload assert {item["assurance"] for item in payload["targets"]} == {"metadata-fallback-explicit"}, payload["targets"] + assert all(item["installer_enforcement"]["enforced"] is True for item in payload["targets"]), payload["targets"] assert (TMP / "runtime_permission_probes.md").exists(), TMP bad_dir = TMP / "bad-dist" @@ -92,6 +122,8 @@ def main() -> None: str(TMP / "bad_runtime_permission_probes.json"), "--output-md", str(TMP / "bad_runtime_permission_probes.md"), + "--install-simulation-json", + str(TMP / "install_simulation.json"), ] ) assert bad_probe["returncode"] == 2, bad_probe diff --git a/tests/verify_skill_atlas.py b/tests/verify_skill_atlas.py index e2381df9..92bed689 100644 --- a/tests/verify_skill_atlas.py +++ b/tests/verify_skill_atlas.py @@ -43,13 +43,54 @@ description: {description} return skill_dir +def write_adoption_report(skill_dir: Path, risk_band: str = "medium") -> None: + reports = skill_dir / "reports" + reports.mkdir(parents=True, exist_ok=True) + (reports / "adoption_drift_report.json").write_text( + json.dumps( + { + "ok": True, + "summary": { + "event_count": 4, + "adoption_sample_count": 3, + "activation_count": 2, + "accepted_count": 1, + "edited_count": 0, + "rejected_count": 1, + "missed_count": 1, + "failed_count": 1, + "adoption_rate": 33.3, + "missed_trigger_count": 1, + "wrong_trigger_count": 0, + "bad_output_count": 1, + "script_error_count": 1, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": risk_band, + }, + "next_iteration_candidates": [ + { + "signal": "bad output", + "recommendation": "Convert the failed output shape into an Output Eval assertion.", + } + ], + "failures": [], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + def main() -> None: tmp_root = ROOT / "tests" / "tmp_skill_atlas" if tmp_root.exists(): shutil.rmtree(tmp_root) tmp_root.mkdir(parents=True, exist_ok=True) - write_skill( + alpha = write_skill( tmp_root, "alpha-release-skill", "Use when turning release notes and changelog snippets into a reusable release brief for teams.", @@ -63,7 +104,7 @@ def main() -> None: }, shared_reference=True, ) - write_skill( + beta = write_skill( tmp_root, "beta-release-skill", "Use when turning changelog notes and release snippets into reusable release briefs for team review.", @@ -75,6 +116,7 @@ def main() -> None: }, shared_reference=True, ) + write_adoption_report(beta, risk_band="high") write_skill( tmp_root, "gamma-doc-skill", @@ -148,20 +190,41 @@ def main() -> None: assert summary["owner_gap_count"] >= 1, summary assert summary["stale_count"] >= 1, summary assert summary["shared_resource_count"] >= 1, summary - assert summary["no_route_opportunity_count"] >= 1, summary + assert summary["no_route_opportunity_count"] >= 2, summary + assert summary["drift_signal_count"] == 2, summary + assert summary["actionable_drift_signal_count"] == 1, summary + assert summary["telemetry_report_count"] == 1, summary assert (output_dir / "catalog.json").exists(), output_dir assert (output_dir / "route_overlap_matrix.csv").exists(), output_dir assert (output_dir / "dependency_graph.json").exists(), output_dir assert (output_dir / "stale_skills.json").exists(), output_dir assert (output_dir / "owner_review_gaps.json").exists(), output_dir - assert (output_dir / "no_route_opportunities.json").exists(), output_dir + assert (output_dir / "drift_signals.json").exists(), output_dir + no_route_path = output_dir / "no_route_opportunities.json" + assert no_route_path.exists(), output_dir html = report_html.read_text(encoding="utf-8") assert "Skill Atlas" in html, html[:1000] assert "Route Collisions" in html, html[:3000] assert "Actionable Issues" in html, html[:3000] + assert "Drift Signals" in html, html[:3000] assert "No-Route Opportunities" in html, html[:3000] + assert "Missed-trigger telemetry" in html, html catalog = json.loads((output_dir / "catalog.json").read_text(encoding="utf-8")) assert catalog["summary"]["skill_count"] == 3, catalog + alpha_catalog = next(item for item in catalog["skills"] if item["name"] == alpha.name) + assert alpha_catalog["telemetry"]["risk_band"] == "missing", alpha_catalog + drift = json.loads((output_dir / "drift_signals.json").read_text(encoding="utf-8")) + assert any(item["name"] == alpha.name and item["signal_types"] == ["no telemetry"] for item in drift), drift + assert any(item["name"] == beta.name and "bad output" in item["signal_types"] and not item["actionable"] for item in drift), drift + no_route = json.loads(no_route_path.read_text(encoding="utf-8")) + assert {item["source_type"] for item in no_route} >= {"failure-case", "telemetry"}, no_route + telemetry_opportunity = next(item for item in no_route if item["source_type"] == "telemetry") + assert telemetry_opportunity["skill"] == beta.name, telemetry_opportunity + assert telemetry_opportunity["signal"] == "missed trigger", telemetry_opportunity + assert telemetry_opportunity["missed_trigger_count"] == 1, telemetry_opportunity + assert telemetry_opportunity["actionable"] is False, telemetry_opportunity + assert "metadata-only telemetry" in telemetry_opportunity["privacy_contract"], telemetry_opportunity + assert "raw prompt" in telemetry_opportunity["privacy_contract"], telemetry_opportunity assert payload["scope_policy"]["present"], payload["scope_policy"] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_skill_interpretation.py b/tests/verify_skill_interpretation.py new file mode 100644 index 00000000..95fffa4d --- /dev/null +++ b/tests/verify_skill_interpretation.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_skill_interpretation" + + +def run(*args: str) -> dict: + proc = subprocess.run( + [sys.executable, str(CLI), *args], + cwd=ROOT, + capture_output=True, + text=True, + ) + payload = json.loads(proc.stdout) + return {"ok": proc.returncode == 0, "payload": payload, "stderr": proc.stderr} + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + init_result = run( + "init", + "interpretation-demo-skill", + "--description", + "Turn messy launch notes into a reusable launch checklist skill with review gates.", + "--output-dir", + str(TMP), + "--mode", + "production", + "--archetype", + "production", + ) + assert init_result["ok"], init_result + skill_dir = Path(init_result["payload"]["root"]) + assert (skill_dir / "reports" / "skill-overview.html").exists(), skill_dir + assert (skill_dir / "reports" / "skill-interpretation.html").exists(), skill_dir + assert (skill_dir / "reports" / "skill-interpretation.json").exists(), skill_dir + report_view = init_result["payload"]["report_view"] + assert report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), report_view + assert "reports/skill-interpretation.html" in report_view["message"], report_view + + rerender_result = run("skill-interpretation", str(skill_dir)) + assert rerender_result["ok"], rerender_result + assert rerender_result["payload"]["artifacts"]["html"].endswith("reports/skill-interpretation.html"), rerender_result + assert rerender_result["payload"]["summary"]["report_kind"] == "skill-interpretation", rerender_result + assert rerender_result["payload"]["summary"]["default_language"] == "zh-CN", rerender_result + assert rerender_result["payload"]["summary"]["section_count"] == 9, rerender_result + assert rerender_result["payload"]["summary"]["source_model_reused"] is True, rerender_result + + payload = json.loads((skill_dir / "reports" / "skill-interpretation.json").read_text(encoding="utf-8")) + assert payload["report_contract"]["schema_version"] == "2.0", payload["report_contract"] + assert payload["report_contract"]["report_kind"] == "skill-interpretation", payload["report_contract"] + assert payload["report_contract"]["html_report"] == "reports/skill-interpretation.html", payload["report_contract"] + assert payload["report_contract"]["default_language"] == "zh-CN", payload["report_contract"] + assert payload["report_contract"]["languages"] == ["zh-CN", "en"], payload["report_contract"] + assert payload["report_contract"]["layout"] == "kami-white-audit-v2", payload["report_contract"] + assert payload["interpretation_contract"]["source_model"] == "skill-overview-v2", payload["interpretation_contract"] + assert payload["interpretation_contract"]["source_model_reused"] is True, payload["interpretation_contract"] + assert len(payload["interpretation_contract"]["includes"]) >= 10, payload["interpretation_contract"] + assert all(len(label) == 4 for label in payload["report_contract"]["nav_labels"]), payload["report_contract"] + assert "reports/skill-interpretation.html" in payload["skill_summary"]["deliverables"], payload["skill_summary"] + + html = (skill_dir / "reports" / "skill-interpretation.html").read_text(encoding="utf-8") + assert 'data-report-lang="zh-CN"' in html, html[:500] + assert 'class="language-switch"' in html, html[:2600] + assert "reports/skill-interpretation.html" in html, html[:10000] + assert "After creation, open reports/skill-interpretation.html before expanding the package further." in html, html[:12000] + for label in ("技能概述", "总览指标", "能力画像", "原理结构", "契约边界", "质量评估", "风险治理", "包体资产", "迭代路线"): + assert f">{label}" in html, label + for chart_label in ("评分雷达", "交付流程", "能力矩阵", "风险热力", "资产分布", "迭代时间"): + assert chart_label in html, chart_label + + schema = json.loads((ROOT / "schemas" / "skill-interpretation.schema.json").read_text(encoding="utf-8")) + assert schema["properties"]["report_contract"]["properties"]["report_kind"]["const"] == "skill-interpretation", schema + assert schema["properties"]["interpretation_contract"]["properties"]["source_model_reused"]["const"] is True, schema + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_ir.py b/tests/verify_skill_ir.py index c81fb01d..914222fb 100644 --- a/tests/verify_skill_ir.py +++ b/tests/verify_skill_ir.py @@ -41,8 +41,20 @@ def main() -> None: assert payload["trigger_surface"]["edge_cases"], payload assert payload["workflow"]["steps"], payload assert payload["workflow"]["decision_points"], payload + assert any("`Governed`" in item for item in payload["workflow"]["decision_points"]), payload["workflow"]["decision_points"] assert payload["resources"]["references"], payload assert payload["resources"]["scripts"], payload + expected_scripts = { + str(path.relative_to(ROOT)) + for path in (ROOT / "scripts").rglob("*") + if path.is_file() and path.suffix in {".py", ".sh", ".js", ".ts"} + } + actual_scripts = set(payload["resources"]["scripts"]) + assert expected_scripts <= actual_scripts, sorted(expected_scripts - actual_scripts) + assert "assets/skill-overview.css" in payload["resources"]["assets"], payload["resources"]["assets"] + assert "assets/skill-overview.js" in payload["resources"]["assets"], payload["resources"]["assets"] + assert "assets/review-studio.css" in payload["resources"]["assets"], payload["resources"]["assets"] + assert "assets/review-viewer.css" in payload["resources"]["assets"], payload["resources"]["assets"] assert payload["resources"]["reports"], payload assert "evals/trigger_cases.json" in payload["eval_plan"]["trigger"], payload["eval_plan"] assert payload["risk"]["trust_boundary"] == "external", payload diff --git a/tests/verify_skill_ir_paths.py b/tests/verify_skill_ir_paths.py new file mode 100644 index 00000000..b781b7e4 --- /dev/null +++ b/tests/verify_skill_ir_paths.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import json +import shutil +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from skill_ir_paths import candidate_paths, find_skill_ir, find_skill_ir_path # noqa: E402 + + +TMP = ROOT / "tests" / "tmp_skill_ir_paths" + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + + root_ir, root_path = find_skill_ir(ROOT, "yao-meta-skill", require_schema=True) + assert root_path == "skill-ir/examples/yao-meta-skill.json", root_path + assert root_ir["schema_version"] == "2.0.0", root_ir + assert find_skill_ir_path(ROOT, "yao-meta-skill", require_schema=True) == root_path + + preferred = TMP / "reports" / "skill-ir.json" + example = TMP / "skill-ir" / "examples" / "demo-skill.json" + write_json(preferred, {"schema_version": "2.0.0", "name": "reports-copy"}) + write_json(example, {"schema_version": "2.0.0", "name": "example-copy"}) + payload, source = find_skill_ir(TMP, "demo-skill", require_schema=True) + assert source == "reports/skill-ir.json", source + assert payload["name"] == "reports-copy", payload + + preferred.unlink() + payload, source = find_skill_ir(TMP, "demo-skill", require_schema=True) + assert source == "skill-ir/examples/demo-skill.json", source + assert payload["name"] == "example-copy", payload + + example.unlink() + write_json(TMP / "skill-ir" / "examples" / "zzz-extra.json", {"schema_version": "2.0.0", "name": "wildcard"}) + payload, source = find_skill_ir(TMP, "missing-name", require_schema=True) + assert source == "skill-ir/examples/zzz-extra.json", source + assert payload["name"] == "wildcard", payload + + (TMP / "skill-ir" / "examples" / "zzz-extra.json").unlink() + write_json(TMP / "reports" / "skill-ir.json", {"name": "no-schema"}) + assert find_skill_ir_path(TMP, "demo-skill", require_schema=True, fallback_source="missing") == "missing" + assert find_skill_ir_path(TMP, "demo-skill", require_schema=False) == "reports/skill-ir.json" + + paths = [str(path.relative_to(TMP)) for path in candidate_paths(TMP, "demo-skill")] + assert paths[:3] == [ + "reports/skill-ir.json", + "skill-ir/examples/demo-skill.json", + "skill-ir/examples/tmp_skill_ir_paths.json", + ], paths + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_os2_audit.py b/tests/verify_skill_os2_audit.py new file mode 100644 index 00000000..271b6656 --- /dev/null +++ b/tests/verify_skill_os2_audit.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_skill_os2_audit.py" +TMP = ROOT / "tests" / "tmp_skill_os2_audit" + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "skill_os2_audit.json" + output_md = TMP / "skill_os2_audit.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["decision"] == "continue-iteration", payload + assert payload["summary"]["world_class_ready"] is False, payload + assert payload["summary"]["missing_count"] == 0, payload + assert payload["summary"]["item_count"] >= 15, payload + assert payload["summary"]["pass_count"] >= 12, payload + assert payload["summary"]["human_required_count"] >= 1, payload + assert payload["summary"]["external_required_count"] >= 2, payload + assert payload["artifacts"]["world_class_evidence_plan"] == "reports/world_class_evidence_plan.md", payload + items = {item["key"]: item for item in payload["items"]} + assert items["skill-ir"]["status"] == "pass", items["skill-ir"] + assert items["target-compiler"]["status"] == "pass", items["target-compiler"] + assert items["output-eval-lab"]["status"] == "pass", items["output-eval-lab"] + assert items["benchmark-reproducibility"]["status"] == "pass", items["benchmark-reproducibility"] + assert items["provider-holdout"]["status"] == "pass", items["provider-holdout"] + assert "model-executed 10" in items["provider-holdout"]["current"], items["provider-holdout"] + assert items["human-adjudication"]["status"] == "human_required", items["human-adjudication"] + assert items["native-permission-enforcement"]["status"] == "external_required", items["native-permission-enforcement"] + assert "installer-enforced targets" in items["native-permission-enforcement"]["current"], items["native-permission-enforcement"] + assert items["native-client-telemetry"]["status"] == "external_required", items["native-client-telemetry"] + assert any(entry["path"] == "scripts/provider_output_eval_runner.py" and entry["exists"] for entry in items["provider-holdout"]["evidence"]) + assert any(entry["path"] == "reports/benchmark_reproducibility.json" and entry["exists"] for entry in items["benchmark-reproducibility"]["evidence"]) + assert any(entry["path"] == "scripts/telemetry_native_host.py" and entry["exists"] for entry in items["native-client-telemetry"]["evidence"]) + markdown = output_md.read_text(encoding="utf-8") + assert "Skill OS 2.0 Audit" in markdown, markdown + assert "reports/world_class_evidence_plan.md" in markdown, markdown + assert "Provider Holdout" in markdown, markdown + assert "Benchmark Reproducibility" in markdown, markdown + assert "`human-adjudication`" in markdown, markdown + assert "`native-client-telemetry`" in markdown, markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_os2_coverage.py b/tests/verify_skill_os2_coverage.py new file mode 100644 index 00000000..5c580dc6 --- /dev/null +++ b/tests/verify_skill_os2_coverage.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_skill_os2_coverage.py" +TMP = ROOT / "tests" / "tmp_skill_os2_coverage" + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "skill_os2_coverage.json" + output_md = TMP / "skill_os2_coverage.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-14", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "local-blueprint-covered-evidence-pending", summary + assert summary["module_count"] == 8, summary + assert summary["recommended_pr_count"] == 13, summary + assert summary["item_count"] == 21, summary + assert summary["pass_count"] == 21, summary + assert summary["warn_count"] == 0, summary + assert summary["missing_count"] == 0, summary + assert summary["extension_track_count"] == 4, summary + assert summary["extension_partial_count"] == 0, summary + assert summary["extension_planned_count"] == 0, summary + assert summary["extension_covered_count"] == 4, summary + assert summary["adaptive_extension_ready"] is True, summary + assert summary["local_blueprint_ready"] is True, summary + assert summary["public_world_class_ready"] is False, summary + assert summary["world_class_evidence_pending_count"] == 4, summary + modules = {item["key"]: item for item in payload["modules"]} + assert set(modules) == { + "skill-ir", + "output-eval-lab", + "runtime-conformance", + "trust-security", + "skill-atlas", + "registry-distribution", + "review-studio", + "telemetry-drift", + }, modules + assert modules["skill-ir"]["status"] == "pass", modules["skill-ir"] + assert modules["review-studio"]["status"] == "pass", modules["review-studio"] + assert any(entry["path"] == "reports/review-studio.html" and entry["exists"] for entry in modules["review-studio"]["evidence"]) + prs = {item["key"]: item for item in payload["recommended_prs"]} + assert set(prs) == { + "benchmark-methodology", + "output-eval-schema", + "output-eval-runner", + "output-quality-scorecard", + "skill-ir-v0", + "compiler-refactor", + "agent-skills-conformance", + "trust-check", + "skill-atlas-generator", + "registry-package-format", + "review-studio-2", + "migration-v2-docs", + "evidence-consistency", + }, prs + assert prs["agent-skills-conformance"]["status"] == "pass", prs["agent-skills-conformance"] + assert prs["benchmark-methodology"]["status"] == "pass", prs["benchmark-methodology"] + assert prs["evidence-consistency"]["status"] == "pass", prs["evidence-consistency"] + assert any(entry["path"] == "reports/benchmark_reproducibility.json" and entry["exists"] for entry in prs["benchmark-methodology"]["evidence"]) + assert payload["source_blueprint"]["core_module_count"] == 8, payload + assert payload["source_blueprint"]["recommended_pr_count"] == 13, payload + assert payload["source_blueprint"]["reference_extension_count"] == 4, payload + extension_tracks = {item["key"]: item for item in payload["reference_extension_tracks"]} + assert extension_tracks["skill-interpretation-report"]["status"] == "covered", extension_tracks + assert extension_tracks["adaptive-self-iteration"]["status"] == "covered", extension_tracks + assert extension_tracks["daily-skillops-report"]["status"] == "covered", extension_tracks + assert extension_tracks["weekly-curator-report"]["status"] == "covered", extension_tracks + assert any( + entry["path"] == "reports/skill-overview.html" and entry["exists"] + for entry in extension_tracks["skill-interpretation-report"]["evidence"] + ), extension_tracks["skill-interpretation-report"] + assert any( + entry["path"] == "scripts/render_skill_interpretation.py" and entry["exists"] + for entry in extension_tracks["skill-interpretation-report"]["evidence"] + ), extension_tracks["skill-interpretation-report"] + assert any( + entry["path"] == "schemas/skill-interpretation.schema.json" and entry["exists"] + for entry in extension_tracks["skill-interpretation-report"]["evidence"] + ), extension_tracks["skill-interpretation-report"] + assert any( + entry["path"] == "scripts/summarize_user_signals.py" and entry["exists"] + for entry in extension_tracks["adaptive-self-iteration"]["evidence"] + ), extension_tracks["adaptive-self-iteration"] + assert any( + entry["path"] == "scripts/apply_adaptation.py" and entry["exists"] + for entry in extension_tracks["adaptive-self-iteration"]["evidence"] + ), extension_tracks["adaptive-self-iteration"] + assert any( + entry["path"] == "reports/adaptation_approval_ledger.json" and entry["exists"] + for entry in extension_tracks["adaptive-self-iteration"]["evidence"] + ), extension_tracks["adaptive-self-iteration"] + assert any( + entry["path"] == "reports/adaptation_regression_report.json" and entry["exists"] + for entry in extension_tracks["adaptive-self-iteration"]["evidence"] + ), extension_tracks["adaptive-self-iteration"] + assert any( + entry["path"] == "scripts/render_daily_skillops_report.py" and entry["exists"] + for entry in extension_tracks["daily-skillops-report"]["evidence"] + ), extension_tracks["daily-skillops-report"] + assert any( + entry["path"] == "tests/verify_daily_skillops.py" and entry["exists"] + for entry in extension_tracks["daily-skillops-report"]["evidence"] + ), extension_tracks["daily-skillops-report"] + assert any( + entry["path"].startswith("reports/skillops/daily/") + and entry["path"].endswith(".json") + and entry["exists"] + for entry in extension_tracks["daily-skillops-report"]["evidence"] + ), extension_tracks["daily-skillops-report"] + assert any( + entry["path"].startswith("reports/skillops/daily/") + and entry["path"].endswith(".md") + and entry["exists"] + for entry in extension_tracks["daily-skillops-report"]["evidence"] + ), extension_tracks["daily-skillops-report"] + assert any( + entry["path"] == "scripts/render_weekly_curator_report.py" and entry["exists"] + for entry in extension_tracks["weekly-curator-report"]["evidence"] + ), extension_tracks["weekly-curator-report"] + assert any( + entry["path"] == "tests/verify_weekly_curator.py" and entry["exists"] + for entry in extension_tracks["weekly-curator-report"]["evidence"] + ), extension_tracks["weekly-curator-report"] + assert any( + entry["path"].startswith("reports/skillops/weekly/") + and entry["path"].endswith(".json") + and entry["exists"] + for entry in extension_tracks["weekly-curator-report"]["evidence"] + ), extension_tracks["weekly-curator-report"] + assert any( + entry["path"].startswith("reports/skillops/weekly/") + and entry["path"].endswith(".md") + and entry["exists"] + for entry in extension_tracks["weekly-curator-report"]["evidence"] + ), extension_tracks["weekly-curator-report"] + assert "Close the four world-class evidence ledger entries" in payload["next_highest_leverage"][0], payload + assert "skill interpretation report" in " ".join(payload["next_highest_leverage"]), payload + assert "Daily SkillOps" in " ".join(payload["next_highest_leverage"]), payload + markdown = output_md.read_text(encoding="utf-8") + assert "Skill OS 2.0 Blueprint Coverage" in markdown, markdown + assert "local blueprint ready: `true`" in markdown, markdown + assert "public world-class ready: `false`" in markdown, markdown + assert "extension covered: `4`" in markdown, markdown + assert "extension partial: `0`" in markdown, markdown + assert "## Core Modules" in markdown, markdown + assert "## Recommended PR Coverage" in markdown, markdown + assert "Evidence Consistency" in markdown, markdown + assert "## Reference Extension Tracks" in markdown, markdown + assert "Skill Interpretation Report" in markdown, markdown + assert "Adaptive Self-Iteration" in markdown, markdown + assert "Daily SkillOps Report" in markdown, markdown + assert "Weekly Curator Report" in markdown, markdown + assert "user-supplied 2.0 reference plan" in markdown, markdown + assert "`agent-skills-conformance`" not in markdown, markdown + assert "Agent Skills Conformance" in markdown, markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index 1cb3f7e0..05d17b27 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -10,7 +10,8 @@ CLI = ROOT / "scripts" / "yao.py" sys.path.insert(0, str(ROOT / "scripts")) from skill_report_layout import render_language_switch, render_report_nav, skill_overview_css, skill_overview_script -from skill_report_model import REPORT_NAV_V2 +from skill_report_i18n import en_for +from skill_report_model import REPORT_NAV_V2, build_report_model def run(*args: str) -> dict: @@ -43,17 +44,79 @@ def main() -> None: assert 'aria-pressed="true"' in language_switch_contract, language_switch_contract css_contract = skill_overview_css() + assert css_contract == (ROOT / "assets" / "skill-overview.css").read_text(encoding="utf-8").strip() assert "position: sticky" in css_contract, css_contract[:1200] assert ".report-nav {" in css_contract, css_contract[:3200] assert "background: #ffffff" in css_contract, css_contract[:1600] - assert ".metrics-lead" in css_contract, css_contract[:7000] + assert ".section-body" in css_contract, css_contract[:7000] + assert ".metrics-primary" in css_contract, css_contract[:7000] + assert ".metrics-flow" in css_contract, css_contract[:9000] + assert ".metric-detail-grid" in css_contract, css_contract[:9000] + assert ".metric-detail-section" in css_contract, css_contract[:9000] + assert ".detail-section-kicker" in css_contract, css_contract[:9000] + assert ".section-head {\n display: grid;\n grid-template-columns: minmax(0, 1fr);" in css_contract, css_contract[:7000] + assert "grid-template-columns: minmax(0, 246px) minmax(0, 1fr)" not in css_contract, css_contract[:7000] + assert ".metrics-primary {\n display: grid;\n grid-template-columns: minmax(360px, 0.92fr) minmax(0, 1.08fr);" in css_contract, css_contract[:9000] + assert ".metric-detail-grid {\n grid-template-columns: repeat(2, minmax(min(100%, 420px), 1fr));" in css_contract, css_contract[:9000] + assert ".metric-card {\n display: grid;\n grid-template-columns: minmax(112px, 0.28fr) minmax(0, 1fr);" in css_contract, css_contract[:11000] + assert "repeat(auto-fit, minmax(min(100%, 320px), 1fr))" not in css_contract, css_contract[:9000] + assert "grid-template-columns: minmax(420px, 1fr)" not in css_contract, css_contract[:9000] + assert "overflow-wrap: break-word" in css_contract, css_contract[:9000] + assert ".metrics-primary, .metric-detail-grid, .two-col" in css_contract, css_contract[-2600:] assert "@media (max-width: 980px)" in css_contract, css_contract[-2200:] script_contract = skill_overview_script() + assert script_contract == (ROOT / "assets" / "skill-overview.js").read_text(encoding="utf-8").strip() assert 'setLanguage("zh-CN")' in script_contract, script_contract[-1000:] assert "scaleX(" in script_contract, script_contract assert "aria-current" in script_contract, script_contract + assert en_for("把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。") == ( + "Turn one-off experience into a reusable, evaluable, and portable skill package." + ) + assert en_for("交付结果：SKILL.md, agents/interface.yaml") == "Deliverables: SKILL.md, agents/interface.yaml" + assert en_for("已生成 12 / 20 类报告证据。") == "Generated 12 / 20 evidence report types." + assert en_for("skill-ir.json 已存在。") == "skill-ir.json exists." + assert en_for("compiled_targets.json 已存在。") == "compiled_targets.json exists." + assert en_for("reports 未发现或为空，完整度扣分。") == ( + "reports was not found or is empty, reducing completeness." + ) + assert en_for("evals/ 证据不足。") == "evals/ evidence is insufficient." + assert en_for("用户提供的工作流、提示词、文档、记录或散乱笔记") == ( + "User-provided workflows, prompts, documents, records, or rough notes." + ) + assert en_for("期望沉淀的复用场景、排除项、约束和质量标准") == ( + "The reusable scenario, exclusions, constraints, and quality standards to capture." + ) + assert en_for("可路由的 SKILL.md") == "A routeable SKILL.md." + assert en_for("agents/interface.yaml 元数据") == "agents/interface.yaml metadata." + assert en_for("必要的 references、scripts、evals、reports 证据") == ( + "Necessary references, scripts, evals, and reports evidence." + ) + assert en_for("入口约 864 个词/字，references 约 565 个词/字。") == ( + "Entrypoint is about 864 words/characters; references are about 565." + ) + assert en_for("Use this skill when the request matches: 中文描述") == ( + "Use this skill when the request matches the frontmatter description." + ) + assert en_for("补齐世界证据") == "Close world-class evidence" + assert en_for("世界级证据仍有 2 项待补；公开完成态 claim 必须继续保持阻塞。") == ( + "World-class evidence still has 2 pending item(s); public completion claims must stay blocked." + ) + assert en_for("补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。") == ( + "Close provider holdout evidence: Missing a real provider model run and token metadata." + ) + assert en_for("提交有效 intake packet，并让 ledger 通过 artifact SHA-256 校验。") == ( + "Submit valid intake packets and let the ledger verify artifact SHA-256 digests." + ) + + root_model = build_report_model(ROOT) + assert root_model["skill_ir"]["source_path"] == "skill-ir/examples/yao-meta-skill.json", root_model["skill_ir"] + assert "skill-ir/examples/yao-meta-skill.json" in root_model["skill_summary"]["deliverables"], root_model[ + "skill_summary" + ] + assert "reports/skill-ir.json" not in root_model["skill_summary"]["deliverables"], root_model["skill_summary"] + tmp_root = ROOT / "tests" / "tmp_skill_overview" if tmp_root.exists(): subprocess.run(["rm", "-rf", str(tmp_root)], check=True) @@ -78,6 +141,8 @@ def main() -> None: assert (created / "reports" / "intent-confidence.json").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created assert (created / "reports" / "skill-overview.json").exists(), created + assert (created / "reports" / "skill-interpretation.html").exists(), created + assert (created / "reports" / "skill-interpretation.json").exists(), created assert (created / "reports" / "compiled_targets.md").exists(), created assert (created / "reports" / "compiled_targets.json").exists(), created assert (created / "reports" / "reference-synthesis.md").exists(), created @@ -113,9 +178,12 @@ def main() -> None: } assert expected_v2_keys.issubset(overview_json.keys()), overview_json.keys() assert "reports/skill-ir.json" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert overview_json["skill_ir"]["source_path"] == "reports/skill-ir.json", overview_json["skill_ir"] assert "reports/compiled_targets.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/output_quality_scorecard.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/output_blind_review_pack.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/benchmark_reproducibility.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/world_class_evidence_ledger.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/runtime_permission_probes.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] @@ -127,6 +195,7 @@ def main() -> None: assert "reports/adoption_drift_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/review_waivers.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/review-studio.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/skill-interpretation.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert overview_json["skill_ir"]["schema_version"] in {"", "2.0.0"}, overview_json.get("skill_ir") assert overview_json["compiled_targets"]["summary"]["target_count"] >= 3, overview_json.get("compiled_targets") assert overview_json["compiled_targets"]["summary"]["block_count"] == 0, overview_json.get("compiled_targets") @@ -137,6 +206,13 @@ def main() -> None: assert overview_json["output_blind_review"]["summary"] == {}, overview_json["output_blind_review"] assert "output_review_adjudication" in overview_json, overview_json.keys() assert overview_json["output_review_adjudication"]["summary"] == {}, overview_json["output_review_adjudication"] + assert "benchmark_reproducibility" in overview_json, overview_json.keys() + assert overview_json["benchmark_reproducibility"]["summary"] == {}, overview_json["benchmark_reproducibility"] + assert "world_class_evidence_ledger" in overview_json, overview_json.keys() + assert overview_json["world_class_evidence_ledger"]["summary"] == {}, overview_json["world_class_evidence_ledger"] + assert "world_class_readiness" in overview_json, overview_json.keys() + assert overview_json["world_class_readiness"]["entry_count"] == 0, overview_json["world_class_readiness"] + assert overview_json["world_class_readiness"]["decision"] == "not-generated", overview_json["world_class_readiness"] assert "runtime_conformance" in overview_json, overview_json.keys() assert "runtime_permissions" in overview_json, overview_json.keys() assert overview_json["runtime_permissions"]["summary"] == {}, overview_json["runtime_permissions"] @@ -157,6 +233,54 @@ def main() -> None: initial_report_html = (created / "reports" / "skill-overview.html").read_text(encoding="utf-8") assert directions_json["directions"][0]["title"] in initial_report_html, initial_report_html[:5000] + assert "世界证据" in initial_report_html, initial_report_html + assert "No world-class ledger has been generated" in initial_report_html, initial_report_html + assert "Skill-specific source text is authored in Chinese" not in initial_report_html, initial_report_html + + sample_ledger = { + "schema_version": "1.0", + "ok": True, + "summary": { + "ledger_entry_count": 2, + "accepted_count": 0, + "pending_count": 2, + "external_pending_count": 1, + "human_pending_count": 1, + "source_check_count": 4, + "source_pass_count": 1, + "ready_to_claim_world_class": False, + "decision": "evidence-pending", + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "status": "pending", + "source_checklist": [ + {"label": "Provider model run", "status": "blocked"}, + {"label": "Timing observed", "status": "pass"}, + ], + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "status": "pending", + "source_checklist": [ + {"label": "No pending decisions", "status": "blocked"}, + ], + }, + ], + } + (created / "reports" / "world_class_evidence_ledger.json").write_text( + json.dumps(sample_ledger, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (created / "reports" / "world_class_evidence_ledger.md").write_text( + "# World Class Evidence Ledger\n", + encoding="utf-8", + ) rerender_result = run("skill-report", str(created)) assert rerender_result["ok"], rerender_result @@ -169,8 +293,16 @@ def main() -> None: assert 'class="skip-link"' in report_html, report_html[:2400] assert 'class="progress-bar"' in report_html, report_html[:2400] assert 'aria-current' in report_html, report_html[-3000:] - assert 'class="metrics-stack"' in report_html, report_html[:9000] - assert 'class="metrics-lead"' in report_html, report_html[:9000] + assert 'class="section-body metrics-report"' in report_html, report_html[:9000] + assert 'class="metrics-flow"' in report_html, report_html[:9000] + assert 'class="metrics-primary"' in report_html, report_html[:9000] + assert 'aria-label="指标图表与成熟度摘要"' in report_html, report_html[:9000] + assert 'class="metrics-stack"' not in report_html, report_html[:9000] + assert 'class="metric-detail-section"' in report_html, report_html[:12000] + assert 'class="detail-section-kicker"' in report_html, report_html[:12000] + assert "指标明细" in report_html, report_html[:12000] + assert 'class="metric-grid metric-detail-grid"' in report_html, report_html[:12000] + assert report_html.index('class="metrics-primary"') < report_html.index('class="metric-grid metric-detail-grid"'), report_html[:12000] assert "class='metric-summary-list'" in report_html, report_html[:9000] assert "class='metric-card-head'" in report_html, report_html[:12000] assert 'data-set-lang="zh-CN"' in report_html, report_html[:2600] @@ -191,12 +323,37 @@ def main() -> None: assert "成熟度" in report_html, report_html[:5000] assert "更新时间" in report_html, report_html[:5000] assert "Skill name" in report_html, report_html[:5000] + assert "Turn one-off experience into a reusable, evaluable, and portable skill package." in report_html, report_html[:9000] + assert "After creation, open reports/skill-overview.html before expanding the package further." in report_html, report_html[:9000] + assert "世界证据" in report_html, report_html + assert "证据待补" in report_html, report_html + assert "世界级证据尚未完成：2 项待补，0 项已接受。" in report_html, report_html + assert "World-class evidence is not complete: 2 pending, 0 accepted." in report_html, report_html + assert "提供商留出" in report_html, report_html + assert "人工盲评" in report_html, report_html + assert "Provider Holdout" in report_html, report_html + assert "阻塞检查" in report_html, report_html + assert "Blocked Checks" in report_html, report_html + assert "Provider model run" in report_html, report_html + assert "No pending decisions" in report_html, report_html + assert '把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。' not in report_html, report_html[:9000] + assert '创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。' not in report_html, report_html[:9000] assert "输入材料" in report_html, report_html[:3000] assert "输出结果" in report_html, report_html[:3400] assert "Top 3 Next Moves" not in report_html, report_html[:3800] assert "下一步" in report_html, report_html[:4200] overview_json = json.loads((created / "reports" / "skill-overview.json").read_text(encoding="utf-8")) + assert "reports/world_class_evidence_ledger.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert overview_json["world_class_readiness"]["entry_count"] == 2, overview_json["world_class_readiness"] + assert overview_json["world_class_readiness"]["pending_count"] == 2, overview_json["world_class_readiness"] + assert overview_json["world_class_readiness"]["source_pass_count"] == 1, overview_json["world_class_readiness"] + assert overview_json["world_class_readiness"]["source_check_count"] == 4, overview_json["world_class_readiness"] + assert overview_json["iteration_roadmap"]["items"][0]["title"] == "补齐世界证据", overview_json["iteration_roadmap"] + assert overview_json["iteration_roadmap"]["items"][0]["source"] == "world_class_evidence_ledger", overview_json["iteration_roadmap"] + assert "世界级证据仍有 2 项待补" in overview_json["iteration_roadmap"]["items"][0]["why"], overview_json["iteration_roadmap"] + assert any("补齐提供商留出证据" in item for item in overview_json["iteration_roadmap"]["items"][0]["actions"]), overview_json["iteration_roadmap"] + assert "Close world-class evidence" in report_html, report_html assert "执行流程" in report_html, report_html[:5000] assert "调用方式" in report_html, report_html[:5000] assert "证据不足" in report_html or "证据充分" in report_html, report_html[:8000] diff --git a/tests/verify_skill_report_charts.py b/tests/verify_skill_report_charts.py index 50af48ec..7dd441a4 100644 --- a/tests/verify_skill_report_charts.py +++ b/tests/verify_skill_report_charts.py @@ -53,10 +53,14 @@ def main() -> None: assert svg.startswith("" in svg, (key, svg[:400]) assert "
    " in svg, (key, svg[:400]) + assert 'data-lang="zh-CN"' in svg and 'data-lang="en"' in svg, (key, svg[:600]) assert "http://" not in svg and "https://" not in svg, (key, svg[:400]) assert "data:image" not in svg, (key, svg[:400]) if key in {"flow", "layers"}: assert 'fill="#F6F8FB"' in svg, (key, svg[:500]) + assert "Rating Radar" in charts["radar"], charts["radar"][:900] + assert "Delivery Flow" in charts["flow"], charts["flow"][:900] + assert "The radar chart compares" in charts["radar"], charts["radar"][-500:] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_skillops_opportunity.py b/tests/verify_skillops_opportunity.py new file mode 100644 index 00000000..a1ac9a01 --- /dev/null +++ b/tests/verify_skillops_opportunity.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +from skillops_opportunity import build_opportunities, decision_policy_contract, summarize_opportunities + + +def main() -> None: + patterns = [ + { + "pattern_id": "evidence_testing", + "support_count": 4, + "confidence": 0.95, + "evidence": [{"record_id": "a"}, {"record_id": "b"}, {"record_id": "c"}], + }, + { + "pattern_id": "approval_safety", + "support_count": 2, + "confidence": 0.79, + "evidence": [{"record_id": "d"}, {"record_id": "e"}], + }, + { + "pattern_id": "unknown_manual", + "support_count": 1, + "confidence": 0.55, + "evidence": [{"record_id": "f"}], + }, + ] + proposals = [ + { + "proposal_id": "adapt-high", + "pattern_id": "evidence_testing", + "title": "Attach tests and evidence refresh to each upgrade", + "risk_level": "medium", + "requires_approval": True, + "target_files": ["tests/verify_adaptation_safety.py"], + "verification_commands": ["python3 tests/verify_adaptation_safety.py"], + "evidence_refs": [{"record_id": "a"}, {"record_id": "b"}, {"record_id": "c"}], + }, + { + "proposal_id": "adapt-approval", + "pattern_id": "approval_safety", + "title": "Keep adaptive iteration approval-gated", + "risk_level": "low", + "requires_approval": True, + "target_files": ["AGENTS.md"], + "verification_commands": ["python3 tests/verify_adaptation_safety.py"], + "evidence_refs": [{"record_id": "d"}, {"record_id": "e"}], + }, + { + "proposal_id": "adapt-unknown", + "pattern_id": "unknown_manual", + "title": "Review repeated preference pattern", + "risk_level": "high", + "requires_approval": True, + "target_files": [], + "verification_commands": [], + "evidence_refs": [{"record_id": "f"}], + }, + ] + opportunities = build_opportunities(patterns, proposals) + assert len(opportunities) == 3, opportunities + assert opportunities[0]["score"] >= opportunities[1]["score"] >= opportunities[2]["score"], opportunities + by_pattern = {item["pattern_id"]: item for item in opportunities} + assert by_pattern["evidence_testing"]["action_type"] == "add_eval", by_pattern["evidence_testing"] + assert by_pattern["evidence_testing"]["decision"] in {"ready_for_approval_review", "proposal_review"}, by_pattern[ + "evidence_testing" + ] + assert by_pattern["approval_safety"]["action_type"] == "agents_update", by_pattern["approval_safety"] + assert by_pattern["unknown_manual"]["action_type"] == "report_only", by_pattern["unknown_manual"] + assert by_pattern["unknown_manual"]["score"] < by_pattern["evidence_testing"]["score"], by_pattern + assert all(item["write_allowed_without_approval"] is False for item in opportunities), opportunities + assert all(0 <= item["score"] <= 100 for item in opportunities), opportunities + summary = summarize_opportunities(opportunities) + assert summary["opportunity_count"] == 3, summary + assert summary["action_type_counts"]["add_eval"] == 1, summary + assert summary["action_type_counts"]["agents_update"] == 1, summary + assert summary["action_type_counts"]["report_only"] == 1, summary + assert summary["top_score"] == opportunities[0]["score"], summary + contract = decision_policy_contract() + assert contract["writes_source_files"] is False, contract + assert contract["auto_patch_enabled"] is False, contract + assert contract["approval_required_for_writes"] is True, contract + assert contract["score_range"] == [0, 100], contract + schema = json.loads((ROOT / "schemas" / "skillops-opportunity.schema.json").read_text(encoding="utf-8")) + assert "score" in schema["properties"], schema + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_telemetry_emit.py b/tests/verify_telemetry_emit.py new file mode 100644 index 00000000..bb03789a --- /dev/null +++ b/tests/verify_telemetry_emit.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_telemetry_emit" +EMITTER = ROOT / "scripts" / "emit_telemetry_event.py" +YAO = ROOT / "scripts" / "yao.py" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + + +def read_jsonl(path: Path) -> list[dict]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "telemetry-emit-demo", + "--description", + "Emit metadata-only telemetry from an external client hook for later adoption drift import.", + "--output-dir", + str(TMP), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = TMP / "telemetry-emit-demo" + spool = TMP / "external-spool.jsonl" + + emitted = run( + [ + sys.executable, + str(EMITTER), + str(skill_dir), + "--output-jsonl", + str(spool), + "--event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "browser-extension", + "--timestamp", + "2026-06-13T12:00:00Z", + ] + ) + assert emitted["ok"], emitted + assert emitted["payload"]["emitted"] is True, emitted + assert emitted["payload"]["artifacts"]["spool_jsonl"].endswith("external-spool.jsonl"), emitted + assert "telemetry-import" in emitted["payload"]["artifacts"]["import_command"], emitted + events = read_jsonl(spool) + assert len(events) == 1, events + assert events[0]["event"] == "skill_activation", events + assert events[0]["source"] == "external", events + assert events[0]["command"] == "browser-extension", events + + dry_run = run( + [ + sys.executable, + str(EMITTER), + str(skill_dir), + "--output-jsonl", + str(spool), + "--event", + "script_run", + "--outcome", + "failed", + "--failure-type", + "script_error", + "--command", + "browser-extension", + "--dry-run", + ] + ) + assert dry_run["ok"], dry_run + assert dry_run["payload"]["dry_run"] is True, dry_run + assert dry_run["payload"]["emitted"] is False, dry_run + assert len(read_jsonl(spool)) == 1, read_jsonl(spool) + + invalid = run( + [ + sys.executable, + str(EMITTER), + str(skill_dir), + "--output-jsonl", + str(spool), + "--command", + "bad/path", + ] + ) + assert invalid["returncode"] == 2, invalid + assert "command must use only" in "\n".join(invalid["payload"]["failures"]), invalid + assert len(read_jsonl(spool)) == 1, read_jsonl(spool) + + cli_emit = run( + [ + sys.executable, + str(YAO), + "telemetry-emit", + str(skill_dir), + "--output-jsonl", + str(spool), + "--event", + "skill_output", + "--activation-type", + "manual", + "--outcome", + "edited", + "--command", + "browser-plugin", + "--timestamp", + "2026-06-13T12:01:00Z", + ] + ) + assert cli_emit["ok"], cli_emit + assert cli_emit["payload"]["emitted"] is True, cli_emit + assert len(read_jsonl(spool)) == 2, read_jsonl(spool) + + imported = run( + [ + sys.executable, + str(YAO), + "telemetry-import", + str(skill_dir), + "--input-jsonl", + str(spool), + "--generated-at", + "2026-06-13T12:02:00Z", + ] + ) + assert imported["ok"], imported + assert imported["payload"]["imported_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["event_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-extension"] == 1, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-plugin"] == 1, imported + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_telemetry_hooks.py b/tests/verify_telemetry_hooks.py new file mode 100644 index 00000000..9ad98f71 --- /dev/null +++ b/tests/verify_telemetry_hooks.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_telemetry_hooks" +HOOKS = ROOT / "scripts" / "render_telemetry_hook_recipes.py" +YAO = ROOT / "scripts" / "yao.py" +SENSITIVE_FLAGS = { + "--prompt", + "--content", + "--input", + "--inputs", + "--output", + "--outputs", + "--transcript", + "--message", + "--messages", + "--note", + "--raw", + "--text", +} + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + + +def read_jsonl(path: Path) -> list[dict]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def assert_metadata_only(argv: list[str]) -> None: + lowered = {item.lower() for item in argv} + assert not (lowered & SENSITIVE_FLAGS), argv + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "telemetry-hooks-demo", + "--description", + "Render local telemetry client hook recipes without collecting raw content.", + "--output-dir", + str(TMP), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = TMP / "telemetry-hooks-demo" + output_json = TMP / "telemetry_hook_recipes.json" + output_md = TMP / "telemetry_hook_recipes.md" + spool = TMP / "client-spool.jsonl" + rendered = run( + [ + sys.executable, + str(HOOKS), + str(skill_dir), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--output-jsonl", + str(spool), + ] + ) + assert rendered["ok"], rendered + payload = rendered["payload"] + assert payload["summary"]["recipe_count"] == 5, payload + assert payload["summary"]["native_auto_capture_count"] == 0, payload + assert payload["summary"]["metadata_only_recipe_count"] == 5, payload + assert payload["privacy_contract"]["raw_content_allowed"] is False, payload + assert {"browser-extension", "chrome-extension", "vscode-extension", "cli-wrapper", "provider-adapter"} == { + item["id"] for item in payload["recipes"] + }, payload + assert output_json.exists(), output_json + assert output_md.exists(), output_md + markdown = output_md.read_text(encoding="utf-8") + assert "Telemetry Hook Recipes" in markdown, markdown + assert "not proof that a host client is already natively integrated" in markdown, markdown + + for recipe in payload["recipes"]: + assert recipe["metadata_only"] is True, recipe + assert recipe["source"] == "external", recipe + assert recipe["integration_status"] == "client-hook-recipe", recipe + assert recipe["emit_argv"][:3] == ["python3", "scripts/yao.py", "telemetry-emit"], recipe + assert recipe["dry_run_argv"][-1] == "--dry-run", recipe + assert "--command" in recipe["emit_argv"], recipe + assert_metadata_only(recipe["emit_argv"]) + assert_metadata_only(recipe["dry_run_argv"]) + dry_run = run(recipe["dry_run_argv"]) + assert dry_run["ok"], dry_run + assert dry_run["payload"]["dry_run"] is True, dry_run + assert dry_run["payload"]["emitted"] is False, dry_run + assert read_jsonl(spool) == [], spool + + browser = next(item for item in payload["recipes"] if item["id"] == "browser-extension") + emitted = run(browser["emit_argv"]) + assert emitted["ok"], emitted + assert emitted["payload"]["emitted"] is True, emitted + events = read_jsonl(spool) + assert len(events) == 1, events + assert events[0]["command"] == "browser-extension", events + assert events[0]["event"] == "skill_activation", events + + imported = run(payload["artifacts"]["import_argv"]) + assert imported["ok"], imported + assert imported["payload"]["imported_count"] == 1, imported + assert imported["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 1, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-extension"] == 1, imported + + cli_output_json = TMP / "cli_telemetry_hook_recipes.json" + cli = run( + [ + sys.executable, + str(YAO), + "telemetry-hooks", + str(skill_dir), + "--output-json", + str(cli_output_json), + "--output-jsonl", + str(spool), + ] + ) + assert cli["ok"], cli + assert cli["payload"]["summary"]["recipe_count"] == 5, cli + assert cli_output_json.exists(), cli_output_json + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_telemetry_import.py b/tests/verify_telemetry_import.py new file mode 100644 index 00000000..9c9dfeb2 --- /dev/null +++ b/tests/verify_telemetry_import.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_telemetry_import" +IMPORTER = ROOT / "scripts" / "import_telemetry_events.py" +YAO = ROOT / "scripts" / "yao.py" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + + +def write_jsonl(path: Path, rows: list[dict]) -> None: + path.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8") + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "external-telemetry-demo", + "--description", + "Import external client telemetry into a reusable metadata-only drift loop.", + "--output-dir", + str(TMP), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = TMP / "external-telemetry-demo" + input_jsonl = TMP / "external-events.jsonl" + write_jsonl( + input_jsonl, + [ + { + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-13T11:00:00Z", + }, + { + "event": "script_run", + "activation_type": "manual", + "outcome": "failed", + "failure_type": "script_error", + "timestamp": "2026-06-13T11:01:00Z", + }, + ], + ) + + dry_run = run( + [ + sys.executable, + str(IMPORTER), + str(skill_dir), + "--input-jsonl", + str(input_jsonl), + "--command", + "chrome-extension", + "--dry-run", + ] + ) + assert dry_run["ok"], dry_run + assert dry_run["payload"]["dry_run"] is True, dry_run + assert dry_run["payload"]["candidate_count"] == 2, dry_run + assert dry_run["payload"]["imported_count"] == 0, dry_run + assert not (skill_dir / "reports" / "telemetry_events.jsonl").exists(), dry_run + + imported = run( + [ + sys.executable, + str(IMPORTER), + str(skill_dir), + "--input-jsonl", + str(input_jsonl), + "--command", + "chrome-extension", + "--generated-at", + "2026-06-13T11:02:00Z", + ] + ) + assert imported["ok"], imported + assert imported["payload"]["imported_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["event_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["chrome-extension"] == 2, imported + events = [ + json.loads(line) + for line in (skill_dir / "reports" / "telemetry_events.jsonl").read_text(encoding="utf-8").splitlines() + ] + assert len(events) == 2, events + assert {event["source"] for event in events} == {"external"}, events + assert {event["command"] for event in events} == {"chrome-extension"}, events + markdown = (skill_dir / "reports" / "adoption_drift_report.md").read_text(encoding="utf-8") + assert "source=`external` command=`chrome-extension`" in markdown, markdown + + unsafe_jsonl = TMP / "unsafe-events.jsonl" + write_jsonl( + unsafe_jsonl, + [ + { + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-13T11:03:00Z", + }, + { + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "prompt": "raw user prompt must never be imported", + "timestamp": "2026-06-13T11:04:00Z", + }, + ], + ) + unsafe = run([sys.executable, str(IMPORTER), str(skill_dir), "--input-jsonl", str(unsafe_jsonl)]) + assert unsafe["returncode"] == 2, unsafe + assert not unsafe["payload"]["ok"], unsafe + assert "raw content fields" in "\n".join(unsafe["payload"]["failures"]), unsafe + events_after_failure = (skill_dir / "reports" / "telemetry_events.jsonl").read_text(encoding="utf-8").splitlines() + assert len(events_after_failure) == 2, events_after_failure + + cli_input_jsonl = TMP / "cli-external-events.jsonl" + write_jsonl( + cli_input_jsonl, + [ + { + "event": "skill_output", + "activation_type": "manual", + "outcome": "edited", + "failure_type": "none", + "timestamp": "2026-06-13T11:05:00Z", + } + ], + ) + cli = run( + [ + sys.executable, + str(YAO), + "telemetry-import", + str(skill_dir), + "--input-jsonl", + str(cli_input_jsonl), + "--command", + "browser-plugin", + "--generated-at", + "2026-06-13T11:06:00Z", + ] + ) + assert cli["ok"], cli + assert cli["payload"]["imported_count"] == 1, cli + assert cli["payload"]["adoption_drift"]["summary"]["event_count"] == 3, cli + assert cli["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 3, cli + assert cli["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-plugin"] == 1, cli + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_telemetry_native_host.py b/tests/verify_telemetry_native_host.py new file mode 100644 index 00000000..b1443e1a --- /dev/null +++ b/tests/verify_telemetry_native_host.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +import json +import shutil +import struct +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_telemetry_native_host" +HOST = ROOT / "scripts" / "telemetry_native_host.py" +YAO = ROOT / "scripts" / "yao.py" +ORIGIN = "chrome-extension://aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + + +def pack_message(payload: dict) -> bytes: + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + return struct.pack(" list[dict]: + messages = [] + offset = 0 + while offset < len(data): + size = struct.unpack(" list[dict]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "telemetry-native-host-demo", + "--description", + "Receive metadata-only telemetry events from a Browser native messaging host.", + "--output-dir", + str(TMP), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = TMP / "telemetry-native-host-demo" + spool = TMP / "native-spool.jsonl" + + valid = run( + [ + sys.executable, + str(HOST), + str(skill_dir), + "--output-jsonl", + str(spool), + "--message-json", + json.dumps( + { + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "command": "chrome-native-host", + } + ), + ] + ) + assert valid["ok"], valid + assert valid["payload"]["emitted"] is True, valid + events = read_jsonl(spool) + assert len(events) == 1, events + assert events[0]["command"] == "chrome-native-host", events + assert events[0]["source"] == "external", events + + invalid = run( + [ + sys.executable, + str(HOST), + str(skill_dir), + "--output-jsonl", + str(spool), + "--message-json", + json.dumps( + { + "event": "skill_activation", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "command": "chrome-native-host", + "prompt": "raw prompt must not pass through native host", + } + ), + ] + ) + assert invalid["returncode"] == 2, invalid + assert "raw content fields" in "\n".join(invalid["payload"]["failures"]), invalid + assert len(read_jsonl(spool)) == 1, read_jsonl(spool) + + stdio_payload = pack_message( + { + "event": "skill_output", + "activation_type": "manual", + "outcome": "edited", + "failure_type": "none", + "command": "browser-native-host", + } + ) + stdio = subprocess.run( + [ + sys.executable, + str(HOST), + str(skill_dir), + "--output-jsonl", + str(spool), + "--stdio", + ], + cwd=ROOT, + input=stdio_payload, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + assert stdio.returncode == 0, stdio.stderr + responses = unpack_messages(stdio.stdout) + assert len(responses) == 1, responses + assert responses[0]["ok"], responses + assert responses[0]["emitted"] is True, responses + assert len(read_jsonl(spool)) == 2, read_jsonl(spool) + + manifest = TMP / "native-host-manifest.json" + launcher = TMP / "yao-telemetry-native-host.sh" + manifest_result = run( + [ + sys.executable, + str(HOST), + str(skill_dir), + "--output-jsonl", + str(spool), + "--write-launcher", + str(launcher), + "--write-manifest", + str(manifest), + "--allowed-origin", + ORIGIN, + ] + ) + assert manifest_result["ok"], manifest_result + assert manifest.exists(), manifest + assert launcher.exists(), launcher + assert launcher.stat().st_mode & 0o111, launcher.stat().st_mode + manifest_payload = json.loads(manifest.read_text(encoding="utf-8")) + assert manifest_payload["name"] == "com.yao.meta_skill.telemetry", manifest_payload + assert manifest_payload["type"] == "stdio", manifest_payload + assert manifest_payload["path"] == str(launcher.resolve()), manifest_payload + assert manifest_payload["allowed_origins"] == [ORIGIN], manifest_payload + + manifest_without_launcher = TMP / "native-host-manifest-without-launcher.json" + manifest_without_launcher_result = run( + [ + sys.executable, + str(HOST), + str(skill_dir), + "--write-manifest", + str(manifest_without_launcher), + "--allowed-origin", + ORIGIN, + ] + ) + assert manifest_without_launcher_result["returncode"] == 2, manifest_without_launcher_result + assert not manifest_without_launcher.exists(), manifest_without_launcher + assert "must be executable" in "\n".join(manifest_without_launcher_result["payload"]["failures"]) + + imported = run( + [ + sys.executable, + str(YAO), + "telemetry-import", + str(skill_dir), + "--input-jsonl", + str(spool), + "--generated-at", + "2026-06-13T12:30:00Z", + ] + ) + assert imported["ok"], imported + assert imported["payload"]["imported_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["chrome-native-host"] == 1, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-native-host"] == 1, imported + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index 49ace96e..42152e09 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -8,6 +8,9 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "trust_check.py" +sys.path.insert(0, str(ROOT / "scripts")) + +from trust_check import iter_scan_files, relpath INTERFACE = """interface: @@ -60,8 +63,13 @@ def main() -> None: assert payload["summary"]["package_hash_scope"] == "source-contract-without-generated-reports", payload assert payload["summary"]["package_hash_file_count"] == payload["summary"]["scanned_files"], payload assert payload["summary"]["package_sha256"], payload + scanned_paths = {relpath(ROOT, path) for path in iter_scan_files(ROOT)} + assert "assets/skill-overview.css" in scanned_paths, scanned_paths + assert "assets/skill-overview.js" in scanned_paths, scanned_paths + assert "assets/review-studio.css" in scanned_paths, scanned_paths + assert "assets/review-viewer.css" in scanned_paths, scanned_paths assert payload["summary"]["internal_module_count"] >= 3, payload - assert payload["summary"]["network_script_count"] == 2, payload + assert payload["summary"]["network_script_count"] == 3, payload assert payload["summary"]["network_policy_covered_count"] == payload["summary"]["network_script_count"], payload assert payload["summary"]["network_policy_missing_count"] == 0, payload assert payload["summary"]["permission_required_count"] >= 3, payload @@ -85,26 +93,91 @@ def main() -> None: assert payload["permission_governance"]["expired_capabilities"] == [], payload["permission_governance"] assert "scripts/check_update.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] assert "scripts/github_benchmark_scan.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] + assert "scripts/provider_output_eval_runner.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] script_map = {item["path"]: item for item in payload["scripts"]} for internal_module in [ + "scripts/build_skill_atlas_layout.py", + "scripts/build_skill_atlas_opportunities.py", + "scripts/compile_skill_targets.py", + "scripts/cross_packager_contracts.py", + "scripts/description_optimizer_reporting.py", + "scripts/reference_synthesis_markdown.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", + "scripts/review_studio_gate_helpers.py", + "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/review_studio_panels.py", + "scripts/trust_check_scripts.py", + "scripts/evidence_consistency_core.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_skill_os2_review.py", + "scripts/evidence_consistency_world_class.py", + "scripts/skill_ir_paths.py", "scripts/skill_report_charts.py", + "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/skill_report_sections.py", + "scripts/skill_report_sources.py", + "scripts/skill_report_world_class.py", + "scripts/skill_os2_coverage_markdown.py", + "scripts/world_class_preflight_layout.py", + "scripts/world_class_evidence_contract.py", + "scripts/world_class_source_checks.py", + "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", + "scripts/yao_cli_parser.py", + "scripts/yao_cli_parser_evidence.py", + "scripts/yao_cli_parser_operations.py", + "scripts/yao_cli_report_commands.py", + "scripts/yao_cli_telemetry.py", ]: assert script_map[internal_module]["interface"] == "internal-module", script_map[internal_module] assert script_map[internal_module]["interface_declared"], script_map[internal_module] warning_text = "\n".join(payload["warnings"]) + assert "build_skill_atlas_layout.py" not in warning_text, payload["warnings"] + assert "build_skill_atlas_opportunities.py" not in warning_text, payload["warnings"] + assert "compile_skill_targets.py" not in warning_text, payload["warnings"] + assert "cross_packager_contracts.py" not in warning_text, payload["warnings"] + assert "description_optimizer_reporting.py" not in warning_text, payload["warnings"] + assert "reference_synthesis_markdown.py" not in warning_text, payload["warnings"] assert "review_studio_formatting.py" not in warning_text, payload["warnings"] + assert "review_studio_gate_contract.py" not in warning_text, payload["warnings"] + assert "review_studio_gate_helpers.py" not in warning_text, payload["warnings"] + assert "review_studio_gates.py" not in warning_text, payload["warnings"] assert "review_studio_layout.py" not in warning_text, payload["warnings"] + assert "review_studio_panels.py" not in warning_text, payload["warnings"] + assert "trust_check_scripts.py" not in warning_text, payload["warnings"] + assert "evidence_consistency_core.py" not in warning_text, payload["warnings"] + assert "evidence_consistency_release.py" not in warning_text, payload["warnings"] + assert "evidence_consistency_skill_os2_review.py" not in warning_text, payload["warnings"] + assert "evidence_consistency_world_class.py" not in warning_text, payload["warnings"] + assert "skill_ir_paths.py" not in warning_text, payload["warnings"] assert "skill_report_charts.py" not in warning_text, payload["warnings"] + assert "skill_report_i18n.py" not in warning_text, payload["warnings"] assert "skill_report_layout.py" not in warning_text, payload["warnings"] assert "skill_report_metrics.py" not in warning_text, payload["warnings"] assert "skill_report_model.py" not in warning_text, payload["warnings"] + assert "skill_report_sections.py" not in warning_text, payload["warnings"] + assert "skill_report_sources.py" not in warning_text, payload["warnings"] + assert "skill_report_world_class.py" not in warning_text, payload["warnings"] + assert "skill_os2_coverage_markdown.py" not in warning_text, payload["warnings"] + assert "world_class_preflight_layout.py" not in warning_text, payload["warnings"] + assert "world_class_evidence_contract.py" not in warning_text, payload["warnings"] + assert "world_class_source_checks.py" not in warning_text, payload["warnings"] + assert "yao_cli_adaptation_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_config.py" not in warning_text, payload["warnings"] + assert "yao_cli_distribution_commands.py" not in warning_text, payload["warnings"] + assert "yao_cli_output_commands.py" not in warning_text, payload["warnings"] + assert "yao_cli_parser.py" not in warning_text, payload["warnings"] + assert "yao_cli_parser_evidence.py" not in warning_text, payload["warnings"] + assert "yao_cli_parser_operations.py" not in warning_text, payload["warnings"] + assert "yao_cli_report_commands.py" not in warning_text, payload["warnings"] + assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"] assert "render_context_reports.py" not in warning_text, payload["warnings"] assert "render_social_preview.py" not in warning_text, payload["warnings"] assert "Network-capable scripts require bounded host policy" not in warning_text, payload["warnings"] @@ -214,6 +287,7 @@ def main() -> None: "openai": "metadata-only", "claude": "adapter metadata", "generic": "adapter metadata", + "vscode": "workspace trust note", }, } }, diff --git a/tests/verify_upgrade_check.py b/tests/verify_upgrade_check.py index 945f3813..cbe21380 100644 --- a/tests/verify_upgrade_check.py +++ b/tests/verify_upgrade_check.py @@ -52,6 +52,7 @@ def main() -> None: assert payload["summary"]["declared_bump"] == "minor", payload assert payload["summary"]["recommended_bump"] == "minor", payload assert "agent-skills-compatible" in payload["upgrade_diff"]["added_targets"], payload + assert "vscode" in payload["upgrade_diff"]["added_targets"], payload assert output_json.exists(), output_json assert "Upgrade Check" in output_md.read_text(encoding="utf-8"), output_md diff --git a/tests/verify_weekly_curator.py b/tests/verify_weekly_curator.py new file mode 100644 index 00000000..a745326d --- /dev/null +++ b/tests/verify_weekly_curator.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_weekly_curator_report.py" +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_weekly_curator" + + +def run_command(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *args], + cwd=ROOT, + capture_output=True, + text=True, + ) + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def seed_reports(skill_dir: Path) -> None: + reports = skill_dir / "reports" + write_json( + reports / "skillops" / "daily" / "2026-06-16.json", + { + "ok": True, + "generated_at": "2026-06-16T08:00:00Z", + "opportunity_summary": {"opportunity_count": 3}, + "opportunities": [ + { + "opportunity_id": "skillops-aaaaaaaaaa", + "proposal_id": "adapt-a", + "pattern_id": "report_ui", + "title": "Improve report layout", + "opportunity_type": "artifact-quality", + "action_type": "patch_existing_skill", + "decision": "ready_for_approval_review", + "priority": "high", + "score": 92, + "risk_level": "medium", + "requires_approval": True, + "write_allowed_without_approval": False, + "evidence_count": 5, + "target_files": ["scripts/render_skill_overview.py"], + "verification_commands": ["python3 tests/verify_skill_overview.py"], + "policy_reason": "Report layout feedback maps to renderer changes.", + }, + { + "opportunity_id": "skillops-bbbbbbbbbb", + "proposal_id": "adapt-b", + "pattern_id": "approval_safety", + "title": "Keep adaptive changes approval-gated", + "opportunity_type": "governance", + "action_type": "agents_update", + "decision": "proposal_review", + "priority": "medium", + "score": 78, + "risk_level": "low", + "requires_approval": True, + "write_allowed_without_approval": False, + "evidence_count": 2, + "target_files": ["AGENTS.md"], + "verification_commands": ["python3 tests/verify_adaptation_safety.py"], + "policy_reason": "Governance feedback maps to durable guidance.", + }, + { + "opportunity_id": "skillops-cccccccccc", + "proposal_id": "adapt-c", + "pattern_id": "evidence_testing", + "title": "Add evidence regression", + "opportunity_type": "quality-gate", + "action_type": "add_eval", + "decision": "proposal_review", + "priority": "medium", + "score": 72, + "risk_level": "medium", + "requires_approval": True, + "write_allowed_without_approval": False, + "evidence_count": 2, + "target_files": ["tests/verify_daily_skillops.py"], + "verification_commands": ["python3 tests/verify_daily_skillops.py"], + "policy_reason": "Verification feedback maps to focused checks.", + }, + ], + }, + ) + write_json( + reports / "skill_atlas.json", + { + "ok": True, + "summary": { + "skill_count": 4, + "route_collision_count": 2, + "actionable_route_collision_count": 1, + "owner_gap_count": 1, + "actionable_owner_gap_count": 0, + "stale_count": 1, + "actionable_stale_count": 0, + "drift_signal_count": 1, + "actionable_drift_signal_count": 0, + "no_route_opportunity_count": 1, + }, + }, + ) + write_json(reports / "benchmark_reproducibility.json", {"ok": True, "summary": {"release_lock_ready": True}}) + write_json(reports / "evidence_consistency.json", {"ok": True, "summary": {"decision": "consistent"}}) + write_json( + reports / "world_class_evidence_ledger.json", + {"ok": True, "summary": {"pending_count": 4, "ready_to_claim_world_class": False}}, + ) + + +def assert_contract(payload: dict) -> None: + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["report_contract"]["contract"] == "weekly-skillops-curator-report", payload + assert payload["curator_contract"]["writes_source_files"] is False, payload + assert payload["curator_contract"]["auto_patch_enabled"] is False, payload + assert payload["curator_contract"]["approval_required_for_writes"] is True, payload + assert payload["curator_contract"]["counts_as_world_class_evidence"] is False, payload + for field in payload["report_contract"]["summary_fields"]: + assert payload[field] == payload["summary"][field], field + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + skill_dir = TMP / "weekly-curator-demo" + skill_dir.mkdir(parents=True) + seed_reports(skill_dir) + output_json = skill_dir / "reports" / "skillops" / "weekly" / "2026-W25.json" + output_md = skill_dir / "reports" / "skillops" / "weekly" / "2026-W25.md" + + proc = run_command( + str(SCRIPT), + str(skill_dir), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-16T08:00:00Z", + ) + assert proc.returncode == 0, proc.stderr + payload = json.loads(proc.stdout) + assert_contract(payload) + assert payload["summary"]["week_id"] == "2026-W25", payload + assert payload["summary"]["daily_report_count"] == 1, payload + assert payload["summary"]["unique_opportunity_count"] == 3, payload + assert payload["summary"]["ready_for_approval_review_count"] == 1, payload + assert payload["summary"]["proposal_review_count"] == 2, payload + assert payload["summary"]["top_score"] == 92, payload + assert payload["summary"]["skill_count"] == 4, payload + assert payload["summary"]["actionable_portfolio_issue_count"] == 2, payload + assert payload["summary"]["release_lock_ready"] is True, payload + assert payload["summary"]["evidence_consistency_ok"] is True, payload + assert payload["summary"]["public_world_class_ready"] is False, payload + assert payload["summary"]["world_class_pending_count"] == 4, payload + assert payload["opportunity_summary"]["action_type_counts"]["patch_existing_skill"] == 1, payload + assert payload["opportunity_summary"]["action_type_counts"]["agents_update"] == 1, payload + assert payload["opportunity_summary"]["action_type_counts"]["add_eval"] == 1, payload + assert all(item["write_allowed_without_approval"] is False for item in payload["curator_queue"]), payload + action_keys = {item["key"] for item in payload["actions"]} + assert {"review-ready-opportunities", "triage-skill-library", "close-world-class-evidence"} <= action_keys, payload + markdown = output_md.read_text(encoding="utf-8") + assert "Weekly SkillOps Curator Report" in markdown, markdown + assert "## Curator Queue" in markdown, markdown + assert "counts_as_world_class_evidence: `false`" in markdown, markdown + + cli_output_json = skill_dir / "reports" / "skillops" / "weekly" / "cli.json" + cli_output_md = skill_dir / "reports" / "skillops" / "weekly" / "cli.md" + cli_proc = run_command( + str(CLI), + "weekly-curator", + str(skill_dir), + "--output-json", + str(cli_output_json), + "--output-md", + str(cli_output_md), + "--generated-at", + "2026-06-16T08:00:00Z", + ) + assert cli_proc.returncode == 0, cli_proc.stderr + cli_payload = json.loads(cli_proc.stdout) + assert_contract(cli_payload) + assert cli_payload["summary"]["unique_opportunity_count"] == 3, cli_payload + assert cli_output_json.exists(), cli_output_json + assert cli_output_md.exists(), cli_output_md + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_claim_guard.py b/tests/verify_world_class_claim_guard.py new file mode 100644 index 00000000..5ac30244 --- /dev/null +++ b/tests/verify_world_class_claim_guard.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_claim_guard.py" +TMP = ROOT / "tests" / "tmp_world_class_claim_guard" +sys.path.insert(0, str(ROOT / "scripts")) +from render_world_class_claim_guard import default_claim_surfaces, rel_path # noqa: E402 + + +def run_guard(*extra: str, check: bool = True, cwd: Path = ROOT) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra, + ], + cwd=cwd, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "world_class_claim_guard.json" + output_md = TMP / "world_class_claim_guard.md" + proc = run_guard("--output-json", str(output_json), "--output-md", str(output_md)) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "claim-guard-pass-evidence-pending", summary + assert summary["ledger_ready_to_claim_world_class"] is False, summary + assert summary["ledger_pending_count"] == 4, summary + assert summary["claim_surface_count"] >= 10, summary + assert summary["json_claim_surface_count"] >= 10, summary + assert summary["metadata_claim_surface_count"] >= summary["json_claim_surface_count"], summary + assert summary["package_claim_surface_count"] >= 5, summary + assert summary["violation_count"] == 0, summary + assert summary["overclaim_guard_active"] is True, summary + assert any(item["path"] == "README.md" for item in payload["scanned_surfaces"]), payload["scanned_surfaces"] + assert any(item["path"] == "manifest.json" for item in payload["scanned_surfaces"]), payload["scanned_surfaces"] + assert any(item["path"] == "evidence/world_class/README.md" for item in payload["scanned_surfaces"]), payload[ + "scanned_surfaces" + ] + assert any(item["path"] == "agents/interface.yaml" for item in payload["scanned_surfaces"]), payload[ + "scanned_surfaces" + ] + assert any(item["path"] == "security/permission_policy.json" for item in payload["scanned_surfaces"]), payload[ + "scanned_surfaces" + ] + assert not any( + item["path"].startswith("dist/install-simulation/") for item in payload["scanned_surfaces"] + ), payload["scanned_surfaces"] + assert any(item["path"] == "reports/world_class_evidence_ledger.json" for item in payload["scanned_surfaces"]), payload["scanned_surfaces"] + + surface_fixture = TMP / "surface-fixture" + for rel in [ + "README.md", + "SKILL.md", + "manifest.json", + "agents/interface.yaml", + "dist/manifest.json", + "dist/targets/openai/adapter.json", + "dist/install-simulation/simulate-skill/SKILL.md", + "security/permission_policy.json", + "evidence/world_class/README.md", + "evidence/world_class/submissions/provider-holdout.json", + ]: + target = surface_fixture / rel + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("{}\n" if target.suffix == ".json" else "fixture\n", encoding="utf-8") + fixture_surfaces = {rel_path(path, surface_fixture) for path in default_claim_surfaces(surface_fixture)} + assert "dist/manifest.json" in fixture_surfaces, fixture_surfaces + assert "dist/targets/openai/adapter.json" in fixture_surfaces, fixture_surfaces + assert "dist/install-simulation/simulate-skill/SKILL.md" not in fixture_surfaces, fixture_surfaces + assert "evidence/world_class/submissions/provider-holdout.json" not in fixture_surfaces, fixture_surfaces + + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Claim Guard" in markdown, markdown + assert "claim-guard-pass-evidence-pending" in markdown, markdown + assert "world-class evidence ledger" in markdown, markdown + assert "JSON claim surfaces scanned" in markdown, markdown + assert "metadata claim surfaces scanned" in markdown, markdown + assert "package/runtime claim surfaces scanned" in markdown, markdown + + safe_surface = TMP / "safe.md" + safe_surface.write_text("ready to claim world-class: `false`\nworld-class evidence is pending.\n", encoding="utf-8") + safe_proc = run_guard( + "--claim-surface", str(safe_surface), + "--output-json", str(TMP / "safe_guard.json"), + "--output-md", str(TMP / "safe_guard.md"), + check=True, + ) + safe_payload = json.loads(safe_proc.stdout) + assert safe_payload["summary"]["violation_count"] == 0, safe_payload + + unsafe_surface = TMP / "unsafe.md" + unsafe_surface.write_text("ready to claim world-class: `true`\n世界级已完成\n", encoding="utf-8") + unsafe_proc = run_guard( + "--claim-surface", str(unsafe_surface), + "--output-json", str(TMP / "unsafe_guard.json"), + "--output-md", str(TMP / "unsafe_guard.md"), + check=False, + ) + assert unsafe_proc.returncode == 2, unsafe_proc.stdout + unsafe_payload = json.loads(unsafe_proc.stdout) + assert unsafe_payload["ok"] is False, unsafe_payload + assert unsafe_payload["summary"]["decision"] == "claim-blocked-overclaim", unsafe_payload + assert unsafe_payload["summary"]["violation_count"] == 2, unsafe_payload + rules = {item["rule"] for item in unsafe_payload["violations"]} + assert {"ready-to-claim-true", "zh-completion-phrase"} <= rules, unsafe_payload["violations"] + + relative_unsafe_surface = TMP / "relative-unsafe.md" + relative_unsafe_surface.write_text("world-class completed\n", encoding="utf-8") + relative_proc = run_guard( + "--claim-surface", + relative_unsafe_surface.relative_to(ROOT).as_posix(), + "--output-json", + str(TMP / "relative_unsafe_guard.json"), + "--output-md", + str(TMP / "relative_unsafe_guard.md"), + check=False, + cwd=TMP, + ) + assert relative_proc.returncode == 2, relative_proc.stdout + relative_payload = json.loads(relative_proc.stdout) + assert relative_payload["summary"]["violation_count"] == 1, relative_payload + assert relative_payload["violations"][0]["path"] == "tests/tmp_world_class_claim_guard/relative-unsafe.md", relative_payload + + unsafe_json_surface = TMP / "unsafe.json" + unsafe_json_surface.write_text('{"world_class_ready": true}\n', encoding="utf-8") + unsafe_json_proc = run_guard( + "--claim-surface", + str(unsafe_json_surface), + "--output-json", + str(TMP / "unsafe_json_guard.json"), + "--output-md", + str(TMP / "unsafe_json_guard.md"), + check=False, + ) + assert unsafe_json_proc.returncode == 2, unsafe_json_proc.stdout + unsafe_json_payload = json.loads(unsafe_json_proc.stdout) + assert unsafe_json_payload["summary"]["violation_count"] == 1, unsafe_json_payload + assert unsafe_json_payload["violations"][0]["rule"] == "json-world-class-ready-true", unsafe_json_payload + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py new file mode 100644 index 00000000..08b6bb05 --- /dev/null +++ b/tests/verify_world_class_evidence_intake.py @@ -0,0 +1,703 @@ +#!/usr/bin/env python3 +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py" +TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" +sys.path.insert(0, str(ROOT / "scripts")) + +from world_class_evidence_contract import validate_payload # noqa: E402 +from world_class_human_fixtures import assert_human_contract_artifact_validation # noqa: E402 +from world_class_native_telemetry_fixtures import assert_native_telemetry_contract_artifact_validation # noqa: E402 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_intake(*extra: str) -> dict: + extra_args = list(extra) + if "--output-json" not in extra_args: + extra_args.extend(["--output-json", str(TMP / "last_intake.json")]) + if "--output-md" not in extra_args: + extra_args.extend(["--output-md", str(TMP / "last_intake.md")]) + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra_args, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def provider_submission(*, valid: bool, artifact_path: str = "reports/output_execution_runs.json") -> dict: + return { + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": False, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "Yao provider operator", + "submitted_at": "2026-06-14", + "summary": "Aggregate provider-backed holdout evidence for ledger review.", + "artifact_refs": [ + { + "path": artifact_path, + "kind": "aggregate-report", + "contains_raw_content": not valid, + "sha256": sha256_file(ROOT / artifact_path) if valid else "example-only", + } + ], + "provenance": { + "provider": "deepseek", + "model": "deepseek-v4-flash", + "credential_material_committed": False, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": valid, + "reviewer_or_operator_identity_present": valid, + "artifact_refs_reviewed": valid, + "privacy_contract_satisfied": valid, + "ledger_reviewer_approved": valid, + "ledger_reviewer": "Yao ledger reviewer" if valid else "", + "ledger_reviewed_at": "2026-06-14" if valid else "", + }, + } + + +def external_submission( + skill_root: Path, + *, + evidence_key: str, + source_type: str, + category: str = "external", + submitted_by: str = "Yao external operator", + artifact_paths: tuple[str, ...], + artifact_kinds: dict[str, str] | None = None, + provenance: dict, +) -> dict: + artifact_kinds = artifact_kinds or {} + return { + "schema_version": "1.0", + "evidence_key": evidence_key, + "template_only": False, + "category": category, + "source_type": source_type, + "submitted_by": submitted_by, + "submitted_at": "2026-06-14", + "summary": f"Completed {evidence_key} evidence for ledger review.", + "artifact_refs": [ + { + "path": path, + "kind": artifact_kinds.get(path, "aggregate-report"), + "contains_raw_content": False, + "sha256": sha256_file(skill_root / path), + } + for path in artifact_paths + ], + "provenance": provenance, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-14", + }, + } + + +def write_provider_artifact(skill_root: Path, *, complete: bool) -> None: + run = { + "case_id": "provider-holdout", + "variant": "with_skill", + "status": "pass", + "execution_mode": "model" if complete else "command", + "model_executed": complete, + "command_executed": True, + "duration_ms": 1234.5, + "provider": "openai" if complete else "local-output-eval-runner", + "model": "gpt-4.1-mini" if complete else "", + "usage": { + "input_tokens": 20, + "output_tokens": 30, + "total_tokens": 50, + "estimated": not complete, + }, + "output_sha256": "a" * 64, + "failure": "", + } + write_json( + skill_root / "reports" / "output_execution_runs.json", + { + "schema_version": "1.0", + "ok": True, + "summary": { + "variant_run_count": 2, + "model_executed_count": 2 if complete else 0, + "timing_observed_count": 2, + "token_observed_count": 2 if complete else 0, + "failure_count": 0, + }, + "runs": [run, {**run, "case_id": "provider-holdout-2"}] if complete else [run], + }, + ) + + +def provider_artifact_submission(skill_root: Path) -> dict: + return external_submission( + skill_root, + evidence_key="provider-holdout", + source_type="provider-output-eval", + artifact_paths=("reports/output_execution_runs.json",), + provenance={ + "provider": "openai", + "model": "gpt-4.1-mini", + "credential_material_committed": False, + }, + ) + + +def write_native_permission_artifacts(skill_root: Path, *, complete: bool) -> None: + capabilities = ["file_write", "network", "subprocess"] + target = { + "target": "vscode", + "status": "pass", + "adapter": "dist/targets/vscode/adapter.json", + "permission_model": "native-client", + "native_enforcement": complete, + "metadata_fallback_explicit": not complete, + "installer_enforcement": { + "target": "vscode", + "source_status": "present", + "enforced": True, + "enforced_capabilities": capabilities, + "missing_capabilities": [], + "failure_details": [], + }, + "assurance": "native-enforced" if complete else "metadata-fallback-explicit", + "declared_capabilities": capabilities, + "checks": [ + {"key": "adapter-present", "passed": True, "detail": "vscode adapter.json is readable"}, + {"key": "native-enforcement-boolean", "passed": True, "detail": "native flag is boolean"}, + ], + "failures": [], + "residual_risks": [] if complete else ["Client-native permission enforcement is not provided by this target."], + } + write_json( + skill_root / "reports" / "runtime_permission_probes.json", + { + "schema_version": "1.0", + "ok": True, + "expected_capabilities": capabilities, + "summary": { + "target_count": 1, + "pass_count": 1, + "fail_count": 0, + "native_enforcement_count": 1 if complete else 0, + "metadata_fallback_count": 0 if complete else 1, + "failure_count": 0, + "installer_enforcement_ready": True, + }, + "targets": [target], + }, + ) + write_json( + skill_root / "reports" / "install_simulation.json", + { + "schema_version": "1.0", + "ok": True, + "summary": { + "installer_permission_enforced_count": 12, + "installer_permission_failure_count": 0, + "permission_target_count": 1, + "permission_capability_count": 3, + "failure_count": 0, + }, + "checks": [ + {"id": "permission-vscode-file_write-approved", "status": "pass", "detail": "file write approved"}, + { + "id": "permission-vscode-file_write-target-enforcement", + "status": "pass", + "detail": "file write enforced", + }, + {"id": "permission-vscode-network-approved", "status": "pass", "detail": "network approved"}, + {"id": "permission-vscode-network-target-enforcement", "status": "pass", "detail": "network enforced"}, + {"id": "permission-vscode-subprocess-approved", "status": "pass", "detail": "subprocess approved"}, + { + "id": "permission-vscode-subprocess-target-enforcement", + "status": "pass", + "detail": "subprocess enforced", + }, + ], + }, + ) + + +def native_permission_submission(skill_root: Path) -> dict: + return external_submission( + skill_root, + evidence_key="native-permission-enforcement", + source_type="runtime-permission-guard", + artifact_paths=("reports/runtime_permission_probes.json", "reports/install_simulation.json"), + artifact_kinds={ + "reports/runtime_permission_probes.json": "runtime-probe-report", + "reports/install_simulation.json": "installer-enforcement-report", + }, + provenance={ + "target": "vscode", + "guard_location": "VS Code extension runtime permission guard", + "guard_scope": "target-client-native", + "guard_blocks_undeclared_capability": True, + "metadata_fallback_retained_for_other_targets": True, + }, + ) + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def schema_contains_path_kind(schema: dict, path: str, kind: str) -> bool: + if isinstance(schema, dict): + properties = schema.get("properties", {}) + if isinstance(properties, dict): + path_schema = properties.get("path", {}) + kind_schema = properties.get("kind", {}) + if ( + isinstance(path_schema, dict) + and isinstance(kind_schema, dict) + and path_schema.get("const") == path + and kind_schema.get("const") == kind + ): + return True + return any(schema_contains_path_kind(value, path, kind) for value in schema.values()) + if isinstance(schema, list): + return any(schema_contains_path_kind(value, path, kind) for value in schema) + return False + + +def assert_external_contract_artifact_validation() -> None: + skill_root = TMP / "external_contract_root" + provider_entry = {"key": "provider-holdout", "category": "external"} + write_provider_artifact(skill_root, complete=True) + provider_valid = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert provider_valid["status"] == "pass", provider_valid + provider_duplicate_ref = provider_artifact_submission(skill_root) + provider_duplicate_ref["artifact_refs"].append(dict(provider_duplicate_ref["artifact_refs"][0])) + provider_duplicate_result = validate_payload( + provider_duplicate_ref, + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert provider_duplicate_result["status"] == "fail", provider_duplicate_result + assert any("must not duplicate another artifact reference" in error for error in provider_duplicate_result["errors"]), ( + provider_duplicate_result["errors"] + ) + provider_wrong_kind = provider_artifact_submission(skill_root) + provider_wrong_kind["artifact_refs"][0]["kind"] = "supporting-evidence" + provider_wrong_kind_result = validate_payload( + provider_wrong_kind, + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert provider_wrong_kind_result["status"] == "fail", provider_wrong_kind_result + assert any("kind must be aggregate-report" in error for error in provider_wrong_kind_result["errors"]), ( + provider_wrong_kind_result["errors"] + ) + provider_wrong_filename = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-copy.json", + root=skill_root, + template_expected=False, + ) + assert provider_wrong_filename["status"] == "fail", provider_wrong_filename + assert any("filename must be provider-holdout.json" in error for error in provider_wrong_filename["errors"]), provider_wrong_filename["errors"] + provider_leak = provider_artifact_submission(skill_root) + provider_leak["provenance"]["messages"] = ["raw messages must not be accepted"] + provider_leak["raw_prompt"] = "raw prompt must not be accepted" + provider_leak_result = validate_payload( + provider_leak, + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert provider_leak_result["status"] == "fail", provider_leak_result + assert any("raw content, credential, secret" in error for error in provider_leak_result["errors"]), provider_leak_result["errors"] + assert any("$.raw_prompt" in error and "$.provenance.messages" in error for error in provider_leak_result["errors"]), provider_leak_result["errors"] + provider_answer_key_leak = provider_artifact_submission(skill_root) + provider_answer_key_leak["provenance"]["Expected_Winner_Variant"] = "A" + provider_answer_key_leak["review_notes"] = [{"answer_key": "blind answer key must not be embedded"}] + provider_answer_key_leak_result = validate_payload( + provider_answer_key_leak, + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert provider_answer_key_leak_result["status"] == "fail", provider_answer_key_leak_result + assert any("answer-key fields" in error for error in provider_answer_key_leak_result["errors"]), ( + provider_answer_key_leak_result["errors"] + ) + assert any( + "$.provenance.Expected_Winner_Variant" in error and "$.review_notes[0].answer_key" in error + for error in provider_answer_key_leak_result["errors"] + ), provider_answer_key_leak_result["errors"] + write_provider_artifact(skill_root, complete=False) + provider_invalid = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert provider_invalid["status"] == "fail", provider_invalid + assert any("summary.model_executed_count must be >0" in error for error in provider_invalid["errors"]), provider_invalid["errors"] + assert any("summary.token_observed_count must be >0" in error for error in provider_invalid["errors"]), provider_invalid["errors"] + + write_provider_artifact(skill_root, complete=True) + forged = json.loads((skill_root / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) + for run in forged["runs"]: + run["provider"] = "local-output-eval-runner" + write_json(skill_root / "reports" / "output_execution_runs.json", forged) + forged_provider = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert forged_provider["status"] == "fail", forged_provider + assert any("matching provider, model, timing" in error for error in forged_provider["errors"]), forged_provider["errors"] + + write_provider_artifact(skill_root, complete=True) + raw_execution = json.loads((skill_root / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) + raw_execution["runs"][0]["raw_provider_prompt"] = "verbatim provider prompt must not be accepted" + raw_execution["runs"][0]["output"] = "raw model output must not be accepted" + raw_execution["run_notes"] = [{"messages": ["raw transcript-like exchange"]}] + write_json(skill_root / "reports" / "output_execution_runs.json", raw_execution) + raw_provider = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert raw_provider["status"] == "fail", raw_provider + assert any("output_execution_runs.runs[0].raw_provider_prompt" in error for error in raw_provider["errors"]), ( + raw_provider["errors"] + ) + assert any("output_execution_runs.runs[0].output" in error for error in raw_provider["errors"]), raw_provider["errors"] + assert any("output_execution_runs.run_notes[0].messages" in error for error in raw_provider["errors"]), ( + raw_provider["errors"] + ) + + permission_entry = {"key": "native-permission-enforcement", "category": "external"} + write_native_permission_artifacts(skill_root, complete=True) + permission_valid = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert permission_valid["status"] == "pass", permission_valid + forged_probe = json.loads((skill_root / "reports" / "runtime_permission_probes.json").read_text(encoding="utf-8")) + forged_probe["targets"][0]["native_enforcement"] = False + forged_probe["targets"][0]["assurance"] = "metadata-fallback-explicit" + forged_probe["targets"][0]["metadata_fallback_explicit"] = True + forged_probe["targets"][0]["residual_risks"] = ["Fallback still requires operator enforcement."] + write_json(skill_root / "reports" / "runtime_permission_probes.json", forged_probe) + permission_forged = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert permission_forged["status"] == "fail", permission_forged + assert any("native target rows must match summary.native_enforcement_count" in error for error in permission_forged["errors"]), ( + permission_forged["errors"] + ) + write_native_permission_artifacts(skill_root, complete=True) + raw_probe = json.loads((skill_root / "reports" / "runtime_permission_probes.json").read_text(encoding="utf-8")) + raw_probe["targets"][0]["messages"] = ["raw target client transcript must not be accepted"] + write_json(skill_root / "reports" / "runtime_permission_probes.json", raw_probe) + raw_permission_probe = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert raw_permission_probe["status"] == "fail", raw_permission_probe + assert any("runtime_permission_probes.targets[0].messages" in error for error in raw_permission_probe["errors"]), ( + raw_permission_probe["errors"] + ) + + write_native_permission_artifacts(skill_root, complete=True) + raw_install = json.loads((skill_root / "reports" / "install_simulation.json").read_text(encoding="utf-8")) + raw_install["checks"][0]["raw_prompt"] = "raw installer prompt must not be accepted" + write_json(skill_root / "reports" / "install_simulation.json", raw_install) + raw_permission_install = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert raw_permission_install["status"] == "fail", raw_permission_install + assert any("install_simulation.checks[0].raw_prompt" in error for error in raw_permission_install["errors"]), ( + raw_permission_install["errors"] + ) + + write_native_permission_artifacts(skill_root, complete=False) + permission_invalid = validate_payload( + native_permission_submission(skill_root), + permission_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-permission-enforcement.json", + root=skill_root, + template_expected=False, + ) + assert permission_invalid["status"] == "fail", permission_invalid + assert any("summary.native_enforcement_count must be >0" in error for error in permission_invalid["errors"]), permission_invalid["errors"] + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + empty_submissions = TMP / "empty-submissions" + empty_submissions.mkdir() + default_json = TMP / "world_class_evidence_intake.json" + default_md = TMP / "world_class_evidence_intake.md" + assert_human_contract_artifact_validation() + assert_external_contract_artifact_validation() + assert_native_telemetry_contract_artifact_validation() + payload = run_intake( + "--output-json", + str(default_json), + "--output-md", + str(default_md), + "--submissions-dir", + str(empty_submissions), + ) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "awaiting-submissions", summary + assert summary["schema_present"] is True, summary + assert summary["ledger_entry_count"] == 4, summary + assert summary["template_count"] == 4, summary + assert summary["template_pass_count"] == 4, summary + assert summary["submission_count"] == 0, summary + assert summary["valid_submission_count"] == 0, summary + assert summary["invalid_submission_count"] == 0, summary + assert summary["valid_packet_source_incomplete_count"] == 0, summary + assert summary["operator_checklist_count"] == 4, summary + assert summary["operator_checklist_ready_count"] == 0, summary + assert summary["ready_for_external_collection"] is True, summary + assert summary["ready_for_ledger_review"] is False, summary + assert summary["ready_to_claim_world_class"] is False, summary + assert summary["overclaim_guard_active"] is True, summary + assert {item["evidence_key"] for item in payload["templates"]} == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, payload["templates"] + assert all(item["status"] == "pass" and item["template_only"] is True for item in payload["templates"]), payload["templates"] + human_template = json.loads((ROOT / "evidence" / "world_class" / "templates" / "human-adjudication.intake.json").read_text(encoding="utf-8")) + decision_ref = next(item for item in human_template["artifact_refs"] if item["path"] == "reports/output_review_decisions.json") + assert "required reason" in decision_ref["note"], human_template + assert human_template["provenance"]["reviewed_at"] == "YYYY-MM-DD", human_template + assert human_template["provenance"]["reviewer_reason_required"] is True, human_template + assert "reason" in human_template["provenance"]["decision_fields"], human_template + assert "visible rubric" in human_template["privacy"]["notes"], human_template + assert "private customer data" in human_template["privacy"]["notes"], human_template + checklist = {item["evidence_key"]: item for item in payload["operator_checklist"]} + assert set(checklist) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, checklist + assert checklist["provider-holdout"]["readiness"] == "awaiting-submission", checklist["provider-holdout"] + assert checklist["provider-holdout"]["template_path"] == "evidence/world_class/templates/provider-holdout.intake.json", checklist["provider-holdout"] + empty_submissions_rel = str(empty_submissions.relative_to(ROOT)) + assert checklist["provider-holdout"]["submission_path"] == f"{empty_submissions_rel}/provider-holdout.json", checklist["provider-holdout"] + assert checklist["provider-holdout"]["commands"]["prepare_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + f"--evidence-key provider-holdout --output-dir {empty_submissions_rel}" + ), checklist["provider-holdout"] + assert checklist["provider-holdout"]["commands"]["validate_intake"] == f"python3 scripts/yao.py world-class-intake . --submissions-dir {empty_submissions_rel}", checklist["provider-holdout"] + assert checklist["provider-holdout"]["commands"]["submission_review"] == f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {empty_submissions_rel}", checklist["provider-holdout"] + assert checklist["provider-holdout"]["commands"]["refresh_ledger"] == f"python3 scripts/yao.py world-class-ledger . --submissions-dir {empty_submissions_rel}", checklist["provider-holdout"] + assert "provider-backed model run" in checklist["provider-holdout"]["must_collect"]["provenance_requirements"], checklist["provider-holdout"] + assert any("--provider-runner openai" in step for step in checklist["provider-holdout"]["must_collect"]["runbook"]), checklist["provider-holdout"] + assert any("--provider-runner deepseek" in step for step in checklist["provider-holdout"]["must_collect"]["runbook"]), checklist["provider-holdout"] + assert "reports/output_execution_runs.json summary.model_executed_count > 0" in checklist["provider-holdout"]["must_collect"]["success_checks"], checklist["provider-holdout"] + assert checklist["provider-holdout"]["anti_overclaim"]["local_command_runner_counts_as_provider_model"] is False, checklist["provider-holdout"] + assert checklist["provider-holdout"]["source_accepted"] is True, checklist["provider-holdout"] + assert checklist["provider-holdout"]["observed_state"]["model_executed_count"] > 0, checklist["provider-holdout"] + assert checklist["provider-holdout"]["observed_state"]["timing_observed_count"] > 0, checklist["provider-holdout"] + assert checklist["provider-holdout"]["observed_state"]["token_observed_count"] > 0, checklist["provider-holdout"] + markdown = default_md.read_text(encoding="utf-8") + assert "World-Class Evidence Intake" in markdown, markdown + assert "ready to claim world-class: `false`" in markdown, markdown + assert "Operator Checklist" in markdown, markdown + assert "valid packet but source incomplete: `0`" in markdown, markdown + assert "operator checklist: `0` ready / `4` total" in markdown, markdown + assert "0 existing / 0 sha256 verified / 0 required verified / 1 refs" in markdown, markdown + assert f"`{empty_submissions_rel}/provider-holdout.json`" in markdown, markdown + assert f"`python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir {empty_submissions_rel}`" in markdown, markdown + assert f"`python3 scripts/yao.py world-class-submission-review . --submissions-dir {empty_submissions_rel}`" in markdown, markdown + assert f"`python3 scripts/yao.py world-class-ledger . --submissions-dir {empty_submissions_rel}`" in markdown, markdown + assert "Source Runbook" in markdown, markdown + assert "--provider-runner openai" in markdown, markdown + assert "--provider-runner deepseek" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown + assert "Templates and planned work do not count as accepted evidence." in markdown, markdown + assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown + assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown + schema = json.loads((ROOT / "evidence" / "world_class" / "intake.schema.json").read_text(encoding="utf-8")) + assert schema_contains_path_kind(schema, "reports/output_execution_runs.json", "aggregate-report"), schema + assert schema_contains_path_kind(schema, "reports/output_review_decisions.json", "review-decisions"), schema + assert schema_contains_path_kind(schema, "reports/runtime_permission_probes.json", "runtime-probe-report"), schema + assert schema_contains_path_kind(schema, "reports/telemetry_hook_recipes.json", "hook-recipes"), schema + + valid_dir = TMP / "valid_submissions" + valid_dir.mkdir() + (valid_dir / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=True), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + valid_payload = run_intake("--submissions-dir", str(valid_dir)) + assert valid_payload["ok"] is True, valid_payload + assert valid_payload["summary"]["decision"] == "intake-ready-for-ledger-review", valid_payload["summary"] + assert valid_payload["summary"]["valid_submission_count"] == 1, valid_payload["summary"] + assert valid_payload["summary"]["invalid_submission_count"] == 0, valid_payload["summary"] + assert valid_payload["summary"]["valid_packet_source_incomplete_count"] == 0, valid_payload["summary"] + assert valid_payload["summary"]["operator_checklist_ready_count"] == 1, valid_payload["summary"] + assert valid_payload["summary"]["ready_for_ledger_review"] is True, valid_payload["summary"] + assert valid_payload["summary"]["ready_to_claim_world_class"] is False, valid_payload["summary"] + assert valid_payload["submissions"][0]["status"] == "pass", valid_payload["submissions"] + assert valid_payload["submissions"][0]["artifact_integrity"]["artifact_existing_count"] == 1, valid_payload["submissions"] + assert valid_payload["submissions"][0]["artifact_integrity"]["artifact_sha256_verified_count"] == 1, valid_payload["submissions"] + assert valid_payload["submissions"][0]["errors"] == [], valid_payload["submissions"] + valid_checklist = {item["evidence_key"]: item for item in valid_payload["operator_checklist"]} + assert valid_checklist["provider-holdout"]["readiness"] == "ready-for-ledger-review", valid_checklist["provider-holdout"] + assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["validate_intake"], valid_checklist["provider-holdout"] + assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["submission_review"], valid_checklist["provider-holdout"] + assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["refresh_ledger"], valid_checklist["provider-holdout"] + + placeholder_dir = TMP / "placeholder_submissions" + placeholder_dir.mkdir() + placeholder_submission = provider_submission(valid=True) + placeholder_submission["submitted_by"] = "operator with provider credentials" + placeholder_submission["submitted_at"] = "YYYY-MM-DD" + (placeholder_dir / "provider-holdout.json").write_text( + json.dumps(placeholder_submission, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + placeholder_payload = run_intake("--submissions-dir", str(placeholder_dir)) + assert placeholder_payload["ok"] is False, placeholder_payload + assert placeholder_payload["summary"]["decision"] == "fix-intake", placeholder_payload["summary"] + assert placeholder_payload["summary"]["invalid_submission_count"] == 1, placeholder_payload["summary"] + placeholder_errors = placeholder_payload["submissions"][0]["errors"] + assert any("submitted_by must not use template placeholder text" in error for error in placeholder_errors), placeholder_errors + assert any("submitted_at must use YYYY-MM-DD" in error for error in placeholder_errors), placeholder_errors + + invalid_dir = TMP / "invalid_submissions" + invalid_dir.mkdir() + (invalid_dir / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=False), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + invalid_payload = run_intake("--submissions-dir", str(invalid_dir)) + assert invalid_payload["ok"] is False, invalid_payload + assert invalid_payload["summary"]["decision"] == "fix-intake", invalid_payload["summary"] + assert invalid_payload["summary"]["invalid_submission_count"] == 1, invalid_payload["summary"] + assert invalid_payload["summary"]["operator_checklist_ready_count"] == 0, invalid_payload["summary"] + assert any("raw content" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"] + assert any("sha256" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"] + assert any("attestation.real_external_or_human_evidence" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"] + invalid_checklist = {item["evidence_key"]: item for item in invalid_payload["operator_checklist"]} + assert invalid_checklist["provider-holdout"]["readiness"] == "fix-submission", invalid_checklist["provider-holdout"] + + unrelated_dir = TMP / "unrelated_artifact_submissions" + unrelated_dir.mkdir() + (unrelated_dir / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=True, artifact_path="reports/context_budget.json"), ensure_ascii=False, indent=2) + + "\n", + encoding="utf-8", + ) + unrelated_payload = run_intake("--submissions-dir", str(unrelated_dir)) + assert unrelated_payload["ok"] is False, unrelated_payload + assert unrelated_payload["summary"]["decision"] == "fix-intake", unrelated_payload["summary"] + assert unrelated_payload["summary"]["invalid_submission_count"] == 1, unrelated_payload["summary"] + unrelated_errors = unrelated_payload["submissions"][0]["errors"] + assert any("required evidence artifact reports/output_execution_runs.json" in error for error in unrelated_errors), unrelated_errors + assert unrelated_payload["submissions"][0]["artifact_integrity"]["artifact_sha256_verified_count"] == 1, unrelated_payload["submissions"] + assert unrelated_payload["submissions"][0]["artifact_integrity"]["required_artifact_verified_count"] == 0, unrelated_payload["submissions"] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py new file mode 100644 index 00000000..430a1b12 --- /dev/null +++ b/tests/verify_world_class_evidence_ledger.py @@ -0,0 +1,587 @@ +#!/usr/bin/env python3 +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_evidence_ledger.py" +TMP = ROOT / "tests" / "tmp_world_class_evidence_ledger" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def provider_submission(artifact_root: Path = ROOT, artifact_path: str = "reports/output_execution_runs.json") -> dict: + return { + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": False, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "Yao provider operator", + "submitted_at": "2026-06-13", + "summary": "Aggregate provider-backed holdout evidence for ledger review.", + "artifact_refs": [ + { + "path": artifact_path, + "kind": "aggregate-report", + "contains_raw_content": False, + "sha256": sha256_file(artifact_root / artifact_path), + } + ], + "provenance": { + "provider": "deepseek", + "model": "deepseek-v4-flash", + "credential_material_committed": False, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-13", + }, + } + + +def accepted_provider_execution_report() -> dict: + return { + "schema_version": "1.0", + "ok": True, + "summary": { + "variant_run_count": 1, + "model_executed_count": 1, + "timing_observed_count": 1, + "token_observed_count": 1, + "failure_count": 0, + }, + "runs": [ + { + "case_id": "provider-holdout", + "variant": "with_skill", + "status": "pass", + "execution_mode": "model", + "model_executed": True, + "command_executed": True, + "duration_ms": 1234.5, + "provider": "deepseek", + "model": "deepseek-v4-flash", + "usage": { + "input_tokens": 20, + "output_tokens": 30, + "total_tokens": 50, + "estimated": False, + }, + "output_sha256": "a" * 64, + "failure": "", + } + ], + } + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + empty_submissions = TMP / "empty-submissions" + empty_submissions.mkdir() + output_json = TMP / "world_class_evidence_ledger.json" + output_md = TMP / "world_class_evidence_ledger.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--submissions-dir", + str(empty_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert payload["report_contract"]["contract"] == "world-class-evidence-ledger", payload + assert payload["report_contract"]["top_level_mirrors_summary"] is True, payload + for field in payload["report_contract"]["summary_fields"]: + assert payload[field] == summary[field], (field, payload[field], summary[field]) + assert summary["ledger_entry_count"] == 4, summary + assert summary["accepted_count"] == 0, summary + assert summary["pending_count"] == 4, summary + assert summary["human_pending_count"] == 1, summary + assert summary["external_pending_count"] == 3, summary + assert summary["submitted_entry_count"] == 0, summary + assert summary["missing_submission_count"] == 4, summary + assert summary["invalid_submission_count"] == 0, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary + assert summary["submitted_but_pending_count"] == 0, summary + assert summary["overclaim_guard_active"] is True, summary + assert summary["ready_to_claim_world_class"] is False, summary + assert payload["ready_to_claim_world_class"] is False, payload + assert payload["pending_count"] == 4, payload + assert payload["accepted_count"] == 0, payload + assert payload["source_blocked_count"] == summary["source_blocked_count"], payload + assert payload["artifacts"]["intake"] == "reports/world_class_evidence_intake.md", payload + entries = {entry["key"]: entry for entry in payload["entries"]} + assert set(entries) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, entries + assert entries["provider-holdout"]["source_accepted"] is True, entries["provider-holdout"] + assert entries["provider-holdout"]["observed_state"]["model_executed_count"] > 0, entries["provider-holdout"] + assert any("--provider-runner openai" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] + assert any("--provider-runner deepseek" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] + assert not any("" in step or "OPENAI_API_KEY=" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] + provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "pass", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "pass", provider_source + assert entries["provider-holdout"]["submission_state"]["status"] == "missing", entries["provider-holdout"] + assert entries["provider-holdout"]["submission_state"]["ledger_counts_as_completion"] is False, entries["provider-holdout"] + assert entries["human-adjudication"]["observed_state"]["pending_count"] == 5, entries["human-adjudication"] + assert entries["human-adjudication"]["observed_state"]["reviewer_metadata_present"] is False, entries["human-adjudication"] + assert entries["human-adjudication"]["observed_state"]["reason_required"] is True, entries["human-adjudication"] + assert entries["human-adjudication"]["observed_state"]["raw_content_allowed"] is False, entries["human-adjudication"] + assert entries["human-adjudication"]["observed_state"]["raw_content_path_count"] == 0, entries["human-adjudication"] + assert entries["human-adjudication"]["observed_state"]["ready_for_human_evidence"] is False, entries["human-adjudication"] + human_source = {item["field"]: item for item in entries["human-adjudication"]["source_checklist"]} + assert human_source["reviewer_metadata_present"]["status"] == "blocked", human_source + assert human_source["reason_required"]["status"] == "pass", human_source + assert human_source["raw_content_allowed"]["status"] == "pass", human_source + assert human_source["ready_for_human_evidence"]["status"] == "blocked", human_source + assert "choice and reason" in human_source["pending_count"]["next_action"], human_source + assert "metadata and rationale" in human_source["ready_for_human_evidence"]["next_action"], human_source + assert entries["native-permission-enforcement"]["observed_state"]["native_enforcement_count"] == 0, entries["native-permission-enforcement"] + assert entries["native-permission-enforcement"]["observed_state"]["installer_enforcement_pass_count"] >= 0, entries["native-permission-enforcement"] + assert any("summary.failure_count == 0" in check for check in entries["native-permission-enforcement"]["success_checks"]), entries["native-permission-enforcement"] + assert entries["native-client-telemetry"]["observed_state"]["external_source_events"] == 0, entries["native-client-telemetry"] + for entry in entries.values(): + assert entry["status"] == "pending", entry + assert entry["success_checks"], entry + assert entry["privacy_contract"], entry + assert entry["anti_overclaim"]["planned_work_counts_as_evidence"] is False, entry + assert entry["anti_overclaim"]["pending_review_counts_as_human_decision"] is False, entry + assert "submission_state" in entry, entry + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Evidence Ledger" in markdown, markdown + assert "overclaim guard active: `true`" in markdown, markdown + assert "submitted entries: `0`" in markdown, markdown + assert "source checks:" in markdown, markdown + assert "Source Runbook" in markdown, markdown + assert "--provider-runner openai" in markdown, markdown + assert "--provider-runner deepseek" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown + assert "Source Evidence Checks" in markdown, markdown + assert "| Provider model run | `10` | `>0` | `pass` |" in markdown, markdown + assert "`provider-holdout`" in markdown, markdown + + submissions = TMP / "submissions" + submissions.mkdir() + (submissions / "provider-holdout.json").write_text( + json.dumps(provider_submission(), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + submitted_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(TMP / "submitted_ledger.json"), + "--output-md", + str(TMP / "submitted_ledger.md"), + "--submissions-dir", + str(submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + submitted_payload = json.loads(submitted_proc.stdout) + submitted_summary = submitted_payload["summary"] + assert submitted_summary["submitted_entry_count"] == 1, submitted_summary + assert submitted_summary["submitted_but_pending_count"] == 0, submitted_summary + assert submitted_summary["invalid_submission_count"] == 0, submitted_summary + assert submitted_summary["accepted_count"] == 1, submitted_summary + assert submitted_summary["pending_count"] == 3, submitted_summary + assert submitted_summary["source_blocked_count"] >= 6, submitted_summary + submitted_provider = {entry["key"]: entry for entry in submitted_payload["entries"]}["provider-holdout"] + assert submitted_provider["status"] == "accepted", submitted_provider + assert submitted_provider["submission_state"]["status"] == "submitted", submitted_provider + assert submitted_provider["submission_state"]["artifact_sha256_verified_count"] == 1, submitted_provider + assert submitted_provider["submission_state"]["attested_real_evidence"] is True, submitted_provider + assert submitted_provider["submission_state"]["ledger_counts_as_completion"] is False, submitted_provider + assert submitted_provider["submission_state"]["errors"] == [], submitted_provider + + accepted_source_skill = TMP / "accepted_source_skill" + (accepted_source_skill / "reports").mkdir(parents=True) + (accepted_source_skill / "reports" / "output_execution_runs.json").write_text( + json.dumps(accepted_provider_execution_report(), ensure_ascii=False, indent=2) + + "\n", + encoding="utf-8", + ) + accepted_source_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "accepted_source_without_submission.json"), + "--output-md", + str(TMP / "accepted_source_without_submission.md"), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + accepted_source_payload = json.loads(accepted_source_proc.stdout) + accepted_source_summary = accepted_source_payload["summary"] + assert accepted_source_summary["source_accepted_count"] == 1, accepted_source_summary + assert accepted_source_summary["accepted_count"] == 0, accepted_source_summary + assert accepted_source_summary["source_accepted_without_valid_submission_count"] == 1, accepted_source_summary + assert accepted_source_summary["source_check_count"] >= 13, accepted_source_summary + accepted_source_provider = { + entry["key"]: entry for entry in accepted_source_payload["entries"] + }["provider-holdout"] + assert accepted_source_provider["source_accepted"] is True, accepted_source_provider + assert accepted_source_provider["status"] == "pending", accepted_source_provider + assert accepted_source_provider["submission_state"]["status"] == "missing", accepted_source_provider + assert all(item["status"] == "pass" for item in accepted_source_provider["source_checklist"]), accepted_source_provider + + (accepted_source_skill / "reports" / "context_budget.json").write_text( + json.dumps({"summary": {"unrelated": True}}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + unrelated_accepted_submissions = TMP / "unrelated_accepted_submissions" + unrelated_accepted_submissions.mkdir() + (unrelated_accepted_submissions / "provider-holdout.json").write_text( + json.dumps( + provider_submission(accepted_source_skill, artifact_path="reports/context_budget.json"), + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + unrelated_accepted_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "unrelated_accepted_provider_ledger.json"), + "--output-md", + str(TMP / "unrelated_accepted_provider_ledger.md"), + "--submissions-dir", + str(unrelated_accepted_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + unrelated_accepted_payload = json.loads(unrelated_accepted_proc.stdout) + unrelated_accepted_summary = unrelated_accepted_payload["summary"] + assert unrelated_accepted_summary["source_accepted_count"] == 1, unrelated_accepted_summary + assert unrelated_accepted_summary["accepted_count"] == 0, unrelated_accepted_summary + assert unrelated_accepted_summary["invalid_submission_count"] == 1, unrelated_accepted_summary + unrelated_accepted_provider = { + entry["key"]: entry for entry in unrelated_accepted_payload["entries"] + }["provider-holdout"] + assert unrelated_accepted_provider["source_accepted"] is True, unrelated_accepted_provider + assert unrelated_accepted_provider["status"] == "pending", unrelated_accepted_provider + assert unrelated_accepted_provider["submission_state"]["status"] == "invalid-contract", unrelated_accepted_provider + assert any( + "required evidence artifact reports/output_execution_runs.json" in error + for error in unrelated_accepted_provider["submission_state"]["errors"] + ), unrelated_accepted_provider + + placeholder_accepted_submissions = TMP / "placeholder_accepted_submissions" + placeholder_accepted_submissions.mkdir() + placeholder_submission = provider_submission(accepted_source_skill) + placeholder_submission["submitted_by"] = "operator with provider credentials" + placeholder_submission["submitted_at"] = "YYYY-MM-DD" + (placeholder_accepted_submissions / "provider-holdout.json").write_text( + json.dumps(placeholder_submission, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + placeholder_accepted_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "placeholder_accepted_provider_ledger.json"), + "--output-md", + str(TMP / "placeholder_accepted_provider_ledger.md"), + "--submissions-dir", + str(placeholder_accepted_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + placeholder_accepted_payload = json.loads(placeholder_accepted_proc.stdout) + placeholder_accepted_summary = placeholder_accepted_payload["summary"] + assert placeholder_accepted_summary["source_accepted_count"] == 1, placeholder_accepted_summary + assert placeholder_accepted_summary["accepted_count"] == 0, placeholder_accepted_summary + assert placeholder_accepted_summary["invalid_submission_count"] == 1, placeholder_accepted_summary + placeholder_accepted_provider = { + entry["key"]: entry for entry in placeholder_accepted_payload["entries"] + }["provider-holdout"] + assert placeholder_accepted_provider["source_accepted"] is True, placeholder_accepted_provider + assert placeholder_accepted_provider["status"] == "pending", placeholder_accepted_provider + assert placeholder_accepted_provider["submission_state"]["status"] == "invalid-contract", placeholder_accepted_provider + assert any( + "submitted_by must not use template placeholder text" in error + for error in placeholder_accepted_provider["submission_state"]["errors"] + ), placeholder_accepted_provider + + unapproved_submissions = TMP / "unapproved_submissions" + unapproved_submissions.mkdir() + unapproved_submission = provider_submission(accepted_source_skill) + unapproved_submission["attestation"]["ledger_reviewer_approved"] = False + (unapproved_submissions / "provider-holdout.json").write_text( + json.dumps(unapproved_submission, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + unapproved_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "unapproved_provider_ledger.json"), + "--output-md", + str(TMP / "unapproved_provider_ledger.md"), + "--submissions-dir", + str(unapproved_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + unapproved_payload = json.loads(unapproved_proc.stdout) + unapproved_summary = unapproved_payload["summary"] + assert unapproved_summary["source_accepted_count"] == 1, unapproved_summary + assert unapproved_summary["submitted_entry_count"] == 0, unapproved_summary + assert unapproved_summary["reviewer_approved_submission_count"] == 0, unapproved_summary + assert unapproved_summary["accepted_count"] == 0, unapproved_summary + unapproved_provider = { + entry["key"]: entry for entry in unapproved_payload["entries"] + }["provider-holdout"] + assert unapproved_provider["source_accepted"] is True, unapproved_provider + assert unapproved_provider["status"] == "pending", unapproved_provider + assert unapproved_provider["submission_state"]["status"] == "invalid-contract", unapproved_provider + assert unapproved_provider["submission_state"]["ledger_reviewer_approved"] is False, unapproved_provider + assert any( + "attestation.ledger_reviewer_approved must be true" in error + for error in unapproved_provider["submission_state"]["errors"] + ), unapproved_provider + + missing_reviewer_submissions = TMP / "missing_reviewer_submissions" + missing_reviewer_submissions.mkdir() + missing_reviewer_submission = provider_submission(accepted_source_skill) + missing_reviewer_submission["attestation"]["ledger_reviewer"] = "" + (missing_reviewer_submissions / "provider-holdout.json").write_text( + json.dumps(missing_reviewer_submission, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + missing_reviewer_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "missing_reviewer_provider_ledger.json"), + "--output-md", + str(TMP / "missing_reviewer_provider_ledger.md"), + "--submissions-dir", + str(missing_reviewer_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + missing_reviewer_payload = json.loads(missing_reviewer_proc.stdout) + missing_reviewer_provider = { + entry["key"]: entry for entry in missing_reviewer_payload["entries"] + }["provider-holdout"] + assert missing_reviewer_provider["status"] == "pending", missing_reviewer_provider + assert missing_reviewer_provider["submission_state"]["status"] == "invalid-contract", missing_reviewer_provider + assert any( + "attestation.ledger_reviewer is required" in error + for error in missing_reviewer_provider["submission_state"]["errors"] + ), missing_reviewer_provider + + self_approved_submissions = TMP / "self_approved_submissions" + self_approved_submissions.mkdir() + self_approved_submission = provider_submission(accepted_source_skill) + self_approved_submission["attestation"]["ledger_reviewer"] = self_approved_submission["submitted_by"] + (self_approved_submissions / "provider-holdout.json").write_text( + json.dumps(self_approved_submission, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + self_approved_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "self_approved_provider_ledger.json"), + "--output-md", + str(TMP / "self_approved_provider_ledger.md"), + "--submissions-dir", + str(self_approved_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + self_approved_payload = json.loads(self_approved_proc.stdout) + self_approved_provider = { + entry["key"]: entry for entry in self_approved_payload["entries"] + }["provider-holdout"] + assert self_approved_provider["status"] == "pending", self_approved_provider + assert self_approved_provider["submission_state"]["status"] == "invalid-contract", self_approved_provider + assert any( + "attestation.ledger_reviewer must be different from submitted_by" in error + for error in self_approved_provider["submission_state"]["errors"] + ), self_approved_provider + + early_review_submissions = TMP / "early_review_submissions" + early_review_submissions.mkdir() + early_review_submission = provider_submission(accepted_source_skill) + early_review_submission["submitted_at"] = "2026-06-13" + early_review_submission["attestation"]["ledger_reviewed_at"] = "2026-06-12" + (early_review_submissions / "provider-holdout.json").write_text( + json.dumps(early_review_submission, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + early_review_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "early_review_provider_ledger.json"), + "--output-md", + str(TMP / "early_review_provider_ledger.md"), + "--submissions-dir", + str(early_review_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + early_review_payload = json.loads(early_review_proc.stdout) + early_review_provider = { + entry["key"]: entry for entry in early_review_payload["entries"] + }["provider-holdout"] + assert early_review_provider["source_accepted"] is True, early_review_provider + assert early_review_provider["status"] == "pending", early_review_provider + assert early_review_provider["submission_state"]["status"] == "invalid-contract", early_review_provider + assert any( + "attestation.ledger_reviewed_at must be at or after submitted_at" in error + for error in early_review_provider["submission_state"]["errors"] + ), early_review_provider + + accepted_submissions = TMP / "accepted_submissions" + accepted_submissions.mkdir() + (accepted_submissions / "provider-holdout.json").write_text( + json.dumps(provider_submission(accepted_source_skill), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + accepted_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "accepted_provider_ledger.json"), + "--output-md", + str(TMP / "accepted_provider_ledger.md"), + "--submissions-dir", + str(accepted_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + accepted_payload = json.loads(accepted_proc.stdout) + accepted_summary = accepted_payload["summary"] + assert accepted_summary["source_accepted_count"] == 1, accepted_summary + assert accepted_summary["accepted_count"] == 1, accepted_summary + assert accepted_summary["reviewer_approved_submission_count"] == 1, accepted_summary + assert accepted_summary["pending_count"] == 3, accepted_summary + assert accepted_summary["source_accepted_without_valid_submission_count"] == 0, accepted_summary + accepted_provider = {entry["key"]: entry for entry in accepted_payload["entries"]}["provider-holdout"] + assert accepted_provider["status"] == "accepted", accepted_provider + assert accepted_provider["submission_state"]["status"] == "submitted", accepted_provider + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_evidence_plan.py b/tests/verify_world_class_evidence_plan.py new file mode 100644 index 00000000..96b9cd43 --- /dev/null +++ b/tests/verify_world_class_evidence_plan.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_evidence_plan.py" +TMP = ROOT / "tests" / "tmp_world_class_evidence" + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "world_class_evidence_plan.json" + output_md = TMP / "world_class_evidence_plan.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["decision"] == "collect-external-evidence", payload + assert payload["summary"]["ready_to_claim_world_class"] is False, payload + assert payload["summary"]["ledger_completion_required"] is True, payload + assert payload["summary"]["evidence_requirement_count"] == 4, payload + assert payload["summary"]["task_count"] == 4, payload + assert payload["summary"]["human_task_count"] == 1, payload + assert payload["summary"]["external_task_count"] == 3, payload + assert payload["artifacts"]["ledger"] == "reports/world_class_evidence_ledger.md", payload + assert payload["artifacts"]["intake"] == "reports/world_class_evidence_intake.md", payload + tasks = {item["key"]: item for item in payload["tasks"]} + requirements = {item["key"]: item for item in payload["evidence_requirements"]} + assert set(tasks) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, tasks + assert set(requirements) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, requirements + assert requirements["provider-holdout"]["status"] == "pass", requirements["provider-holdout"] + provider_runbook = requirements["provider-holdout"]["runbook"] + assert any("--provider-runner openai" in command for command in provider_runbook), requirements["provider-holdout"] + assert any("--provider-runner deepseek" in command for command in provider_runbook), requirements["provider-holdout"] + assert any(command.startswith("Set one provider API key") for command in provider_runbook), requirements["provider-holdout"] + assert any("DEEPSEEK_API_KEY" in command for command in provider_runbook), requirements["provider-holdout"] + assert not any("" in command or "OPENAI_API_KEY=" in command for command in provider_runbook), requirements["provider-holdout"] + assert any("world-class-intake . --submissions-dir evidence/world_class/submissions" in command for command in provider_runbook), requirements["provider-holdout"] + assert any("evidence/world_class/templates/provider-holdout.intake.json" in command for command in provider_runbook), requirements["provider-holdout"] + assert tasks["provider-holdout"]["status"] == "pass", tasks["provider-holdout"] + assert any("--provider-runner deepseek" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] + assert "reports/output_review_decisions.json" in tasks["human-adjudication"]["evidence_artifacts"], tasks["human-adjudication"] + assert any("output-review-import" in command for command in tasks["human-adjudication"]["runbook"]), tasks["human-adjudication"] + assert any("required reason" in command for command in tasks["human-adjudication"]["runbook"]), tasks["human-adjudication"] + assert any("summary.reviewer_metadata_present is true" in check for check in tasks["human-adjudication"]["success_checks"]), tasks["human-adjudication"] + assert any("summary.ready_for_human_evidence is true" in check for check in tasks["human-adjudication"]["success_checks"]), tasks["human-adjudication"] + assert "scripts/import_output_review_decisions.py" in tasks["human-adjudication"]["evidence_artifacts"], tasks["human-adjudication"] + assert any("importer rejects raw prompt" in item for item in tasks["human-adjudication"]["privacy_contract"]), tasks["human-adjudication"] + assert any("Reviewer reasons must be rubric-based" in item for item in tasks["human-adjudication"]["privacy_contract"]), tasks["human-adjudication"] + assert any("runtime-permissions" in command for command in tasks["native-permission-enforcement"]["runbook"]), tasks["native-permission-enforcement"] + assert any("install-simulate" in command for command in tasks["native-permission-enforcement"]["runbook"]), tasks["native-permission-enforcement"] + assert any("summary.failure_count == 0" in check for check in tasks["native-permission-enforcement"]["success_checks"]), tasks["native-permission-enforcement"] + assert any("installer_enforcement_pass_count" in check for check in tasks["native-permission-enforcement"]["success_checks"]), tasks["native-permission-enforcement"] + assert any("telemetry_native_host.py" in command for command in tasks["native-client-telemetry"]["runbook"]), tasks["native-client-telemetry"] + for task in tasks.values(): + assert task["success_checks"], task + assert task["evidence_artifacts"], task + assert "reports/world_class_evidence_intake.json" in task["evidence_artifacts"], task + assert task["privacy_contract"], task + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Evidence Plan" in markdown, markdown + assert "tasks: `4`" in markdown, markdown + assert "ready to claim world-class: `false`" in markdown, markdown + assert "ledger completion required: `true`" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_operator_runbook.py b/tests/verify_world_class_operator_runbook.py new file mode 100644 index 00000000..fef0414b --- /dev/null +++ b/tests/verify_world_class_operator_runbook.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +import hashlib +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_operator_runbook.py" +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_world_class_operator_runbook" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def provider_submission() -> dict: + return { + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": False, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "Yao provider operator", + "submitted_at": "2026-06-14", + "summary": "Aggregate provider-backed holdout evidence for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_execution_runs.json", + "kind": "aggregate-report", + "contains_raw_content": False, + "sha256": sha256_file(ROOT / "reports" / "output_execution_runs.json"), + } + ], + "provenance": {"provider": "deepseek", "model": "deepseek-v4-flash", "credential_material_committed": False}, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-14", + }, + } + + +def run_direct(*extra: str) -> dict: + proc = subprocess.run( + [sys.executable, str(SCRIPT), str(ROOT), "--generated-at", "2026-06-14", *extra], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def run_cli(*extra: str) -> dict: + env = dict(os.environ) + env["YAO_CLI_TELEMETRY"] = "0" + env.pop("YAO_CLI_TELEMETRY_EVENTS", None) + proc = subprocess.run( + [sys.executable, str(CLI), *extra], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + check=True, + ) + return json.loads(proc.stdout) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + empty_submissions = TMP / "empty-submissions" + empty_submissions.mkdir() + output_json = TMP / "world_class_operator_runbook.json" + output_md = TMP / "world_class_operator_runbook.md" + output_html = TMP / "world_class_operator_runbook.html" + payload = run_direct( + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--output-html", + str(output_html), + "--submissions-dir", + str(empty_submissions), + ) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "collect-evidence", summary + assert summary["evidence_item_count"] == 4, summary + assert summary["pending_count"] == 4, summary + assert summary["awaiting_submission_count"] == 4, summary + assert summary["ready_for_ledger_review_count"] == 0, summary + assert summary["valid_packet_source_incomplete_count"] == 0, summary + assert summary["invalid_submission_count"] == 0, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary + assert summary["repair_checklist_count"] >= summary["source_blocked_count"], summary + assert summary["phase_queue_count"] == 2, summary + assert summary["phase_queue_blocked_count"] == 2, summary + assert summary["phase_queue_row_count"] == summary["repair_checklist_count"], summary + assert summary["phase_queue_next_phase"] == "unblock-access", summary + assert summary["phase_queue_counts_as_completion"] is False, summary + assert summary["coordination_step_count"] == 6, summary + assert summary["coordination_user_required_step_count"] == 6, summary + assert summary["coordination_pending_evidence_keys"] == [ + "human-adjudication", + "native-client-telemetry", + "native-permission-enforcement", + "provider-holdout", + ], summary + assert summary["coordination_counts_as_completion"] is False, summary + assert summary["release_gate_ready"] is False, summary + assert summary["release_gate_blocked_count"] == 4, summary + assert summary["release_gate_check_count"] == 5, summary + assert summary["release_gate_counts_as_completion"] is False, summary + assert summary["ready_to_claim_world_class"] is False, summary + assert summary["runbook_counts_as_completion"] is False, summary + assert payload["repair_checklist"], payload + assert len(payload["phase_queue"]) == summary["phase_queue_count"], payload["phase_queue"] + assert sum(item["row_count"] for item in payload["phase_queue"]) == summary["phase_queue_row_count"], payload[ + "phase_queue" + ] + items = {item["evidence_key"]: item for item in payload["items"]} + assert set(items) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, items + coordination_plan = payload["coordination_plan"] + assert len(coordination_plan) == summary["coordination_step_count"], coordination_plan + assert all(step["counts_as_completion"] is False for step in coordination_plan), coordination_plan + coordination_by_key = {step["evidence_key"]: step for step in coordination_plan if step["evidence_key"]} + assert set(coordination_by_key) == set(items), coordination_by_key + assert "output-exec . --provider-runner " in coordination_by_key["provider-holdout"]["command"], ( + coordination_by_key + ) + assert "output-review-kit" in coordination_by_key["human-adjudication"]["command"], coordination_by_key + assert "runtime-permissions" in coordination_by_key["native-permission-enforcement"]["command"], ( + coordination_by_key + ) + assert "telemetry-import" in coordination_by_key["native-client-telemetry"]["command"], coordination_by_key + assert payload["release_gate"]["ready"] is False, payload["release_gate"] + assert payload["release_gate"]["blocked_count"] == summary["release_gate_blocked_count"], payload["release_gate"] + assert payload["release_gate"]["check_count"] == summary["release_gate_check_count"], payload["release_gate"] + assert payload["release_gate"]["counts_as_completion"] is False, payload["release_gate"] + release_checks = {item["key"]: item for item in payload["release_gate"]["checks"]} + assert release_checks["world_class_ledger_ready"]["passed"] is False, release_checks + assert release_checks["claim_guard_clean"]["passed"] is False, release_checks + assert release_checks["benchmark_public_claim_ready"]["passed"] is False, release_checks + assert release_checks["review_studio_clean"]["passed"] is False, release_checks + assert release_checks["evidence_consistency_clean"]["passed"] is True, release_checks + provider = items["provider-holdout"] + assert provider["review_state"] == "awaiting-submission", provider + assert provider["source_accepted"] is True, provider + assert provider["blocked_source_check_count"] == 0, provider + assert provider["repair_blocked_count"] == 1, provider + assert provider["repair_counts_as_completion"] is False, provider + assert provider["phase_queue_blocked_count"] == 1, provider + assert provider["phase_queue_counts_as_completion"] is False, provider + assert [item["phase"] for item in provider["phase_queue"]] == ["unblock-access"], provider + assert any("--provider-runner openai" in step for step in provider["execution_runbook"]), provider + assert any("--provider-runner deepseek" in step for step in provider["execution_runbook"]), provider + assert not any("" in step or "OPENAI_API_KEY=" in step for step in provider["execution_runbook"]), provider + assert provider["next_source_actions"] == [], provider + assert provider["commands"]["prepare_submission"].startswith("python3 scripts/yao.py world-class-submission-kit"), provider + assert "world-class-intake" in provider["commands"]["validate_intake"], provider + assert "world-class-submission-review" in provider["commands"]["review_queue"], provider + assert "world-class-ledger" in provider["commands"]["refresh_ledger"], provider + assert "world-class-claim-guard" in provider["commands"]["guard_claim"], provider + assert "provider-backed model run" in provider["must_collect"]["provenance_requirements"], provider + assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider["must_collect"]["success_checks"], provider + provider_source = {item["field"]: item for item in provider["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "pass", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "pass", provider_source + human = items["human-adjudication"] + human_source = {item["field"]: item for item in human["source_checklist"]} + assert human["observed_state"]["raw_content_allowed"] is False, human + assert human_source["raw_content_allowed"]["status"] == "pass", human_source + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Operator Runbook" in markdown, markdown + assert "runbook counts as completion: `false`" in markdown, markdown + assert "phase queue counts as completion: `false`" in markdown, markdown + assert "coordination counts as completion: `false`" in markdown, markdown + assert "release gate counts as completion: `false`" in markdown, markdown + assert "## Coordination Plan" in markdown, markdown + assert "`review-and-release-gate`" in markdown, markdown + assert "## Release Gate" in markdown, markdown + assert "world-class-claim-guard" in markdown, markdown + assert "## Phase Queue" in markdown, markdown + assert "| `unblock-access` | `blocked` |" in markdown, markdown + assert "Valid intake means ready for submission review; ledger review still requires passing source evidence." in markdown, markdown + assert "| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |" in markdown, markdown + assert "| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `0` | none | operator with provider credentials |" in markdown, markdown + assert "Source Runbook" in markdown, markdown + assert "### Phase Queue" in markdown, markdown + assert "--provider-runner openai" in markdown, markdown + assert "--provider-runner deepseek" in markdown, markdown + assert "" not in markdown, markdown + assert "OPENAI_API_KEY=" not in markdown, markdown + assert "### Next Source Actions" in markdown, markdown + assert "| Token usage observed | `10` | `>0` | `pass` |" in markdown, markdown + assert "Source Evidence Snapshot" in markdown, markdown + assert "| Check | Current | Expected | Status | Next action |" in markdown, markdown + assert "| Provider model run | `10` | `>0` | `pass` | Run provider-backed output-exec with real credentials. |" in markdown, markdown + html = output_html.read_text(encoding="utf-8") + assert "World-Class Operator Runbook" in html, html[:400] + assert "ledger and claim guard" in html, html + assert "position:sticky" in html, html + assert "Ready0" in html, html + assert "Invalid0" in html, html + assert f"Queue{summary['phase_queue_blocked_count']}/{summary['phase_queue_count']}" in html, html + assert f"Blocked{summary['source_blocked_count']}" in html, html + assert "
    Blocked
    0
    " in html, html + assert "
    Queue
    1
    " in html, html + assert "Phase Queue" in html, html + assert "Coordination Plan" in html, html + assert "Release Gate" in html, html + assert "review-and-release-gate" in html, html + assert "blocked-until-evidence-accepted" in html, html + assert "Next Source Actions" in html, html + assert "Source Runbook" in html, html + assert "--provider-runner openai" in html, html + assert "--provider-runner deepseek" in html, html + assert "<redacted>" not in html and "" not in html, html + assert "OPENAI_API_KEY=<redacted>" not in html, html + assert "Source Evidence Snapshot" in html, html + assert "model_executed_count" in html, html + assert "model_executed_count: 10 / >0" in html, html + assert "raw_content_allowed: False / false" in html, html + assert "= 6, submitted_summary + assert submitted_summary["phase_queue_count"] == 2, submitted_summary + assert submitted_summary["phase_queue_blocked_count"] == 2, submitted_summary + assert submitted_summary["phase_queue_counts_as_completion"] is False, submitted_summary + assert submitted_summary["ready_to_claim_world_class"] is False, submitted_summary + submitted_provider = {item["evidence_key"]: item for item in submitted["items"]}["provider-holdout"] + assert submitted_provider["ledger_status"] == "accepted", submitted_provider + assert submitted_provider["intake_readiness"] == "ready-for-ledger-review", submitted_provider + assert submitted_provider["review_state"] == "accepted", submitted_provider + assert submitted_provider["source_accepted"] is True, submitted_provider + assert submitted_provider["blocked_source_check_count"] == 0, submitted_provider + assert submitted_provider["phase_queue_blocked_count"] == 0, submitted_provider + assert submitted_provider["next_source_actions"] == [], submitted_provider + assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["validate_intake"], submitted_provider + assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["review_queue"], submitted_provider + assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["refresh_ledger"], submitted_provider + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_preflight.py b/tests/verify_world_class_preflight.py new file mode 100644 index 00000000..ebfd7daf --- /dev/null +++ b/tests/verify_world_class_preflight.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_preflight.py" +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_world_class_preflight" + + +def run_preflight(extra_env: dict[str, str] | None = None, *extra: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ) + env.pop("OPENAI_API_KEY", None) + env.pop("DEEPSEEK_API_KEY", None) + env.pop("YAO_OUTPUT_EVAL_MODEL", None) + if extra_env: + env.update(extra_env) + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-16", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + check=True, + ) + + +def run_cli(*extra: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ) + env["YAO_CLI_TELEMETRY"] = "0" + env.pop("OPENAI_API_KEY", None) + env.pop("DEEPSEEK_API_KEY", None) + return subprocess.run( + [sys.executable, str(CLI), "world-class-preflight", str(ROOT), *extra], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + check=True, + ) + + +def by_key(items: list[dict], key: str) -> dict: + return next(item for item in items if item.get("evidence_key") == key) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + + output_json = TMP / "world_class_evidence_preflight.json" + output_md = TMP / "world_class_evidence_preflight.md" + output_html = TMP / "world_class_evidence_preflight.html" + proc = run_preflight(None, "--output-json", str(output_json), "--output-md", str(output_md), "--output-html", str(output_html)) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["decision"] == "collection-preflight-blocked", summary + assert summary["ready_to_claim_world_class"] is False, summary + assert summary["preflight_counts_as_evidence"] is False, summary + assert summary["credential_value_exposed"] is False, summary + assert summary["evidence_item_count"] == 4, summary + assert summary["pending_count"] == 4, summary + assert summary["precheck_count"] >= 12, summary + assert summary["precheck_missing_count"] >= 1, summary + assert summary["precheck_external_required_count"] == 2, summary + assert summary["precheck_human_required_count"] == 1, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary + assert summary["repair_checklist_count"] == ( + summary["source_blocked_count"] + + summary["precheck_missing_count"] + + summary["precheck_human_required_count"] + + summary["precheck_external_required_count"] + ), summary + assert summary["repair_blocked_count"] == summary["repair_checklist_count"], summary + assert summary["repair_ready_count"] == 0, summary + assert summary["repair_phase_counts"]["unblock-access"] >= 1, summary + assert summary["repair_phase_counts"]["collect-source"] >= 1, summary + assert summary["phase_queue_count"] == 2, summary + assert summary["phase_queue_blocked_count"] == 2, summary + assert summary["phase_queue_row_count"] == summary["repair_checklist_count"], summary + assert summary["phase_queue_next_phase"] == "unblock-access", summary + assert summary["phase_queue_next_action_id"] == "human-adjudication-precheck-human-reviewer", summary + assert summary["phase_queue_next_command"] == ( + "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" + ), summary + assert summary["phase_queue_counts_as_completion"] is False, summary + assert summary["next_repair_action_id"] == "human-adjudication-precheck-human-reviewer", summary + assert summary["next_repair_phase"] == "unblock-access", summary + assert summary["next_repair_owner"] == "human reviewer", summary + assert summary["next_repair_command"] == ( + "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" + ), summary + assert summary["repair_counts_as_completion"] is False, summary + assert payload["submissions"]["preflight_counts_submission_as_completion"] is False, payload + assert payload["submissions"]["drafts_count_as_evidence"] is False, payload + assert payload["submissions"]["submission_kit_command"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--output-dir evidence/world_class/submissions" + ), payload["submissions"] + assert payload["submissions"]["submission_kit_prefill_command"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--output-dir evidence/world_class/submissions --prefill-artifacts" + ), payload["submissions"] + assert payload["submissions"]["artifact_prefill_counts_as_evidence"] is False, payload + phase_queue = payload["phase_queue"] + assert [item["phase"] for item in phase_queue] == ["unblock-access", "collect-source"], phase_queue + assert all(item["counts_as_completion"] is False for item in phase_queue), phase_queue + assert phase_queue[0]["status"] == "blocked", phase_queue + assert phase_queue[0]["blocked_count"] >= 1, phase_queue + assert phase_queue[0]["next_action_id"] == "human-adjudication-precheck-human-reviewer", phase_queue + assert phase_queue[0]["verification_command"] == ( + "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" + ), phase_queue + assert "human reviewer" in phase_queue[0]["owners"], phase_queue + assert "human-adjudication" in phase_queue[0]["evidence_keys"], phase_queue + assert phase_queue[1]["phase"] == "collect-source", phase_queue + assert phase_queue[1]["row_count"] == summary["source_blocked_count"], phase_queue + role_contract = payload["submissions"]["artifact_role_contract"] + assert role_contract["role_source"] == "world-class-submission-kit", role_contract + assert role_contract["counts_as_evidence"] is False, role_contract + assert role_contract["artifact_prefill_counts_as_evidence"] is False, role_contract + assert role_contract["submission_ref_total_count"] >= 4, role_contract + assert role_contract["submission_ref_ready_count"] <= role_contract["submission_ref_total_count"], role_contract + assert role_contract["supporting_evidence_total_count"] >= 4, role_contract + roles = {item["role"]: item for item in role_contract["roles"]} + assert roles["submission-ref"]["copy_to_artifact_refs"] is True, roles + assert roles["supporting-evidence"]["copy_to_artifact_refs"] is False, roles + assert payload["artifacts"]["html"] == "reports/world_class_evidence_preflight.html", payload["artifacts"] + submission_commands = payload["submissions"]["commands"] + assert submission_commands["prepare_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--output-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["prepare_prefilled_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--output-dir evidence/world_class/submissions --prefill-artifacts" + ), submission_commands + assert submission_commands["validate_intake"] == ( + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["submission_review"] == ( + "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["refresh_ledger"] == ( + "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["guard_claim"] == "python3 scripts/yao.py world-class-claim-guard .", submission_commands + + provider = by_key(payload["items"], "provider-holdout") + assert provider["status"] == "blocked", provider + assert provider["commands"]["prepare_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--evidence-key provider-holdout --output-dir evidence/world_class/submissions" + ), provider + assert provider["commands"]["prepare_prefilled_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts" + ), provider + assert provider["submission_kit"]["drafts_count_as_evidence"] is False, provider + assert provider["submission_kit"]["artifact_prefill_counts_as_evidence"] is False, provider + assert provider["submission_kit"]["prefill_command"] == provider["commands"]["prepare_prefilled_submission"], provider + assert provider["submission_kit"]["output_dir"] == "evidence/world_class/submissions", provider + assert provider["submission_kit"]["draft_path"] == "evidence/world_class/submissions/provider-holdout.json", provider + provider_roles = provider["submission_kit"]["artifact_role_contract"] + assert provider_roles["submission_ref_total_count"] == 1, provider_roles + assert provider_roles["submission_ref_ready_count"] == 1, provider_roles + assert provider_roles["supporting_evidence_total_count"] >= 1, provider_roles + provider_checks = {item["key"]: item for item in provider["prechecks"]} + assert provider_checks["provider-api-key"]["status"] == "missing", provider_checks + assert provider_checks["provider-api-key"]["actual"] == "not-set", provider_checks + assert provider_checks["provider-api-key"]["secret_value_redacted"] is True, provider_checks + assert provider_checks["provider-api-key"]["env_any"] == ["OPENAI_API_KEY", "DEEPSEEK_API_KEY"], provider_checks + assert "sk-test-secret" not in proc.stdout, proc.stdout + assert "OPENAI_API_KEY" in proc.stdout, proc.stdout + assert "DEEPSEEK_API_KEY" in proc.stdout, proc.stdout + provider_repairs = {item["target"]: item for item in provider["repair_checklist"]} + provider_phases = {item["phase"]: item for item in provider["phase_queue"]} + assert set(provider_phases) == {"unblock-access"}, provider_phases + assert provider_phases["unblock-access"]["next_action_id"] == "provider-holdout-precheck-provider-api-key", provider_phases + assert provider_repairs["provider-api-key"]["repair_type"] == "precheck", provider_repairs + assert provider_repairs["provider-api-key"]["action_id"] == "provider-holdout-precheck-provider-api-key", provider_repairs + assert provider_repairs["provider-api-key"]["phase"] == "unblock-access", provider_repairs + assert provider_repairs["provider-api-key"]["priority"] == 20, provider_repairs + assert provider_repairs["provider-api-key"]["owner"] == "operator with provider credentials", provider_repairs + assert provider_repairs["provider-api-key"]["verification_command"].endswith( + "world-class-preflight . --submissions-dir evidence/world_class/submissions" + ), provider_repairs + assert provider_repairs["provider-api-key"]["counts_as_completion"] is False, provider_repairs + provider_source = {item["field"]: item for item in provider["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "pass", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "pass", provider_source + + human = by_key(payload["items"], "human-adjudication") + assert human["status"] == "ready-for-human-review", human + human_checks = {item["key"]: item for item in human["prechecks"]} + assert human_checks["human-reviewer"]["status"] == "human-required", human_checks + assert "reviewer identity" in human["next_action"], human + assert any("Record a reviewer choice and reason" in row["next_action"] for row in human["source_checklist"]), human + assert any("required rationale" in item["next_action"] for item in human["prechecks"]), human + assert any("reviewed_at" in item["next_action"] for item in human["prechecks"]), human + human_repairs = {item["target"]: item for item in human["repair_checklist"]} + assert human_repairs["human-reviewer"]["repair_type"] == "precheck", human_repairs + assert human_repairs["human-reviewer"]["owner"] == "human reviewer", human_repairs + assert human_repairs["pending_count"]["repair_type"] == "source-check", human_repairs + assert "output-review" in human_repairs["pending_count"]["verification_command"], human_repairs + + native = by_key(payload["items"], "native-permission-enforcement") + assert native["status"] == "blocked", native + assert any(item["status"] == "external-required" for item in native["prechecks"]), native + + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Evidence Preflight" in markdown, markdown + assert "ready to claim world-class: `false`" in markdown, markdown + assert "preflight counts as evidence: `false`" in markdown, markdown + assert "credential value exposed: `false`" in markdown, markdown + assert "Submission Kit Handoff" in markdown, markdown + assert "Phase Queue" in markdown, markdown + assert "Phase queue rows group the same repair checklist into operator execution phases" in markdown, markdown + assert "phase queue: `2` blocked / `2` phases" in markdown, markdown + assert "next phase action: `human-adjudication-precheck-human-reviewer`" in markdown, markdown + assert "| Priority | Phase | Status | Rows | Owners | Evidence | Verify | Next action |" in markdown, markdown + assert "Repair Checklist" in markdown, markdown + assert "Repair rows convert preflight and source blockers into a prioritized operator queue" in markdown, markdown + assert "repair rows:" in markdown, markdown + assert "next repair action: `human-adjudication-precheck-human-reviewer`" in markdown, markdown + assert "| Priority | Phase | Owner | Evidence | Type | Target | Status | Verify | Next action |" in markdown, markdown + assert "`unblock-access`" in markdown, markdown + assert "operator with provider credentials" in markdown, markdown + assert "`provider-api-key`" in markdown, markdown + assert "Provider model run" in markdown, markdown + assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in markdown, markdown + assert "world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts" in markdown, markdown + assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions" in markdown, markdown + assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts" in markdown, markdown + assert "drafts count as evidence: `false`" in markdown, markdown + assert "artifact prefill counts as evidence: `false`" in markdown, markdown + assert "does not make a draft count as evidence" in markdown, markdown + assert "submission refs ready:" in markdown, markdown + assert "`submission-ref` rows are the only checklist rows expected in `artifact_refs`" in markdown, markdown + assert "| `supporting-evidence` | `false` |" in markdown, markdown + assert "values are never printed" in markdown, markdown + html = output_html.read_text(encoding="utf-8") + assert "World-Class Evidence Preflight" in html, html + assert "World-Class Evidence Preflight" in html, html + assert "Phase Queue" in html, html + assert "phase-row blocked" in html, html + assert "
    Rows
    " in html, html + assert "
    Counts
    " in html, html + assert "does not count as completion" in html, html + assert "Evidence Queue" in html, html + assert "Submission Kit" in html, html + assert "Repair Rows" in html, html + assert "Repairs" in html, html + assert "repair-row blocked" in html, html + assert "
    Priority
    " in html, html + assert "
    Phase
    " in html, html + assert "
    Owner
    " in html, html + assert "operator with provider credentials" in html, html + assert "world-class-preflight . --submissions-dir evidence/world_class/submissions" in html, html + assert "provider-api-key" in html, html + assert "Provider model run" in html, html + assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in html, html + assert "world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts" in html, html + assert "provider-holdout" in html, html + assert "Artifact prefill is convenience data only." in html, html + assert "artifact prefill counts as evidence" in html, html + assert "Artifact Roles" in html, html + assert "copy to artifact_refs: true" in html, html + assert "supporting-evidence" in html, html + assert "Source Checks" in html, html + assert "ready_to_claim_world_class" in html, html + assert "Environment variables are displayed only as set or not-set" in html, html + + env_json = TMP / "preflight_with_env.json" + env_proc = run_preflight( + {"DEEPSEEK_API_KEY": "sk-test-secret", "YAO_OUTPUT_EVAL_MODEL": "deepseek-test"}, + "--output-json", + str(env_json), + "--output-md", + str(TMP / "preflight_with_env.md"), + "--output-html", + str(TMP / "preflight_with_env.html"), + ) + env_payload = json.loads(env_proc.stdout) + env_provider = by_key(env_payload["items"], "provider-holdout") + env_provider_checks = {item["key"]: item for item in env_provider["prechecks"]} + assert env_provider_checks["provider-api-key"]["status"] == "pass", env_provider_checks + assert env_provider_checks["provider-api-key"]["actual"] == "set", env_provider_checks + assert env_provider_checks["provider-api-key"]["set_envs"] == ["DEEPSEEK_API_KEY"], env_provider_checks + assert env_provider_checks["provider-model"]["status"] == "pass", env_provider_checks + assert "sk-test-secret" not in env_proc.stdout, env_proc.stdout + assert "sk-test-secret" not in (TMP / "preflight_with_env.html").read_text(encoding="utf-8"), env_payload + assert env_payload["summary"]["credential_value_exposed"] is False, env_payload + assert env_payload["summary"]["ready_to_claim_world_class"] is False, env_payload + assert env_payload["summary"]["repair_checklist_count"] == payload["summary"]["repair_checklist_count"] - 1, env_payload + assert env_payload["summary"]["phase_queue_row_count"] == payload["summary"]["phase_queue_row_count"] - 1, env_payload + assert env_payload["summary"]["phase_queue_count"] == 2, env_payload + + spaced_dir = TMP / "submission kit spaced" + spaced_proc = run_preflight( + None, + "--submissions-dir", + str(spaced_dir), + "--output-json", + str(TMP / "preflight_spaced.json"), + "--output-md", + str(TMP / "preflight_spaced.md"), + "--output-html", + str(TMP / "preflight_spaced.html"), + ) + spaced_payload = json.loads(spaced_proc.stdout) + quoted_spaced = "'tests/tmp_world_class_preflight/submission kit spaced'" + assert quoted_spaced in spaced_payload["submissions"]["commands"]["prepare_submission"], spaced_payload["submissions"] + assert quoted_spaced in spaced_payload["submissions"]["commands"]["prepare_prefilled_submission"], spaced_payload["submissions"] + assert quoted_spaced in by_key(spaced_payload["items"], "provider-holdout")["commands"]["prepare_submission"], spaced_payload["items"] + assert quoted_spaced in by_key(spaced_payload["items"], "provider-holdout")["commands"]["prepare_prefilled_submission"], spaced_payload["items"] + + cli_proc = run_cli( + "--output-json", + str(TMP / "cli_preflight.json"), + "--output-md", + str(TMP / "cli_preflight.md"), + "--output-html", + str(TMP / "cli_preflight.html"), + "--generated-at", + "2026-06-16", + ) + cli_payload = json.loads(cli_proc.stdout) + assert cli_payload["summary"]["decision"] == "collection-preflight-blocked", cli_payload + assert cli_payload["summary"]["preflight_counts_as_evidence"] is False, cli_payload + assert (TMP / "cli_preflight.md").exists(), cli_payload + assert (TMP / "cli_preflight.html").exists(), cli_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_submission_kit.py b/tests/verify_world_class_submission_kit.py new file mode 100644 index 00000000..50bffc1a --- /dev/null +++ b/tests/verify_world_class_submission_kit.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +INTAKE_SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py" +KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py" +TMP = ROOT / "tests" / "tmp_world_class_submission_kit" + + +def run_intake(*extra: str) -> dict: + extra_args = list(extra) + if "--output-json" not in extra_args: + extra_args.extend(["--output-json", str(TMP / "last_intake.json")]) + if "--output-md" not in extra_args: + extra_args.extend(["--output-md", str(TMP / "last_intake.md")]) + proc = subprocess.run( + [ + sys.executable, + str(INTAKE_SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra_args, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def run_kit(*extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(KIT_SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + + +def assert_documented_submission_commands() -> None: + expected_fragments = [ + 'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"', + 'world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR"', + 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"', + 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts', + 'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"', + 'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"', + 'world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR"', + ] + for relative_path in ("README.md", "evidence/world_class/README.md"): + text = (ROOT / relative_path).read_text(encoding="utf-8") + for fragment in expected_fragments: + assert fragment in text, (relative_path, fragment) + assert "/tmp/yao-world-class-submission-kit" not in text, relative_path + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + assert_documented_submission_commands() + + kit_dir = TMP / "submission_kit" + kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") + kit_payload = json.loads(kit_proc.stdout) + assert kit_payload["ok"] is True, kit_payload + assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"] + assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["submission_ref_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["submission_ref_ready_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["supporting_artifact_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["supporting_artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"] + assert kit_payload["summary"]["source_pass_count"] == 3, kit_payload["summary"] + assert kit_payload["summary"]["source_blocked_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_submission_ref_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_submission_ref_ready_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["repair_checklist_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["repair_blocked_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["repair_ready_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["repair_phase_counts"] == {}, kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_blocked_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_row_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_next_phase"] == "", kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_next_action_id"] == "", kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_next_command"] == "", kit_payload["summary"] + assert kit_payload["summary"]["phase_queue_counts_as_completion"] is False, kit_payload["summary"] + assert kit_payload["summary"]["next_repair_action_id"] == "", kit_payload["summary"] + assert kit_payload["summary"]["next_repair_phase"] == "", kit_payload["summary"] + assert kit_payload["summary"]["next_repair_owner"] == "", kit_payload["summary"] + assert kit_payload["summary"]["next_repair_command"] == "", kit_payload["summary"] + assert kit_payload["summary"]["repair_counts_as_completion"] is False, kit_payload["summary"] + assert kit_payload["summary"]["handoff_step_count"] == 7, kit_payload["summary"] + assert kit_payload["summary"]["handoff_blocked_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["handoff_fix_required_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["handoff_counts_as_completion"] is False, kit_payload["summary"] + assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] + assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] + assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] + assert kit_payload["safety"]["operator_handoff_counts_as_evidence"] is False, kit_payload["safety"] + assert "review_submission" in kit_payload["commands"], kit_payload["commands"] + assert kit_payload["files"][0]["output_path"].endswith( + f"tests/{TMP.name}/submission_kit/provider-holdout.json" + ), kit_payload["files"] + + assert len(kit_payload["evidence_matrix"]) == 1, kit_payload["evidence_matrix"] + matrix_row = kit_payload["evidence_matrix"][0] + assert matrix_row["evidence_key"] == "provider-holdout", matrix_row + assert matrix_row["stage"] == "validate-packet", matrix_row + assert matrix_row["draft_status"] == "written", matrix_row + assert matrix_row["artifact_ready_count"] >= 1, matrix_row + assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row + assert matrix_row["submission_ref_ready_count"] == 1, matrix_row + assert matrix_row["submission_ref_total_count"] == 1, matrix_row + assert matrix_row["supporting_artifact_ready_count"] >= 1, matrix_row + assert matrix_row["supporting_artifact_total_count"] >= matrix_row["supporting_artifact_ready_count"], matrix_row + assert matrix_row["source_pass_count"] == 3, matrix_row + assert matrix_row["source_check_count"] == 3, matrix_row + assert matrix_row["source_blocked_count"] == 0, matrix_row + assert matrix_row["counts_as_completion"] is False, matrix_row + assert "validate intake" in matrix_row["next_action"].lower() or "packet" in matrix_row["next_action"].lower(), matrix_row + + repair_rows = {item["target"]: item for item in kit_payload["repair_checklist"]} + assert repair_rows == {}, repair_rows + + assert kit_payload["phase_queue"] == [], kit_payload["phase_queue"] + + handoff_steps = {item["step_id"]: item for item in kit_payload["operator_handoff"]} + assert list(handoff_steps) == [ + "prepare-drafts", + "collect-source", + "edit-submission", + "validate-intake", + "review-submission", + "refresh-ledger", + "guard-claim", + ], handoff_steps + assert handoff_steps["prepare-drafts"]["status"] == "ready", handoff_steps + assert handoff_steps["collect-source"]["status"] == "ready", handoff_steps + assert handoff_steps["collect-source"]["counts_as_completion"] is False, handoff_steps + assert handoff_steps["collect-source"]["blocking_condition"] == "", handoff_steps + assert "world-class-submission-review" in handoff_steps["review-submission"]["command"], handoff_steps + assert all(item["counts_as_completion"] is False for item in kit_payload["operator_handoff"]), handoff_steps + + artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} + assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["artifact_role"] == "submission-ref", artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["submission_ref_required"] is True, artifact_rows + assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows + assert artifact_rows["reports/output_execution_runs.md"]["artifact_role"] == "supporting-evidence", artifact_rows + assert artifact_rows["reports/output_execution_runs.md"]["submission_ref_required"] is False, artifact_rows + + source_rows = {item["field"]: item for item in kit_payload["source_checklist"]} + assert source_rows["model_executed_count"]["status"] == "pass", source_rows + assert source_rows["model_executed_count"]["actual"] > 0, source_rows + assert source_rows["timing_observed_count"]["status"] == "pass", source_rows + assert source_rows["token_observed_count"]["status"] == "pass", source_rows + + kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) + assert kit_draft["template_only"] is True, kit_draft + assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft + assert kit_draft["attestation"]["ledger_reviewer_approved"] is False, kit_draft + assert kit_draft["attestation"]["ledger_reviewer"] == "", kit_draft + assert kit_draft["attestation"]["ledger_reviewed_at"] == "", kit_draft + assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft + assert kit_draft["provenance"]["run_command"] == ( + "python3 scripts/yao.py output-exec --provider-runner --provider-model --timeout-seconds 60" + ), kit_draft + assert kit_draft["provenance"]["credential_env"] == "", kit_draft + assert "" not in json.dumps(kit_draft), kit_draft + + kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8")) + assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"] + assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"] + assert kit_manifest["summary"]["artifact_ref_prefill_count"] == 0, kit_manifest["summary"] + assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"] + assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"] + assert kit_manifest["repair_checklist"] == kit_payload["repair_checklist"], kit_manifest["repair_checklist"] + assert kit_manifest["phase_queue"] == kit_payload["phase_queue"], kit_manifest["phase_queue"] + assert kit_manifest["artifacts"]["html"].endswith( + f"tests/{TMP.name}/submission_kit/index.html" + ), kit_manifest["artifacts"] + + kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8") + assert "Drafts are not accepted evidence." in kit_readme, kit_readme + assert "Execution Runbook" in kit_readme, kit_readme + assert "--provider-runner openai" in kit_readme, kit_readme + assert "--provider-runner deepseek" in kit_readme, kit_readme + assert "" not in kit_readme, kit_readme + assert "validate intake" in kit_readme, kit_readme + assert "Artifact Checklist" in kit_readme, kit_readme + assert "Evidence Matrix" in kit_readme, kit_readme + assert "Phase Queue" in kit_readme, kit_readme + assert "This queue groups repair rows by execution phase" in kit_readme, kit_readme + assert "Repair Checklist" in kit_readme, kit_readme + assert "Operator Handoff" in kit_readme, kit_readme + assert "Handoff rows are procedural" in kit_readme, kit_readme + assert "ledger_reviewed_at" in kit_readme, kit_readme + assert "`review-submission`" in kit_readme, kit_readme + assert "Submission refs" in kit_readme, kit_readme + assert "Supporting assets" in kit_readme, kit_readme + assert "`submission-ref`" in kit_readme, kit_readme + assert "`supporting-evidence`" in kit_readme, kit_readme + assert "`collect-source`" in kit_readme, kit_readme + assert "Queue rows are procedural guidance only" in kit_readme, kit_readme + assert "Matrix rows are guidance only" in kit_readme, kit_readme + assert "Repair rows are procedural guidance and do not count as completion evidence." in kit_readme, kit_readme + assert "Source Evidence Snapshot" in kit_readme, kit_readme + assert "Provider model run" in kit_readme, kit_readme + assert "reports/output_execution_runs.json" in kit_readme, kit_readme + assert "Provider model run" in kit_readme, kit_readme + + kit_html = (kit_dir / "index.html").read_text(encoding="utf-8") + assert "World-Class Evidence Submission Kit" in kit_html, kit_html + assert "Drafts are not accepted evidence" in kit_html, kit_html + assert "provider-holdout" in kit_html, kit_html + assert "Artifact Checklist" in kit_html, kit_html + assert "Evidence Matrix" in kit_html, kit_html + assert "Phase Queue" in kit_html, kit_html + assert "No phase queue rows were generated" in kit_html, kit_html + assert "Repair Checklist" in kit_html, kit_html + assert "No repair rows were generated" in kit_html, kit_html + assert "collect-source" in kit_html, kit_html + assert "operator with provider credentials" in kit_html, kit_html + assert "--provider-runner openai" in kit_html, kit_html + assert "--provider-runner deepseek" in kit_html, kit_html + assert "Operator Handoff" in kit_html, kit_html + assert "handoff-card ready" in kit_html, kit_html + assert "does not count as completion" in kit_html, kit_html + assert "matrix-card validate-packet" in kit_html, kit_html + assert ( + f"
    Submission refs
    {matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready
    " + in kit_html + ), kit_html + assert ( + f"
    Supporting assets
    {matrix_row['supporting_artifact_ready_count']}/{matrix_row['supporting_artifact_total_count']} ready
    " + in kit_html + ), kit_html + assert "
    Source
    3/3 pass
    " in kit_html, kit_html + assert "Source Evidence Snapshot" in kit_html, kit_html + assert "
    Field
    model_executed_count
    " in kit_html, kit_html + assert "

    Provider model run

    " in kit_html, kit_html + assert "
    Current
    10
    " in kit_html, kit_html + assert "
    Expected
    >0
    " in kit_html, kit_html + assert "World-Class Evidence Submission Kit" in kit_html, kit_html + assert "Execution Runbook" in kit_html, kit_html + assert "--provider-runner openai" in kit_html, kit_html + assert "--provider-runner deepseek" in kit_html, kit_html + assert "<redacted>" not in kit_html and "" not in kit_html, kit_html + assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html + assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html + + prefilled_kit_dir = TMP / "prefilled_submission_kit" + prefilled_proc = run_kit( + "--output-dir", + str(prefilled_kit_dir), + "--evidence-key", + "provider-holdout", + "--prefill-artifacts", + ) + prefilled_payload = json.loads(prefilled_proc.stdout) + assert prefilled_payload["summary"]["artifact_prefill_enabled"] is True, prefilled_payload["summary"] + assert prefilled_payload["summary"]["artifact_ref_prefill_count"] == 1, prefilled_payload["summary"] + assert prefilled_payload["summary"]["artifact_ref_unfilled_count"] == 0, prefilled_payload["summary"] + assert prefilled_payload["summary"]["drafts_count_as_evidence"] is False, prefilled_payload["summary"] + assert prefilled_payload["safety"]["artifact_prefill_counts_as_evidence"] is False, prefilled_payload["safety"] + assert prefilled_payload["files"][0]["prefilled_artifact_ref_count"] == 1, prefilled_payload["files"] + prefilled_draft = json.loads((prefilled_kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) + assert prefilled_draft["template_only"] is True, prefilled_draft + assert prefilled_draft["attestation"]["real_external_or_human_evidence"] is False, prefilled_draft + assert prefilled_draft["attestation"]["ledger_reviewer_approved"] is False, prefilled_draft + assert len(prefilled_draft["artifact_refs"][0]["sha256"]) == 64, prefilled_draft + assert prefilled_draft["artifact_refs"][0]["contains_raw_content"] is False, prefilled_draft + prefilled_readme = (prefilled_kit_dir / "README.md").read_text(encoding="utf-8") + assert "Optional artifact prefill only inserts SHA-256 digests" in prefilled_readme, prefilled_readme + assert "does not mark a draft as real evidence" in prefilled_readme, prefilled_readme + prefilled_html = (prefilled_kit_dir / "index.html").read_text(encoding="utf-8") + assert "artifact prefill only inserts local SHA-256 digests" in prefilled_html, prefilled_html + assert "
    Prefill
    1 artifact refs
    " in prefilled_html, prefilled_html + + native_kit_dir = TMP / "native_permission_kit" + native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement") + native_kit_payload = json.loads(native_kit_proc.stdout) + native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]} + assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 1, native_kit_payload["summary"] + if "dist/targets/openai/adapter.json" in native_rows: + assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows + assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows + assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows + else: + glob_rows = [ + item for item in native_kit_payload["artifact_checklist"] + if item["source_pattern"] == "dist/targets/*/adapter.json" + ] + assert len(glob_rows) == 1, glob_rows + assert glob_rows[0]["status"] == "glob-no-match", glob_rows[0] + assert glob_rows[0]["artifact_ref_ready"] is False, glob_rows[0] + assert glob_rows[0]["concrete_reference_required"] is True, glob_rows[0] + native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8") + assert "Glob patterns are expanded into concrete files" in native_readme, native_readme + + draft_intake = run_intake("--submissions-dir", str(kit_dir)) + assert draft_intake["ok"] is False, draft_intake + assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"] + assert draft_intake["summary"]["invalid_submission_count"] == 1, draft_intake["summary"] + assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"] + assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"] + + spaced_kit_dir = TMP / "submission kit spaced" + spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout") + spaced_kit_payload = json.loads(spaced_kit_proc.stdout) + quoted_spaced_dir = f"'tests/{TMP.name}/submission kit spaced'" + assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"] + assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"] + spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8") + assert quoted_spaced_dir in spaced_readme, spaced_readme + + existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") + existing_payload = json.loads(existing_proc.stdout) + assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"] + assert existing_payload["files"][0]["status"] == "exists", existing_payload["files"] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_submission_review.py b/tests/verify_world_class_submission_review.py new file mode 100644 index 00000000..26943f16 --- /dev/null +++ b/tests/verify_world_class_submission_review.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +import hashlib +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_submission_review.py" +TMP = ROOT / "tests" / "tmp_world_class_submission_review" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def provider_submission(*, valid: bool = True) -> dict: + return { + "schema_version": "1.0", + "evidence_key": "provider-holdout", + "template_only": False, + "category": "external", + "source_type": "provider-output-eval", + "submitted_by": "Yao provider operator", + "submitted_at": "2026-06-14", + "summary": "Aggregate provider-backed holdout evidence for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_execution_runs.json", + "kind": "aggregate-report", + "contains_raw_content": not valid, + "sha256": sha256_file(ROOT / "reports" / "output_execution_runs.json") if valid else "example-only", + } + ], + "provenance": { + "provider": "deepseek", + "model": "deepseek-v4-flash", + "credential_material_committed": False, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": valid, + "reviewer_or_operator_identity_present": valid, + "artifact_refs_reviewed": valid, + "privacy_contract_satisfied": valid, + "ledger_reviewer_approved": valid, + "ledger_reviewer": "Yao ledger reviewer" if valid else "", + "ledger_reviewed_at": "2026-06-14" if valid else "", + }, + } + + +def run_review(*extra: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + empty_submissions = TMP / "empty-submissions" + empty_submissions.mkdir() + + output_json = TMP / "world_class_submission_review.json" + output_md = TMP / "world_class_submission_review.md" + default_proc = run_review( + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--submissions-dir", + str(empty_submissions), + ) + payload = json.loads(default_proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert payload["report_contract"]["contract"] == "world-class-submission-review", payload + assert payload["report_contract"]["top_level_mirrors_summary"] is True, payload + for field in payload["report_contract"]["summary_fields"]: + assert payload[field] == summary[field], (field, payload[field], summary[field]) + assert summary["decision"] == "awaiting-submissions", summary + assert summary["review_item_count"] == 4, summary + assert summary["awaiting_submission_count"] == 4, summary + assert summary["ready_for_ledger_review_count"] == 0, summary + assert summary["valid_packet_source_incomplete_count"] == 0, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary + assert summary["review_counts_submission_as_completion"] is False, summary + assert payload["ready_to_claim_world_class"] is False, payload + assert payload["review_item_count"] == 4, payload + assert payload["awaiting_submission_count"] == 4, payload + assert payload["source_blocked_count"] == summary["source_blocked_count"], payload + provider_item = {item["evidence_key"]: item for item in payload["items"]}["provider-holdout"] + assert provider_item["review_state"] == "awaiting-submission", provider_item + assert provider_item["submission_status"] == "missing", provider_item + assert provider_item["source_accepted"] is True, provider_item + provider_source = {item["field"]: item for item in provider_item["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "pass", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "pass", provider_source + human_item = {item["evidence_key"]: item for item in payload["items"]}["human-adjudication"] + human_source = {item["field"]: item for item in human_item["source_checklist"]} + assert human_item["observed_state"]["raw_content_allowed"] is False, human_item + assert human_item["observed_state"]["raw_content_path_count"] == 0, human_item + assert human_source["raw_content_allowed"]["status"] == "pass", human_source + assert "prompt_sha256" in " ".join(human_item["success_checks"]), human_item + assert "prompt_sha256" in " ".join(human_item["privacy_contract"]), human_item + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Submission Review" in markdown, markdown + assert "review counts submission as completion: `false`" in markdown, markdown + assert "Provider model run: 10 / >0 => pass" in markdown, markdown + assert "`provider-holdout`" in markdown, markdown + + submissions = TMP / "valid_submissions" + submissions.mkdir() + (submissions / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=True), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + # The kit manifest must not be treated as an evidence packet. + (submissions / "submission_manifest.json").write_text( + json.dumps({"schema_version": "1.0", "not": "an evidence packet"}, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + submitted_proc = run_review( + "--submissions-dir", + str(submissions), + "--output-json", + str(TMP / "submitted_review.json"), + "--output-md", + str(TMP / "submitted_review.md"), + check=False, + ) + assert submitted_proc.returncode == 0, submitted_proc.stdout + submitted_payload = json.loads(submitted_proc.stdout) + submitted_summary = submitted_payload["summary"] + assert submitted_payload["ok"] is True, submitted_payload + assert submitted_summary["decision"] == "awaiting-submissions", submitted_summary + assert submitted_summary["valid_packet_source_incomplete_count"] == 0, submitted_summary + assert submitted_summary["awaiting_submission_count"] == 3, submitted_summary + assert submitted_summary["invalid_submission_count"] == 0, submitted_summary + assert submitted_summary["accepted_count"] == 1, submitted_summary + assert submitted_summary["source_pass_count"] + submitted_summary["source_blocked_count"] == submitted_summary["source_check_count"], submitted_summary + assert submitted_summary["source_blocked_count"] >= 6, submitted_summary + submitted_provider = {item["evidence_key"]: item for item in submitted_payload["items"]}["provider-holdout"] + assert submitted_provider["review_state"] == "accepted", submitted_provider + assert submitted_provider["intake_status"] == "pass", submitted_provider + assert submitted_provider["submission_status"] == "submitted", submitted_provider + assert submitted_provider["artifact_ref_count"] == 1, submitted_provider + assert submitted_provider["source_accepted"] is True, submitted_provider + assert submitted_provider["intake_errors"] == [], submitted_provider + assert "model_executed_count" in submitted_provider["observed_state"], submitted_provider + submitted_source = {item["field"]: item for item in submitted_provider["source_checklist"]} + assert submitted_source["model_executed_count"]["status"] == "pass", submitted_source + assert submitted_source["timing_observed_count"]["status"] == "pass", submitted_source + + invalid_dir = TMP / "invalid_submissions" + invalid_dir.mkdir() + (invalid_dir / "provider-holdout.json").write_text( + json.dumps(provider_submission(valid=False), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + invalid_proc = run_review( + "--submissions-dir", + str(invalid_dir), + "--output-json", + str(TMP / "invalid_review.json"), + "--output-md", + str(TMP / "invalid_review.md"), + check=False, + ) + assert invalid_proc.returncode == 2, invalid_proc.stdout + invalid_payload = json.loads(invalid_proc.stdout) + assert invalid_payload["ok"] is False, invalid_payload + assert invalid_payload["summary"]["decision"] == "fix-submissions", invalid_payload["summary"] + invalid_provider = {item["evidence_key"]: item for item in invalid_payload["items"]}["provider-holdout"] + assert invalid_provider["review_state"] == "fix-submission", invalid_provider + assert invalid_provider["intake_errors"], invalid_provider + assert any("sha256" in error for error in invalid_provider["intake_errors"]), invalid_provider + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 7b4aaff1..9bd902e2 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -1,34 +1,20 @@ #!/usr/bin/env python3 import json -import os import subprocess import sys from pathlib import Path - -ROOT = Path(__file__).resolve().parent.parent -CLI = ROOT / "scripts" / "yao.py" -BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" -sys.path.insert(0, str(ROOT / "scripts")) -import yao_cli_config # noqa: E402 - - -def run(*args: str, input_text: str | None = None) -> dict: - proc = subprocess.run( - [sys.executable, str(CLI), *args], - cwd=ROOT, - capture_output=True, - text=True, - input=input_text, - env=os.environ, - ) - payload = json.loads(proc.stdout) - return { - "ok": proc.returncode == 0, - "returncode": proc.returncode, - "payload": payload, - "stderr": proc.stderr, - } +from yao_cli_helpers import ( + BENCHMARK_FIXTURE_DIR, + ROOT, + assert_cli_module_contracts, + assert_created_skill_reports, + assert_creation_report_view, + assert_help_surface, + refresh_root_report_consistency_inputs, + run, + run_with_env, +) def main() -> None: @@ -38,56 +24,118 @@ def main() -> None: tmp_root.mkdir(parents=True, exist_ok=True) remote_version = tmp_root / "remote-version.txt" remote_version.write_text("9.9.9\n", encoding="utf-8") - assert yao_cli_config.resolve_target("root")["title"] == "Root Description Optimization" - assert yao_cli_config.resolve_promotion_target("root") == "yao-meta-skill" - assert yao_cli_config.infer_archetype("Standardize team review workflow.", "")[0] == "production" - assert yao_cli_config.infer_archetype("Govern release policy.", "")[0] == "governed" - assert "--entry" in yao_cli_config.baseline_compare_args() + assert_cli_module_contracts() + assert_help_surface() init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root)) assert init_result["ok"], init_result created = Path(init_result["payload"]["root"]) assert (created / "SKILL.md").exists(), created assert (created / "README.md").exists(), created - assert (created / "reports" / "intent-dialogue.md").exists(), created - assert (created / "reports" / "intent-confidence.md").exists(), created - assert (created / "reports" / "skill-overview.html").exists(), created - assert (created / "reports" / "review-studio.html").exists(), created - assert (created / "reports" / "review-studio.json").exists(), created - assert (created / "reports" / "review-viewer.html").exists(), created - assert (created / "reports" / "reference-scan.md").exists(), created - assert (created / "reports" / "reference-synthesis.md").exists(), created - assert (created / "reports" / "output-risk-profile.md").exists(), created - assert (created / "reports" / "artifact-design-profile.md").exists(), created - assert (created / "reports" / "prompt-quality-profile.md").exists(), created - assert (created / "reports" / "system-model.md").exists(), created - assert (created / "reports" / "skill-ir.json").exists(), created - assert (created / "reports" / "compiled_targets.md").exists(), created - assert (created / "reports" / "compiled_targets.json").exists(), created - assert (created / "reports" / "iteration-directions.md").exists(), created - assert (created / "reports" / "adoption_drift_report.md").exists(), created - assert (created / "reports" / "adoption_drift_report.json").exists(), created - assert (created / "reports" / "review_waivers.md").exists(), created - assert (created / "reports" / "review_waivers.json").exists(), created - assert (created / "reports" / "review_annotations.md").exists(), created - assert (created / "reports" / "review_annotations.json").exists(), created + assert_created_skill_reports(created) assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created - init_report_view = init_result["payload"]["report_view"] - assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view - assert Path(init_report_view["html_report"]).exists(), init_report_view - assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view - assert Path(init_report_view["review_studio"]).exists(), init_report_view - assert "Skill 已创建完成" in init_report_view["message"], init_report_view - assert "Review Studio 2.0" in init_report_view["message"], init_report_view - assert "目标编译" in init_report_view["message"], init_report_view - assert "reports/compiled_targets.md" in init_report_view["message"], init_report_view - assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view - assert "默认使用中文简体" in init_report_view["message"], init_report_view - assert "切换英文版" in init_report_view["message"], init_report_view + assert_creation_report_view(init_result["payload"]["report_view"]) init_skill_ir = init_result["payload"]["skill_ir"] assert init_skill_ir["name"] == "cli-demo-skill", init_skill_ir assert init_skill_ir["trigger_samples"] >= 1, init_skill_ir + telemetry_log = tmp_root / "cli-telemetry-events.jsonl" + telemetry_env = { + "YAO_CLI_TELEMETRY": "1", + "YAO_CLI_TELEMETRY_EVENTS": str(telemetry_log), + } + telemetry_ok = run_with_env(telemetry_env, "validate", str(created)) + assert telemetry_ok["ok"], telemetry_ok + telemetry_fail = run_with_env( + telemetry_env, + "output-exec", + "--runner-command", + json.dumps([sys.executable, str(ROOT / "scripts" / "local_output_eval_runner.py")]), + "--provider-runner", + "openai", + ) + assert not telemetry_fail["ok"], telemetry_fail + telemetry_events = [json.loads(line) for line in telemetry_log.read_text(encoding="utf-8").splitlines()] + assert len(telemetry_events) == 2, telemetry_events + assert telemetry_events[0]["event"] == "script_run", telemetry_events + assert telemetry_events[0]["source"] == "yao_cli", telemetry_events + assert telemetry_events[0]["command"] == "validate", telemetry_events + assert telemetry_events[0]["outcome"] == "accepted", telemetry_events + assert telemetry_events[0]["failure_type"] == "none", telemetry_events + assert telemetry_events[1]["command"] == "output-exec", telemetry_events + assert telemetry_events[1]["outcome"] == "failed", telemetry_events + assert telemetry_events[1]["failure_type"] == "script_error", telemetry_events + + skill_os2_audit_result = run( + "skill-os2-audit", + str(ROOT), + "--output-json", + str(tmp_root / "skill_os2_audit.json"), + "--output-md", + str(tmp_root / "skill_os2_audit.md"), + "--generated-at", + "2026-06-13", + ) + assert skill_os2_audit_result["ok"], skill_os2_audit_result + assert skill_os2_audit_result["payload"]["summary"]["decision"] == "continue-iteration", skill_os2_audit_result + assert skill_os2_audit_result["payload"]["summary"]["world_class_ready"] is False, skill_os2_audit_result + + skill_os2_coverage_result = run( + "skill-os2-coverage", + str(ROOT), + "--output-json", + str(tmp_root / "skill_os2_coverage.json"), + "--output-md", + str(tmp_root / "skill_os2_coverage.md"), + "--generated-at", + "2026-06-14", + ) + assert skill_os2_coverage_result["ok"], skill_os2_coverage_result + assert skill_os2_coverage_result["payload"]["summary"]["local_blueprint_ready"] is True, skill_os2_coverage_result + assert skill_os2_coverage_result["payload"]["summary"]["public_world_class_ready"] is False, skill_os2_coverage_result + + python_compat_result = run( + "python-compat", + str(ROOT), + "--output-json", + str(tmp_root / "python_compatibility.json"), + "--output-md", + str(tmp_root / "python_compatibility.md"), + "--generated-at", + "2026-06-14", + ) + assert python_compat_result["ok"], python_compat_result + assert python_compat_result["payload"]["summary"]["target_python"] == "3.11", python_compat_result + assert python_compat_result["payload"]["summary"]["issue_count"] == 0, python_compat_result + assert python_compat_result["payload"]["summary"]["file_count"] >= 50, python_compat_result + + architecture_result = run( + "architecture-audit", + str(ROOT), + "--output-json", + str(tmp_root / "architecture_maintainability.json"), + "--output-md", + str(tmp_root / "architecture_maintainability.md"), + "--generated-at", + "2026-06-14", + ) + assert architecture_result["ok"], architecture_result + assert architecture_result["payload"]["summary"]["hotspot_count"] == 0, architecture_result + assert architecture_result["payload"]["summary"]["watchlist_count"] == 0, architecture_result + assert architecture_result["payload"]["summary"]["blocker_count"] == 0, architecture_result + assert architecture_result["payload"]["summary"]["command_handler_count"] >= 60, architecture_result + assert architecture_result["payload"]["summary"]["entrypoint_command_handler_count"] < 30, architecture_result + refresh_root_report_consistency_inputs(run, ROOT) + + evidence_consistency_result = run( + "evidence-consistency", str(ROOT), + "--output-json", str(tmp_root / "evidence_consistency.json"), + "--output-md", str(tmp_root / "evidence_consistency.md"), + "--generated-at", "2026-06-15", + ) + assert evidence_consistency_result["ok"], evidence_consistency_result + assert evidence_consistency_result["payload"]["summary"]["decision"] == "consistent", evidence_consistency_result + quickstart_result = run( "quickstart", "--output-dir", @@ -111,19 +159,13 @@ def main() -> None: ) assert quickstart_result["ok"], quickstart_result quickstart_root = Path(quickstart_result["payload"]["root"]) - assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root - assert (quickstart_root / "reports" / "review-studio.html").exists(), quickstart_root - assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "artifact-design-profile.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "prompt-quality-profile.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "system-model.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "compiled_targets.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "compiled_targets.json").exists(), quickstart_root - assert (quickstart_root / "reports" / "adoption_drift_report.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "review_waivers.md").exists(), quickstart_root - assert (quickstart_root / "reports" / "review_annotations.md").exists(), quickstart_root + quickstart_reports = [ + "review-viewer.html", "skill-interpretation.html", "skill-interpretation.json", "review-studio.html", + "github-benchmark-scan.md", "intent-confidence.md", "reference-synthesis.md", + "artifact-design-profile.md", "prompt-quality-profile.md", "system-model.md", "compiled_targets.md", + "compiled_targets.json", "adoption_drift_report.md", "review_waivers.md", "review_annotations.md", + ] + assert all((quickstart_root / "reports" / path).exists() for path in quickstart_reports), quickstart_root assert quickstart_result["payload"]["archetype"] == "production", quickstart_result assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result assert quickstart_result["payload"]["guidance"]["problem_diagnosis"]["candidates"], quickstart_result @@ -132,25 +174,22 @@ def main() -> None: assert quickstart_result["payload"]["reference_mode"]["mode"] == "silent", quickstart_result quickstart_report_view = quickstart_result["payload"]["report_view"] assert quickstart_report_view["html_report"].endswith("reports/skill-overview.html"), quickstart_report_view + assert quickstart_report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), quickstart_report_view assert Path(quickstart_report_view["html_report"]).exists(), quickstart_report_view + assert Path(quickstart_report_view["interpretation_report"]).exists(), quickstart_report_view assert Path(quickstart_report_view["review_studio"]).exists(), quickstart_report_view assert "Skill 已创建完成" in quickstart_report_view["message"], quickstart_report_view assert "默认使用中文简体" in quickstart_report_view["message"], quickstart_report_view - assert quickstart_result["payload"]["guidance"]["next_steps"][0].startswith("Open reports/skill-overview.html"), quickstart_result - assert "reports/review-studio.html" in quickstart_result["payload"]["guidance"]["next_steps"][2], quickstart_result - assert "audit report" in quickstart_result["payload"]["guidance"]["next_steps"][0], quickstart_result - assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["reference_synthesis"].endswith( - "reports/reference-synthesis.md" - ), quickstart_result - assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["prompt_quality_profile"].endswith( - "reports/prompt-quality-profile.md" - ), quickstart_result - assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["system_model"].endswith( - "reports/system-model.md" - ), quickstart_result - assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["review_studio"].endswith( - "reports/review-studio.html" - ), quickstart_result + assert quickstart_result["payload"]["guidance"]["next_steps"][0].startswith("Open reports/skill-interpretation.html"), quickstart_result + assert quickstart_result["payload"]["guidance"]["next_steps"][1].startswith("Open reports/skill-overview.html"), quickstart_result + assert "reports/review-studio.html" in quickstart_result["payload"]["guidance"]["next_steps"][3], quickstart_result + assert "interpretation report" in quickstart_result["payload"]["guidance"]["next_steps"][0], quickstart_result + evidence_artifacts = quickstart_result["payload"]["reviewer_evidence"]["artifacts"] + assert evidence_artifacts["reference_synthesis"].endswith("reports/reference-synthesis.md"), quickstart_result + assert evidence_artifacts["prompt_quality_profile"].endswith("reports/prompt-quality-profile.md"), quickstart_result + assert evidence_artifacts["system_model"].endswith("reports/system-model.md"), quickstart_result + assert evidence_artifacts["skill_interpretation"].endswith("reports/skill-interpretation.html"), quickstart_result + assert evidence_artifacts["review_studio"].endswith("reports/review-studio.html"), quickstart_result assert "uncertainty_or_conflict" not in quickstart_result["payload"], quickstart_result quickstart_manifest = json.loads((quickstart_root / "manifest.json").read_text(encoding="utf-8")) assert quickstart_manifest["status"] == "active", quickstart_manifest @@ -191,6 +230,14 @@ def main() -> None: assert skill_report_result["ok"], skill_report_result assert skill_report_result["payload"]["artifacts"]["html"].endswith("reports/skill-overview.html"), skill_report_result + skill_interpretation_result = run("skill-interpretation", str(created)) + assert skill_interpretation_result["ok"], skill_interpretation_result + assert skill_interpretation_result["payload"]["artifacts"]["html"].endswith( + "reports/skill-interpretation.html" + ), skill_interpretation_result + assert skill_interpretation_result["payload"]["summary"]["report_kind"] == "skill-interpretation", skill_interpretation_result + assert skill_interpretation_result["payload"]["summary"]["default_language"] == "zh-CN", skill_interpretation_result + review_viewer_result = run("review-viewer", str(created)) assert review_viewer_result["ok"], review_viewer_result assert review_viewer_result["payload"]["artifacts"]["html"].endswith("reports/review-viewer.html"), review_viewer_result @@ -198,7 +245,15 @@ def main() -> None: review_studio_result = run("review-studio", str(created)) assert review_studio_result["ok"], review_studio_result assert review_studio_result["payload"]["artifacts"]["html"].endswith("reports/review-studio.html"), review_studio_result - assert review_studio_result["payload"]["summary"]["gate_count"] == 13, review_studio_result + assert review_studio_result["payload"]["summary"]["gate_count"] == 16, review_studio_result + created_world_class_gate = next(item for item in review_studio_result["payload"]["gates"] if item["key"] == "world-class-evidence") + assert created_world_class_gate["status"] == "pass", created_world_class_gate + assert "optional" in created_world_class_gate["detail"], created_world_class_gate + created_architecture_gate = next( + item for item in review_studio_result["payload"]["gates"] if item["key"] == "architecture-maintainability" + ) + assert created_architecture_gate["status"] == "pass", created_architecture_gate + assert "optional" in created_architecture_gate["detail"], created_architecture_gate review_waivers_result = run( "review-waivers", @@ -328,9 +383,9 @@ def main() -> None: assert created_skill_ir["schema_version"] == "2.0.0", created_skill_ir assert created_skill_ir["trigger_surface"]["description"], created_skill_ir - compile_result = run("compile-skill", str(created), "--target", "openai", "--target", "claude", "--target", "generic") + compile_result = run("compile-skill", str(created), "--target", "openai", "--target", "claude", "--target", "generic", "--target", "vscode") assert compile_result["ok"], compile_result - assert compile_result["payload"]["summary"]["target_count"] == 3, compile_result + assert compile_result["payload"]["summary"]["target_count"] == 4, compile_result assert compile_result["payload"]["summary"]["block_count"] == 0, compile_result assert compile_result["payload"]["artifacts"]["markdown"].endswith("reports/compiled_targets.md"), compile_result @@ -369,6 +424,16 @@ def main() -> None: assert output_exec_result["payload"]["summary"]["recorded_fixture_count"] == 10, output_exec_result assert (created / "reports" / "output_execution_runs.md").exists(), output_exec_result + output_exec_conflict = run( + "output-exec", + "--runner-command", + json.dumps([sys.executable, str(ROOT / "scripts" / "local_output_eval_runner.py")]), + "--provider-runner", + "openai", + ) + assert not output_exec_conflict["ok"], output_exec_conflict + assert "Use either --runner-command or --provider-runner" in output_exec_conflict["payload"]["failures"][0], output_exec_conflict + output_review_result = run( "output-review", "--blind-pack", @@ -385,6 +450,9 @@ def main() -> None: assert output_review_result["ok"], output_review_result assert output_review_result["payload"]["summary"]["judgment_count"] == 0, output_review_result assert output_review_result["payload"]["summary"]["pending_count"] == 5, output_review_result + assert output_review_result["payload"]["summary"]["reviewer_checklist_count"] == 5, output_review_result + assert output_review_result["payload"]["summary"]["reviewer_checklist_pending_count"] == 5, output_review_result + assert all(not item["answer_key_visible"] for item in output_review_result["payload"]["reviewer_checklist"]), output_review_result assert (created / "reports" / "output_review_adjudication.md").exists(), output_review_result conformance_result = run("conformance", str(created)) @@ -458,7 +526,6 @@ def main() -> None: promote_result = run("promote-check") assert promote_result["ok"], promote_result assert promote_result["payload"]["summary"]["blocked"] == 0, promote_result - review_result = run("review", "--target", "root") assert review_result["ok"], review_result assert review_result["payload"]["artifacts"]["review_md"].endswith("reports/iteration_bundles/yao-meta-skill/review.md") @@ -475,17 +542,39 @@ def main() -> None: assert report_result["ok"], report_result assert "iteration_ledger" in report_result["payload"]["artifacts"], report_result assert "portability_score" in report_result["payload"]["artifacts"], report_result + assert "python_compatibility" in report_result["payload"]["artifacts"], report_result + assert "architecture_maintainability" in report_result["payload"]["artifacts"], report_result assert "artifact_design_profile" in report_result["payload"]["artifacts"], report_result assert "prompt_quality_profile" in report_result["payload"]["artifacts"], report_result assert "compiled_targets" in report_result["payload"]["artifacts"], report_result assert "output_execution" in report_result["payload"]["artifacts"], report_result + assert "output_review_kit" in report_result["payload"]["artifacts"], report_result assert "output_review_adjudication" in report_result["payload"]["artifacts"], report_result assert "adoption_drift" in report_result["payload"]["artifacts"], report_result assert "review_annotations" in report_result["payload"]["artifacts"], report_result + assert "world_class_evidence_plan" in report_result["payload"]["artifacts"], report_result + assert "world_class_evidence_ledger" in report_result["payload"]["artifacts"], report_result + assert "world_class_evidence_intake" in report_result["payload"]["artifacts"], report_result + assert "world_class_claim_guard" in report_result["payload"]["artifacts"], report_result + assert "benchmark_reproducibility" in report_result["payload"]["artifacts"], report_result + assert "evidence_consistency" in report_result["payload"]["artifacts"], report_result + assert all(key in report_result["payload"]["artifacts"] for key in ("skill_os2_audit", "skill_os2_coverage")), report_result + assert any(step["command"].startswith("render_skill_os2_audit.py ") and step["ok"] for step in report_result["payload"]["steps"]), report_result + assert "weekly_curator" in report_result["payload"]["artifacts"], report_result + assert report_result["payload"]["artifacts"]["skill_overview"] == "reports/skill-overview.json", report_result + assert report_result["payload"]["artifacts"]["skill_interpretation"] == "reports/skill-interpretation.json", report_result + assert report_result["payload"]["artifacts"]["skill_interpretation_html"] == "reports/skill-interpretation.html", report_result + assert (report_result["payload"]["artifacts"]["review_studio"], report_result["payload"]["artifacts"]["review_studio_html"]) == ("reports/review-studio.json", "reports/review-studio.html"), report_result + assert report_result["payload"]["artifacts"]["review_viewer"] == "reports/review-viewer.json", report_result + assert report_result["payload"]["artifacts"]["review_viewer_html"] == "reports/review-viewer.html", report_result report_output_execution = json.loads((ROOT / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) assert report_output_execution["summary"]["command_executed_count"] == 10, report_output_execution assert report_output_execution["summary"]["recorded_fixture_count"] == 0, report_output_execution - assert report_output_execution["summary"]["model_executed_count"] == 0, report_output_execution + if report_output_execution["summary"]["model_executed_count"] > 0: + assert report_output_execution["summary"]["token_observed_count"] == 10, report_output_execution + assert report_output_execution["summary"]["token_estimated_count"] == 0, report_output_execution + else: + assert report_output_execution["summary"]["model_executed_count"] == 0, report_output_execution package_dir = tmp_root / "dist" package_result = run("package", ".", "--platform", "generic", "--output-dir", str(package_dir)) @@ -505,6 +594,8 @@ def main() -> None: "claude", "--platform", "generic", + "--platform", + "vscode", "--expectations", str(ROOT / "evals" / "packaging_expectations.json"), "--output-dir", @@ -530,23 +621,9 @@ def main() -> None: "2026-06-13", ) assert package_verify_result["ok"], package_verify_result - assert package_verify_result["payload"]["summary"]["adapter_count"] == 3, package_verify_result + assert package_verify_result["payload"]["summary"]["adapter_count"] == 4, package_verify_result assert package_verify_result["payload"]["summary"]["archive_sha256"], package_verify_result - runtime_permissions_result = run( - "runtime-permissions", - ".", - "--package-dir", - str(package_zip_dir), - "--output-json", - str(tmp_root / "runtime_permission_probes.json"), - "--output-md", - str(tmp_root / "runtime_permission_probes.md"), - ) - assert runtime_permissions_result["ok"], runtime_permissions_result - assert runtime_permissions_result["payload"]["summary"]["metadata_fallback_count"] == 3, runtime_permissions_result - assert runtime_permissions_result["payload"]["summary"]["native_enforcement_count"] == 0, runtime_permissions_result - install_simulate_result = run( "install-simulate", ".", @@ -563,7 +640,26 @@ def main() -> None: ) assert install_simulate_result["ok"], install_simulate_result assert install_simulate_result["payload"]["summary"]["archive_extracted"], install_simulate_result - assert install_simulate_result["payload"]["summary"]["adapter_count"] == 3, install_simulate_result + assert install_simulate_result["payload"]["summary"]["adapter_count"] == 4, install_simulate_result + + runtime_permissions_result = run( + "runtime-permissions", + ".", + "--package-dir", + str(package_zip_dir), + "--install-simulation-json", + str(tmp_root / "install_simulation.json"), + "--output-json", + str(tmp_root / "runtime_permission_probes_with_install.json"), + "--output-md", + str(tmp_root / "runtime_permission_probes.md"), + ) + assert runtime_permissions_result["ok"], runtime_permissions_result + runtime_install_summary = runtime_permissions_result["payload"]["summary"] + assert runtime_install_summary["metadata_fallback_count"] == 4, runtime_install_summary + assert runtime_install_summary["native_enforcement_count"] == 0, runtime_install_summary + assert runtime_install_summary["installer_enforcement_pass_count"] == 4, runtime_install_summary + assert runtime_install_summary["installer_permission_failure_count"] == 0, runtime_install_summary upgrade_result = run( "upgrade-check", @@ -600,6 +696,5 @@ def main() -> None: print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) - if __name__ == "__main__": main() diff --git a/tests/verify_yao_cli_world_class.py b/tests/verify_yao_cli_world_class.py new file mode 100644 index 00000000..fe34c3a6 --- /dev/null +++ b/tests/verify_yao_cli_world_class.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +import json +import os +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +CLI = ROOT / "scripts" / "yao.py" +TMP = ROOT / "tests" / "tmp_yao_cli_world_class" + + +def run(*args: str) -> dict: + env = dict(os.environ) + env["YAO_CLI_TELEMETRY"] = "0" + env.pop("YAO_CLI_TELEMETRY_EVENTS", None) + proc = subprocess.run( + [sys.executable, str(CLI), *args], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + ) + payload = json.loads(proc.stdout) + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stderr": proc.stderr, + } + + +def main() -> None: + if TMP.exists(): + subprocess.run(["rm", "-rf", str(TMP)], check=True) + TMP.mkdir(parents=True, exist_ok=True) + empty_submissions = TMP / "empty_submissions" + empty_submissions.mkdir(parents=True, exist_ok=True) + + world_class_evidence_result = run( + "world-class-evidence", + str(ROOT), + "--output-json", + str(TMP / "world_class_evidence_plan.json"), + "--output-md", + str(TMP / "world_class_evidence_plan.md"), + "--generated-at", + "2026-06-13", + ) + assert world_class_evidence_result["ok"], world_class_evidence_result + assert world_class_evidence_result["payload"]["summary"]["decision"] == "collect-external-evidence", world_class_evidence_result + assert world_class_evidence_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_evidence_result + + world_class_ledger_result = run( + "world-class-ledger", + str(ROOT), + "--output-json", + str(TMP / "world_class_evidence_ledger.json"), + "--output-md", + str(TMP / "world_class_evidence_ledger.md"), + "--submissions-dir", + str(empty_submissions), + "--generated-at", + "2026-06-13", + ) + assert world_class_ledger_result["ok"], world_class_ledger_result + ledger_payload = world_class_ledger_result["payload"] + assert ledger_payload["report_contract"]["top_level_mirrors_summary"] is True, world_class_ledger_result + assert ledger_payload["pending_count"] == ledger_payload["summary"]["pending_count"] == 4, world_class_ledger_result + assert ledger_payload["summary"]["missing_submission_count"] == 4, world_class_ledger_result + assert ledger_payload["summary"]["submitted_entry_count"] == 0, world_class_ledger_result + + world_class_intake_result = run( + "world-class-intake", + str(ROOT), + "--submissions-dir", + str(empty_submissions), + "--output-json", + str(TMP / "world_class_evidence_intake.json"), + "--output-md", + str(TMP / "world_class_evidence_intake.md"), + "--generated-at", + "2026-06-14", + ) + assert world_class_intake_result["ok"], world_class_intake_result + intake_summary = world_class_intake_result["payload"]["summary"] + assert intake_summary["decision"] == "awaiting-submissions", world_class_intake_result + assert intake_summary["template_pass_count"] == 4, world_class_intake_result + assert intake_summary["operator_checklist_count"] == 4, world_class_intake_result + assert intake_summary["operator_checklist_ready_count"] == 0, world_class_intake_result + assert intake_summary["ready_to_claim_world_class"] is False, world_class_intake_result + provider_checklist = next( + item for item in world_class_intake_result["payload"]["operator_checklist"] if item["evidence_key"] == "provider-holdout" + ) + assert provider_checklist["readiness"] == "awaiting-submission", provider_checklist + empty_submissions_arg = str(empty_submissions.relative_to(ROOT)) + assert provider_checklist["commands"]["prepare_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + f"--evidence-key provider-holdout --output-dir {empty_submissions_arg}" + ), provider_checklist + assert world_class_ledger_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_ledger_result + + world_class_submission_kit_result = run( + "world-class-submission-kit", + str(ROOT), + "--output-dir", + str(TMP / "world_class_submission_kit"), + "--evidence-key", + "provider-holdout", + "--generated-at", + "2026-06-14", + "--output-html", + str(TMP / "world_class_submission_kit.html"), + ) + kit_payload = world_class_submission_kit_result["payload"] + assert world_class_submission_kit_result["ok"], world_class_submission_kit_result + assert kit_payload["summary"]["decision"] == "submission-kit-ready", world_class_submission_kit_result + assert kit_payload["summary"]["written_count"] == 1, world_class_submission_kit_result + assert kit_payload["summary"]["drafts_count_as_evidence"] is False, world_class_submission_kit_result + assert kit_payload["summary"]["repair_checklist_count"] == 0, world_class_submission_kit_result + assert kit_payload["summary"]["repair_counts_as_completion"] is False, world_class_submission_kit_result + assert kit_payload["repair_checklist"] == [], world_class_submission_kit_result + assert kit_payload["artifacts"]["html"].endswith("tests/tmp_yao_cli_world_class/world_class_submission_kit.html"), world_class_submission_kit_result + assert (TMP / "world_class_submission_kit" / "provider-holdout.json").exists(), world_class_submission_kit_result + assert (TMP / "world_class_submission_kit" / "submission_manifest.json").exists(), world_class_submission_kit_result + assert (TMP / "world_class_submission_kit.html").exists(), world_class_submission_kit_result + + world_class_submission_review_result = run( + "world-class-submission-review", + str(ROOT), + "--submissions-dir", + str(TMP / "world_class_submission_kit"), + "--output-json", + str(TMP / "world_class_submission_review.json"), + "--output-md", + str(TMP / "world_class_submission_review.md"), + "--generated-at", + "2026-06-14", + ) + assert not world_class_submission_review_result["ok"], world_class_submission_review_result + assert world_class_submission_review_result["returncode"] == 2, world_class_submission_review_result + review_payload = world_class_submission_review_result["payload"] + assert review_payload["report_contract"]["top_level_mirrors_summary"] is True, world_class_submission_review_result + assert review_payload["summary"]["decision"] == "fix-submissions", world_class_submission_review_result + assert review_payload["invalid_submission_count"] == review_payload["summary"]["invalid_submission_count"] == 1, world_class_submission_review_result + assert review_payload["items"][0]["review_state"] == "fix-submission", world_class_submission_review_result + assert (TMP / "world_class_submission_review.md").exists(), world_class_submission_review_result + + world_class_claim_guard_result = run( + "world-class-claim-guard", + str(ROOT), + "--output-json", + str(TMP / "world_class_claim_guard.json"), + "--output-md", + str(TMP / "world_class_claim_guard.md"), + "--generated-at", + "2026-06-14", + ) + assert world_class_claim_guard_result["ok"], world_class_claim_guard_result + assert world_class_claim_guard_result["payload"]["summary"]["decision"] == "claim-guard-pass-evidence-pending", world_class_claim_guard_result + assert world_class_claim_guard_result["payload"]["summary"]["violation_count"] == 0, world_class_claim_guard_result + assert world_class_claim_guard_result["payload"]["summary"]["ledger_pending_count"] == 4, world_class_claim_guard_result + + benchmark_reproducibility_result = run( + "benchmark-reproducibility", + str(ROOT), + "--output-json", + str(TMP / "benchmark_reproducibility.json"), + "--output-md", + str(TMP / "benchmark_reproducibility.md"), + "--generated-at", + "2026-06-13", + ) + assert benchmark_reproducibility_result["ok"], benchmark_reproducibility_result + assert benchmark_reproducibility_result["payload"]["summary"]["reproducibility_ready"] is True, benchmark_reproducibility_result + assert benchmark_reproducibility_result["payload"]["summary"]["world_class_ready"] is False, benchmark_reproducibility_result + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/world_class_human_fixtures.py b/tests/world_class_human_fixtures.py new file mode 100644 index 00000000..492d093f --- /dev/null +++ b/tests/world_class_human_fixtures.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path + +from world_class_evidence_contract import validate_payload + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def fixture_review_integrity() -> dict: + return { + "blind_pack_sha256": "f" * 64, + "case_count": 2, + "case_ids": ["case-a", "case-b"], + "prompt_sha256_by_case": { + "case-a": "a" * 64, + "case-b": "b" * 64, + }, + } + + +def fixture_reviewer_attestation() -> dict: + return { + "blind_review_completed_before_answer_key": True, + "answer_key_not_opened_before_decisions": True, + "raw_content_excluded": True, + "reviewer_reason_required": True, + } + + +def artifact_blind_pack_sha256(skill_root: Path) -> str: + path = skill_root / "reports" / "output_review_adjudication.json" + if not path.exists(): + return fixture_review_integrity()["blind_pack_sha256"] + payload = json.loads(path.read_text(encoding="utf-8")) + integrity = payload.get("review_integrity", {}) if isinstance(payload, dict) else {} + return str(integrity.get("blind_pack_sha256", fixture_review_integrity()["blind_pack_sha256"])) + + +def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict: + blind_pack_sha256 = artifact_blind_pack_sha256(skill_root) + return { + "schema_version": "1.0", + "evidence_key": "human-adjudication", + "template_only": False, + "category": "human", + "source_type": "blind-ab-review", + "submitted_by": reviewer, + "submitted_at": "2026-06-14", + "summary": "Completed blind A/B reviewer decisions for ledger review.", + "artifact_refs": [ + { + "path": "reports/output_review_adjudication.json", + "kind": "adjudication-report", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "output_review_adjudication.json"), + }, + { + "path": "reports/output_review_decisions.json", + "kind": "review-decisions", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "output_review_decisions.json"), + }, + ], + "provenance": { + "reviewer": reviewer, + "reviewed_at": "2026-06-14", + "blind_pack_path": "reports/output_blind_review_pack.md", + "blind_pack_sha256": blind_pack_sha256, + "answer_key_opened_after_decisions": True, + "blind_review_completed_before_answer_key": True, + "decision_fields": ["case_id", "winner_variant", "confidence", "reason", "reviewer_attestation", "review_integrity"], + "reviewer_reason_required": True, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-14", + }, + } + + +def human_pairs(*, complete: bool) -> list[dict]: + if complete: + return [ + { + "case_id": "case-a", + "status": "match", + "expected_revealed": True, + "reviewer_winner_variant": "A", + "confidence": 0.9, + "reason": "Variant A follows the rubric.", + }, + { + "case_id": "case-b", + "status": "disagree", + "expected_revealed": True, + "reviewer_winner_variant": "B", + "confidence": 0.8, + "reason": "Variant B handles the boundary.", + }, + ] + return [ + { + "case_id": "case-a", + "status": "match", + "expected_revealed": True, + "reviewer_winner_variant": "A", + "confidence": 0.9, + "reason": "Variant A follows the rubric.", + }, + { + "case_id": "case-b", + "status": "pending", + "expected_revealed": False, + "reviewer_winner_variant": "", + "confidence": None, + "reason": "", + }, + ] + + +def write_human_artifacts(skill_root: Path, *, complete: bool = True, reviewer: str = "Yao QA") -> None: + reviewed_at = "2026-06-14" + summary = { + "pair_count": 2, + "judgment_count": 2 if complete else 1, + "pending_count": 0 if complete else 1, + "agreement_count": 2 if complete else 1, + "disagreement_count": 0, + "invalid_decision_count": 0, + "answer_revealed_count": 2 if complete else 1, + "pending_answer_hidden_count": 0 if complete else 1, + "agreement_rate": 100.0, + "needs_review": not complete, + "failure_count": 0, + "reviewer_metadata_present": complete, + "reason_required": True, + "blind_review_attested": complete, + "blind_review_completed_before_answer_key": complete, + "answer_key_not_opened_before_decisions": complete, + "raw_content_excluded_attested": complete, + "reviewer_reason_required_attested": complete, + "ready_for_human_evidence": complete, + "reviewer_checklist_count": 2, + "reviewer_checklist_ready_count": 2 if complete else 1, + } + decisions = [ + {"case_id": "case-a", "winner_variant": "A", "confidence": 0.9, "reason": "Variant A follows the rubric."}, + { + "case_id": "case-b", + "winner_variant": "B" if complete else "", + "confidence": 0.8 if complete else None, + "reason": "Variant B handles the boundary." if complete else "", + }, + ] + write_json( + skill_root / "reports" / "output_review_adjudication.json", + { + "schema_version": "1.0", + "ok": True, + "summary": summary, + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "review_integrity": fixture_review_integrity(), + "reviewer_attestation": fixture_reviewer_attestation() if complete else {}, + "pairs": human_pairs(complete=complete), + "failures": [], + }, + ) + write_json( + skill_root / "reports" / "output_review_decisions.json", + { + "schema_version": "1.0", + "reviewer": reviewer, + "reviewed_at": reviewed_at, + "review_integrity": fixture_review_integrity(), + "reviewer_attestation": fixture_reviewer_attestation() if complete else {}, + "decisions": decisions, + }, + ) + + +def assert_human_submission_error(skill_root: Path, entry: dict, expected_error: str) -> None: + result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert result["status"] == "fail", result + assert any(expected_error in error for error in result["errors"]), result["errors"] + + +def assert_human_contract_artifact_validation() -> None: + entry = {"key": "human-adjudication", "category": "human"} + skill_root = TMP / "human_contract_root" + write_human_artifacts(skill_root, complete=True) + valid_result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert valid_result["status"] == "pass", valid_result + assert valid_result["artifact_integrity"]["required_artifact_verified_count"] == 2, valid_result + + wrong_kind_submission = human_submission(skill_root) + wrong_kind_submission["artifact_refs"][1]["kind"] = "aggregate-report" + wrong_kind_result = validate_payload( + wrong_kind_submission, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert wrong_kind_result["status"] == "fail", wrong_kind_result + assert any("kind must be review-decisions" in error for error in wrong_kind_result["errors"]), wrong_kind_result["errors"] + + write_human_artifacts(skill_root, complete=False) + pending_result = validate_payload( + human_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert pending_result["status"] == "fail", pending_result + assert any("summary.pending_count must be 0" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("summary.judgment_count must equal" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("summary.needs_review must be false" in error for error in pending_result["errors"]), pending_result["errors"] + assert any("A/B winner_variant" in error for error in pending_result["errors"]), pending_result["errors"] + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + mismatch_result = validate_payload( + human_submission(skill_root, reviewer="Different Reviewer"), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert mismatch_result["status"] == "fail", mismatch_result + assert any("provenance.reviewer must match decisions.reviewer" in error for error in mismatch_result["errors"]), mismatch_result["errors"] + + reviewed_at_mismatch = human_submission(skill_root, reviewer="Yao QA") + reviewed_at_mismatch["provenance"]["reviewed_at"] = "2026-06-15" + reviewed_at_result = validate_payload( + reviewed_at_mismatch, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert reviewed_at_result["status"] == "fail", reviewed_at_result + assert any("provenance.reviewed_at must match decisions.reviewed_at" in error for error in reviewed_at_result["errors"]), ( + reviewed_at_result["errors"] + ) + + answer_key_order = human_submission(skill_root, reviewer="Yao QA") + answer_key_order["provenance"]["answer_key_opened_after_decisions"] = False + answer_key_result = validate_payload( + answer_key_order, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert answer_key_result["status"] == "fail", answer_key_result + assert any("answer_key_opened_after_decisions" in error for error in answer_key_result["errors"]), answer_key_result["errors"] + + missing_reason_contract = human_submission(skill_root, reviewer="Yao QA") + missing_reason_contract["provenance"]["reviewer_reason_required"] = False + missing_reason_contract["provenance"]["decision_fields"] = ["case_id", "winner_variant", "confidence"] + reason_contract_result = validate_payload( + missing_reason_contract, + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "human-adjudication.json", + root=skill_root, + template_expected=False, + ) + assert reason_contract_result["status"] == "fail", reason_contract_result + assert any("reviewer_reason_required" in error for error in reason_contract_result["errors"]), reason_contract_result["errors"] + assert any("decision_fields must include" in error for error in reason_contract_result["errors"]), reason_contract_result["errors"] + + forged_adjudication = json.loads((skill_root / "reports" / "output_review_adjudication.json").read_text(encoding="utf-8")) + forged_adjudication["summary"]["ready_for_human_evidence"] = False + forged_adjudication["summary"]["reviewer_metadata_present"] = False + forged_adjudication["summary"]["reason_required"] = False + write_json(skill_root / "reports" / "output_review_adjudication.json", forged_adjudication) + assert_human_submission_error(skill_root, entry, "summary.ready_for_human_evidence must be true") + assert_human_submission_error(skill_root, entry, "summary.reviewer_metadata_present must be true") + assert_human_submission_error(skill_root, entry, "summary.reason_required must be true") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + raw_adjudication = json.loads((skill_root / "reports" / "output_review_adjudication.json").read_text(encoding="utf-8")) + raw_adjudication["pairs"][0]["prompt"] = "verbatim blind review prompt must not be accepted" + raw_adjudication["pairs"][0]["messages"] = ["raw reviewer transcript must not be accepted"] + write_json(skill_root / "reports" / "output_review_adjudication.json", raw_adjudication) + assert_human_submission_error(skill_root, entry, "output_review_adjudication.pairs[0].prompt") + assert_human_submission_error(skill_root, entry, "output_review_adjudication.pairs[0].messages") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][1]["case_id"] = "unknown-case" + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "case_id set must match adjudication pairs") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][0]["confidence"] = 1.2 + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "confidence must be between 0 and 1") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][0]["expected_winner_variant"] = "A" + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "answer-key fields") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][0]["metadata"] = {"raw_output": "verbatim model answer must not ship"} + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "decisions[1].metadata.raw_output") + + write_human_artifacts(skill_root, complete=True, reviewer="Yao QA") + forged_decisions = json.loads((skill_root / "reports" / "output_review_decisions.json").read_text(encoding="utf-8")) + forged_decisions["decisions"][0]["metadata"] = {"API_KEY": "credential material must not ship"} + write_json(skill_root / "reports" / "output_review_decisions.json", forged_decisions) + assert_human_submission_error(skill_root, entry, "decisions[1].metadata.API_KEY") diff --git a/tests/world_class_native_telemetry_fixtures.py b/tests/world_class_native_telemetry_fixtures.py new file mode 100644 index 00000000..129ffc3e --- /dev/null +++ b/tests/world_class_native_telemetry_fixtures.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path + +from world_class_evidence_contract import validate_payload + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, payload: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def native_telemetry_submission(skill_root: Path) -> dict: + return { + "schema_version": "1.0", + "evidence_key": "native-client-telemetry", + "template_only": False, + "category": "external", + "source_type": "native-client-telemetry", + "submitted_by": "Yao client integrator", + "submitted_at": "2026-06-14", + "summary": "Completed native-client-telemetry evidence for ledger review.", + "artifact_refs": [ + { + "path": "reports/adoption_drift_report.json", + "kind": "adoption-drift-report", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "adoption_drift_report.json"), + }, + { + "path": "reports/telemetry_hook_recipes.json", + "kind": "hook-recipes", + "contains_raw_content": False, + "sha256": sha256_file(skill_root / "reports" / "telemetry_hook_recipes.json"), + }, + ], + "provenance": { + "client": "Chrome extension production build", + "native_host_manifest": "/Users/laoyao/.config/chrome/native-hosts/yao-meta-skill.json", + "event_source": "external", + "metadata_only": True, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "attestation": { + "real_external_or_human_evidence": True, + "reviewer_or_operator_identity_present": True, + "artifact_refs_reviewed": True, + "privacy_contract_satisfied": True, + "ledger_reviewer_approved": True, + "ledger_reviewer": "Yao ledger reviewer", + "ledger_reviewed_at": "2026-06-14", + }, + } + + +def write_native_telemetry_artifacts(skill_root: Path, *, complete: bool) -> None: + recent_events = ( + [ + { + "command": "chrome-extension", + "event": "skill_activation", + "skill": "yao-meta-skill", + "source": "external", + "version": "1.1.0", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-14T10:00:00Z", + } + ] + if complete + else [] + ) + write_json( + skill_root / "reports" / "adoption_drift_report.json", + { + "schema_version": "2.0", + "ok": True, + "privacy_contract": { + "raw_content_allowed": False, + "raw_event_log_packaged": False, + }, + "summary": { + "source_types": {"external": 1 if complete else 0}, + "adoption_sample_count": 1 if complete else 0, + }, + "adoption_by_skill": [ + { + "skill": "yao-meta-skill", + "events": 1 if complete else 0, + "adoption_events": 1 if complete else 0, + "accepted": 1 if complete else 0, + "edited": 0, + "rejected": 0, + "missed": 0, + "adoption_rate": 100.0 if complete else 0, + } + ], + "recent_events": recent_events, + }, + ) + write_json( + skill_root / "reports" / "telemetry_hook_recipes.json", + { + "schema_version": "1.0", + "ok": True, + "privacy_contract": {"raw_content_allowed": False}, + "summary": {"recipe_count": 1, "metadata_only_recipe_count": 1}, + "recipes": [ + { + "id": "chrome-extension", + "source": "external", + "metadata_only": True, + "native_auto_capture": False, + "event": "skill_activation", + "outcome": "accepted", + } + ], + }, + ) + + +def assert_native_telemetry_contract_artifact_validation() -> None: + entry = {"key": "native-client-telemetry", "category": "external"} + skill_root = TMP / "native_telemetry_contract_root" + write_native_telemetry_artifacts(skill_root, complete=True) + valid = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert valid["status"] == "pass", valid + + forged_adoption = json.loads((skill_root / "reports" / "adoption_drift_report.json").read_text(encoding="utf-8")) + forged_adoption["recent_events"][0]["source"] = "manual" + write_json(skill_root / "reports" / "adoption_drift_report.json", forged_adoption) + forged = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert forged["status"] == "fail", forged + assert any("external event rows must cover summary.source_types.external" in error for error in forged["errors"]), ( + forged["errors"] + ) + + write_native_telemetry_artifacts(skill_root, complete=True) + raw_adoption = json.loads((skill_root / "reports" / "adoption_drift_report.json").read_text(encoding="utf-8")) + raw_adoption["recent_events"][0]["raw_prompt"] = "verbatim user prompt must not be accepted" + raw_adoption["recent_events"][0]["messages"] = ["raw transcript-like event payload"] + write_json(skill_root / "reports" / "adoption_drift_report.json", raw_adoption) + raw_adoption_result = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert raw_adoption_result["status"] == "fail", raw_adoption_result + assert any("adoption_drift_report.recent_events[0].raw_prompt" in error for error in raw_adoption_result["errors"]), ( + raw_adoption_result["errors"] + ) + assert any("adoption_drift_report.recent_events[0].messages" in error for error in raw_adoption_result["errors"]), ( + raw_adoption_result["errors"] + ) + + write_native_telemetry_artifacts(skill_root, complete=True) + raw_recipe = json.loads((skill_root / "reports" / "telemetry_hook_recipes.json").read_text(encoding="utf-8")) + raw_recipe["recipes"][0]["message"] = "raw client message must not be accepted" + write_json(skill_root / "reports" / "telemetry_hook_recipes.json", raw_recipe) + raw_recipe_result = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert raw_recipe_result["status"] == "fail", raw_recipe_result + assert any("telemetry_hook_recipes.recipes[0].message" in error for error in raw_recipe_result["errors"]), ( + raw_recipe_result["errors"] + ) + + write_native_telemetry_artifacts(skill_root, complete=False) + invalid = validate_payload( + native_telemetry_submission(skill_root), + entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "native-client-telemetry.json", + root=skill_root, + template_expected=False, + ) + assert invalid["status"] == "fail", invalid + assert any("summary.source_types.external must be >0" in error for error in invalid["errors"]), invalid["errors"] + assert any("summary.adoption_sample_count must be >0" in error for error in invalid["errors"]), invalid["errors"] diff --git a/tests/yao_cli_helpers.py b/tests/yao_cli_helpers.py new file mode 100644 index 00000000..dae5daf1 --- /dev/null +++ b/tests/yao_cli_helpers.py @@ -0,0 +1,135 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +CLI = ROOT / "scripts" / "yao.py" +BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" + +sys.path.insert(0, str(ROOT / "scripts")) +import yao as yao_cli_module # noqa: E402 +import yao_cli_adaptation_commands # noqa: E402 +import yao_cli_config # noqa: E402 +import yao_cli_distribution_commands # noqa: E402 +import yao_cli_output_commands # noqa: E402 +import yao_cli_parser # noqa: E402 +import yao_cli_parser_evidence # noqa: E402 +import yao_cli_parser_operations # noqa: E402 +import yao_cli_report_commands # noqa: E402 +import yao_cli_runtime # noqa: E402 +from yao_cli_report_refresh import refresh_root_report_consistency_inputs # noqa: E402 + + +def run(*args: str, input_text: str | None = None) -> dict: + env = dict(os.environ) + env["YAO_CLI_TELEMETRY"] = "0" + env.pop("YAO_CLI_TELEMETRY_EVENTS", None) + proc = subprocess.run( + [sys.executable, str(CLI), *args], + cwd=ROOT, + capture_output=True, + text=True, + input=input_text, + env=env, + ) + payload = json.loads(proc.stdout) + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stderr": proc.stderr, + } + + +def run_with_env(extra_env: dict[str, str], *args: str) -> dict: + env = dict(os.environ) + env.update(extra_env) + proc = subprocess.run( + [sys.executable, str(CLI), *args], + cwd=ROOT, + capture_output=True, + text=True, + env=env, + ) + payload = json.loads(proc.stdout) + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stderr": proc.stderr, + } + + +def assert_cli_module_contracts() -> None: + assert yao_cli_config.resolve_target("root")["title"] == "Root Description Optimization" + assert yao_cli_config.resolve_promotion_target("root") == "yao-meta-skill" + assert yao_cli_config.infer_archetype("Standardize team review workflow.", "")[0] == "production" + assert yao_cli_config.infer_archetype("Govern release policy.", "")[0] == "governed" + assert "--entry" in yao_cli_config.baseline_compare_args() + assert "scripts/provider_output_eval_runner.py" in yao_cli_config.provider_output_runner_command("openai") + assert "--allow-custom-base-url" in yao_cli_config.provider_output_runner_command("openai", allow_custom_base_url=True) + deepseek_command = yao_cli_config.provider_output_runner_command( + "deepseek", + model="deepseek-v4-flash", + temperature=0.0, + ) + assert "--api-format" in deepseek_command and "chat-completions" in deepseek_command, deepseek_command + assert "--thinking" in deepseek_command and "disabled" in deepseek_command, deepseek_command + assert "--temperature" in deepseek_command and "0.0" in deepseek_command, deepseek_command + assert "DEEPSEEK_API_KEY" in deepseek_command, deepseek_command + assert "api.deepseek.com" not in deepseek_command, deepseek_command + for module in ( + yao_cli_parser, + yao_cli_parser_evidence, + yao_cli_parser_operations, + yao_cli_runtime, + yao_cli_adaptation_commands, + yao_cli_distribution_commands, + yao_cli_output_commands, + yao_cli_report_commands, + ): + assert module.SCRIPT_INTERFACE == "internal-module" + assert callable(yao_cli_module.command_review_studio) + + +def assert_help_surface() -> None: + parser_help = yao_cli_module.build_parser().format_help() + expected_help = ( + "quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage " + "world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard " + "benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry" + ).split() + assert all(item in parser_help for item in expected_help), parser_help + + +def assert_created_skill_reports(created: Path) -> None: + expected_reports = [ + "intent-dialogue.md", "intent-confidence.md", "skill-overview.html", "skill-interpretation.html", + "skill-interpretation.json", "review-studio.html", "review-studio.json", "review-viewer.html", + "reference-scan.md", "reference-synthesis.md", "output-risk-profile.md", "artifact-design-profile.md", + "prompt-quality-profile.md", "system-model.md", "skill-ir.json", "compiled_targets.md", + "compiled_targets.json", "iteration-directions.md", "adoption_drift_report.md", + "adoption_drift_report.json", "review_waivers.md", "review_waivers.json", "review_annotations.md", + "review_annotations.json", + ] + assert all((created / "reports" / path).exists() for path in expected_reports), created + + +def assert_creation_report_view(report_view: dict) -> None: + assert report_view["html_report"].endswith("reports/skill-overview.html"), report_view + assert report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), report_view + assert Path(report_view["html_report"]).exists(), report_view + assert Path(report_view["interpretation_report"]).exists(), report_view + assert report_view["review_studio"].endswith("reports/review-studio.html"), report_view + assert Path(report_view["review_studio"]).exists(), report_view + assert "Skill 已创建完成" in report_view["message"], report_view + assert "reports/skill-interpretation.html" in report_view["message"], report_view + assert "Review Studio 2.0" in report_view["message"], report_view + assert "目标编译" in report_view["message"], report_view + assert "reports/compiled_targets.md" in report_view["message"], report_view + assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in report_view["message"], report_view + assert "默认使用中文简体" in report_view["message"], report_view + assert "切换英文版" in report_view["message"], report_view diff --git a/tests/yao_cli_report_refresh.py b/tests/yao_cli_report_refresh.py new file mode 100644 index 00000000..f112f163 --- /dev/null +++ b/tests/yao_cli_report_refresh.py @@ -0,0 +1,16 @@ +from pathlib import Path +from typing import Callable + + +def refresh_root_report_consistency_inputs(run: Callable[..., dict], root: Path) -> None: + """Refresh root reports that evidence-consistency compares by value.""" + + for refresh_args in [ + ("benchmark-reproducibility", str(root), "--generated-at", "2026-06-15"), + ("skill-report", str(root)), + ("skill-interpretation", str(root)), + ("world-class-preflight", str(root), "--generated-at", "2026-06-15"), + ("review-studio", str(root)), + ]: + refresh_result = run(*refresh_args) + assert refresh_result["ok"], refresh_result