feat: guard world class completion claims
This commit is contained in:
yaojingang
@@ -2,7 +2,7 @@ PYTHON ?= python3
|
||||
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
|
||||
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
|
||||
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
|
||||
eval:
|
||||
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
@@ -100,6 +100,9 @@ world-class-ledger-check:
|
||||
world-class-intake-check:
|
||||
$(PYTHON) tests/verify_world_class_evidence_intake.py
|
||||
|
||||
world-class-claim-guard-check:
|
||||
$(PYTHON) tests/verify_world_class_claim_guard.py
|
||||
|
||||
benchmark-reproducibility-check:
|
||||
$(PYTHON) tests/verify_benchmark_reproducibility.py
|
||||
|
||||
@@ -217,11 +220,11 @@ sync-local-install: package-check
|
||||
sync-active-install: package-check
|
||||
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
|
||||
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-claim-guard-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
|
||||
ci-test:
|
||||
$(PYTHON) scripts/ci_test.py
|
||||
|
||||
clean:
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_skill_os2_coverage tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_world_class_evidence_intake tests/tmp_world_class_claim_guard tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
||||
|
||||
@@ -28,6 +28,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
- a world-class evidence plan that turns remaining provider, human, native-permission, and real-client telemetry gaps into executable evidence tasks
|
||||
- a world-class evidence ledger that records which external and human evidence is accepted or still pending without treating planned work as proof
|
||||
- a world-class evidence intake contract that validates external and human evidence packets for provenance, privacy, artifact refs, and anti-overclaim rules before ledger review
|
||||
- a world-class claim guard that scans public claim surfaces and blocks premature completed/true claims while the evidence ledger still has pending external or human evidence
|
||||
- a benchmark reproducibility manifest that checks methodology sections, required artifacts, failure disclosure, and reproduction commands
|
||||
- Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports
|
||||
- a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks
|
||||
@@ -139,6 +140,11 @@ python3 scripts/yao.py registry-audit .
|
||||
python3 scripts/yao.py package-verify . --package-dir dist --require-zip
|
||||
python3 scripts/yao.py install-simulate . --package-dir dist
|
||||
python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json
|
||||
python3 scripts/yao.py world-class-evidence .
|
||||
python3 scripts/yao.py world-class-ledger .
|
||||
python3 scripts/yao.py world-class-intake .
|
||||
python3 scripts/yao.py world-class-claim-guard .
|
||||
python3 scripts/yao.py benchmark-reproducibility .
|
||||
```
|
||||
|
||||
## Local Development Source
|
||||
@@ -287,6 +293,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
|
||||
- remaining world-class evidence tasks are published in [reports/world_class_evidence_plan.md](reports/world_class_evidence_plan.md)
|
||||
- current world-class evidence acceptance state is published in [reports/world_class_evidence_ledger.md](reports/world_class_evidence_ledger.md)
|
||||
- world-class evidence intake readiness is published in [reports/world_class_evidence_intake.md](reports/world_class_evidence_intake.md)
|
||||
- world-class public claim guard status is published in [reports/world_class_claim_guard.md](reports/world_class_claim_guard.md)
|
||||
- benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md)
|
||||
- target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md)
|
||||
- registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md)
|
||||
@@ -403,7 +410,7 @@ Utility scripts that make the meta-skill operational:
|
||||
- `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries
|
||||
- `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs
|
||||
- `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence
|
||||
- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint
|
||||
- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, skill-os2-coverage, world-class-evidence, world-class-ledger, world-class-intake, world-class-claim-guard, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint
|
||||
- `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report
|
||||
- `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot
|
||||
- `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger
|
||||
@@ -417,6 +424,7 @@ Utility scripts that make the meta-skill operational:
|
||||
- `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence
|
||||
- `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts
|
||||
- `render_world_class_evidence_intake.py`: validates world-class external and human evidence packets against provenance, privacy, artifact, and anti-overclaim requirements before ledger review
|
||||
- `render_world_class_claim_guard.py`: scans README, docs, and reports for premature world-class completion claims while accepted evidence is still pending
|
||||
- `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims
|
||||
- `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation
|
||||
- `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1"
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1",
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094"
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15",
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -48,7 +48,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094",
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
|
||||
@@ -3,13 +3,12 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"skill_dir": ".",
|
||||
"commit": "60995ed569b1c99aa6f6ae2b4406e04327202beb",
|
||||
"commit": "c1c85093d1ea67954d414789f96c87bbc3964868",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": true,
|
||||
"changed_file_count": 56,
|
||||
"changed_file_count": 42,
|
||||
"sample": [
|
||||
" M .gitignore",
|
||||
" M Makefile",
|
||||
" M README.md",
|
||||
" M registry/index.json",
|
||||
@@ -20,17 +19,18 @@
|
||||
" M reports/context_budget.json",
|
||||
" M reports/install_simulation.json",
|
||||
" M reports/output_execution_runs.json",
|
||||
" M reports/output_execution_runs.md"
|
||||
" M reports/output_execution_runs.md",
|
||||
" M reports/package_verification.json"
|
||||
]
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 18,
|
||||
"required_artifact_count": 19,
|
||||
"missing_artifact_count": 0,
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 17,
|
||||
"command_count": 18,
|
||||
"command_executed_count": 10,
|
||||
"timing_observed_count": 10,
|
||||
"model_executed_count": 0,
|
||||
@@ -42,7 +42,7 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 56
|
||||
"changed_file_count": 42
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -115,8 +115,8 @@
|
||||
"label": "output_execution",
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "8c40546b610c75ebb6dcee431cec4f603431c0c45ff49dcdc947473749c2f03e"
|
||||
"bytes": 7966,
|
||||
"sha256": "5a1eaa33c1cb916aa0962a9e826e97fc53fd9ca9bd6aa298ff91aa2068ff3415"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -150,36 +150,36 @@
|
||||
"label": "trust_report",
|
||||
"path": "reports/security_trust_report.json",
|
||||
"exists": true,
|
||||
"bytes": 85082,
|
||||
"sha256": "097f2a477efcc62b35ec8e94e2b800f7af6c207ad36fa096257972d0865a9dce"
|
||||
"bytes": 86196,
|
||||
"sha256": "1faacec874dd129b1d03cd53f93ee0abcb77fe6b5fbac7ab9726da23699ff4c1"
|
||||
},
|
||||
{
|
||||
"label": "registry_audit",
|
||||
"path": "reports/registry_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 3183,
|
||||
"sha256": "5c606b5a3b54bbdc814dc67a331d1c66756a9be25d16791bc53dd21ef19e07ec"
|
||||
"sha256": "c796a29122c32df656421f25eb2fa13726c7dffd85c676d9c9218673777cb18c"
|
||||
},
|
||||
{
|
||||
"label": "package_verification",
|
||||
"path": "reports/package_verification.json",
|
||||
"exists": true,
|
||||
"bytes": 19325,
|
||||
"sha256": "8dab6035e72ceda3756677d1ffa60aa7ad862c1af311899bb8a8dab198b750d7"
|
||||
"sha256": "224f5f71478530766baac79902cf19da4abb800e256273168fbd9e51d763ff4d"
|
||||
},
|
||||
{
|
||||
"label": "install_simulation",
|
||||
"path": "reports/install_simulation.json",
|
||||
"exists": true,
|
||||
"bytes": 8557,
|
||||
"sha256": "3b1456caa214bb0056943e110dbd0b2182faa5068b3a8fdb4272e9d8b38f759a"
|
||||
"sha256": "ae7cccad47924b9f4e2572ae32a9f68209dead4741c982d81446d8ea8474b822"
|
||||
},
|
||||
{
|
||||
"label": "skill_os2_audit",
|
||||
"path": "reports/skill_os2_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 13946,
|
||||
"sha256": "5b7c9427a3403d989651fad9cfe38e9dc2c859513fb6c3bd7a5574a7656883a2"
|
||||
"sha256": "3f1c6fdf7b52a365d9f458797a444b70aa34acffe78f24b4c08b4a428db5e32b"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_plan",
|
||||
@@ -201,6 +201,13 @@
|
||||
"exists": true,
|
||||
"bytes": 1775,
|
||||
"sha256": "124b9239d7f35022735090712aadf2bf1d4dfba28f6bcf18f956fa0a73fe4482"
|
||||
},
|
||||
{
|
||||
"label": "world_class_claim_guard",
|
||||
"path": "reports/world_class_claim_guard.json",
|
||||
"exists": true,
|
||||
"bytes": 7809,
|
||||
"sha256": "7365f6cd85b89326816590cd2403b1210e9afba45f0bc3ca1af6b4a07299b9ad"
|
||||
}
|
||||
],
|
||||
"missing_artifacts": [],
|
||||
@@ -285,6 +292,11 @@
|
||||
"command": "python3 scripts/yao.py world-class-intake .",
|
||||
"evidence": "reports/world_class_evidence_intake.json"
|
||||
},
|
||||
{
|
||||
"label": "world-class claim guard",
|
||||
"command": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
"evidence": "reports/world_class_claim_guard.json"
|
||||
},
|
||||
{
|
||||
"label": "full ci",
|
||||
"command": "make ci-test",
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Commit: `60995ed569b1c99aa6f6ae2b4406e04327202beb`
|
||||
Commit: `c1c85093d1ea67954d414789f96c87bbc3964868`
|
||||
Working tree dirty at generation: `true`
|
||||
|
||||
## Summary
|
||||
|
||||
- reproducibility ready: `true`
|
||||
- methodology complete: `true`
|
||||
- required artifacts: `18`
|
||||
- required artifacts: `19`
|
||||
- missing artifacts: `0`
|
||||
- output cases: `5`
|
||||
- disclosed failure cases: `3`
|
||||
- reproduction commands: `17`
|
||||
- reproduction commands: `18`
|
||||
- provider evidence complete: `false`
|
||||
- human review complete: `false`
|
||||
- world-class ready: `false`
|
||||
- changed files at generation: `56`
|
||||
- changed files at generation: `42`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
|
||||
|
||||
@@ -40,19 +40,20 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `8c40546b610c` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `5a1eaa33c1cb` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `ddd9af90d42e` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `097f2a477efc` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `5c606b5a3b54` |
|
||||
| package_verification | `reports/package_verification.json` | present | `8dab6035e72c` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `3b1456caa214` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `5b7c9427a340` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `1faacec874dd` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `c796a29122c3` |
|
||||
| package_verification | `reports/package_verification.json` | present | `224f5f714785` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `ae7cccad4792` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `3f1c6fdf7b52` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `ed1274b7c18c` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `5ed043491c3d` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `124b9239d7f3` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `7365f6cd85b8` |
|
||||
|
||||
## Reproduction Commands
|
||||
|
||||
@@ -88,6 +89,8 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
- evidence: `reports/world_class_evidence_ledger.json`
|
||||
- `python3 scripts/yao.py world-class-intake .`
|
||||
- evidence: `reports/world_class_evidence_intake.json`
|
||||
- `python3 scripts/yao.py world-class-claim-guard .`
|
||||
- evidence: `reports/world_class_claim_guard.json`
|
||||
- `make ci-test`
|
||||
- evidence: `CI target output`
|
||||
|
||||
|
||||
@@ -197,6 +197,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -223,8 +224,7 @@
|
||||
"scripts/verify_package.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py"
|
||||
"scripts/yao_cli_parser.py"
|
||||
],
|
||||
"assets": [
|
||||
"templates/basic_skill.md.j2",
|
||||
@@ -383,7 +383,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -429,6 +429,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -479,14 +480,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -508,7 +509,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -621,7 +622,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -667,6 +668,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -717,14 +719,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -744,7 +746,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -1030,6 +1032,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -1056,8 +1059,7 @@
|
||||
"scripts/verify_package.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py"
|
||||
"scripts/yao_cli_parser.py"
|
||||
],
|
||||
"assets": [
|
||||
"templates/basic_skill.md.j2",
|
||||
@@ -1216,7 +1218,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -1262,6 +1264,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -1312,14 +1315,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -1341,7 +1344,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -1454,7 +1457,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -1500,6 +1503,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -1550,14 +1554,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -1577,7 +1581,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -1863,6 +1867,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -1889,8 +1894,7 @@
|
||||
"scripts/verify_package.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py"
|
||||
"scripts/yao_cli_parser.py"
|
||||
],
|
||||
"assets": [
|
||||
"templates/basic_skill.md.j2",
|
||||
@@ -2049,7 +2053,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -2095,6 +2099,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -2145,14 +2150,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -2174,7 +2179,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -2280,7 +2285,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -2326,6 +2331,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -2376,14 +2382,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -2403,7 +2409,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -2680,6 +2686,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -2706,8 +2713,7 @@
|
||||
"scripts/verify_package.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py"
|
||||
"scripts/yao_cli_parser.py"
|
||||
],
|
||||
"assets": [
|
||||
"templates/basic_skill.md.j2",
|
||||
@@ -2866,7 +2872,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -2912,6 +2918,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -2962,14 +2969,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -2991,7 +2998,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -3097,7 +3104,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -3143,6 +3150,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -3193,14 +3201,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -3220,7 +3228,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -3497,6 +3505,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -3523,8 +3532,7 @@
|
||||
"scripts/verify_package.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py"
|
||||
"scripts/yao_cli_parser.py"
|
||||
],
|
||||
"assets": [
|
||||
"templates/basic_skill.md.j2",
|
||||
@@ -3683,7 +3691,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -3729,6 +3737,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -3779,14 +3788,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -3808,7 +3817,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -3918,7 +3927,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 57,
|
||||
"script_count": 58,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -3964,6 +3973,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -4014,14 +4024,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 70,
|
||||
"checked_count": 71,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"subprocess_script_count": 8,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -4041,7 +4051,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 57,
|
||||
"file_write": 58,
|
||||
"subprocess": 8,
|
||||
"interactive": 0
|
||||
},
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 751,
|
||||
"other_text_tokens": 1040839,
|
||||
"other_text_tokens": 1051179,
|
||||
"estimated_initial_load_tokens": 944,
|
||||
"estimated_total_text_tokens": 1041590,
|
||||
"relevant_file_count": 447,
|
||||
"estimated_total_text_tokens": 1051930,
|
||||
"relevant_file_count": 453,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 137.7
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"installed_skill_dir": "[temporary-install-root]/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 539,
|
||||
"archive_entry_count": 543,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.19,
|
||||
"duration_ms": 28.37,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.31,
|
||||
"duration_ms": 27.79,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.17,
|
||||
"duration_ms": 28.28,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.25,
|
||||
"duration_ms": 28.12,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.36,
|
||||
"duration_ms": 27.94,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.79,
|
||||
"duration_ms": 28.3,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.53,
|
||||
"duration_ms": 27.54,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.87,
|
||||
"duration_ms": 27.72,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.84,
|
||||
"duration_ms": 28.27,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.76,
|
||||
"duration_ms": 28.16,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 28.19 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 28.31 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 28.17 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.25 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.36 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.79 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.53 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.87 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.84 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.76 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 28.37 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.79 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 28.28 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.12 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 27.94 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.3 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.54 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.72 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.27 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.16 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094",
|
||||
"archive_entry_count": 539,
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404",
|
||||
"archive_entry_count": 543,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
- Package directory: `dist`
|
||||
- Targets: `4 / 4` adapters present
|
||||
- Archive present: `True`
|
||||
- Archive SHA256: `b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094`
|
||||
- Archive SHA256: `55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404`
|
||||
- Failures: `0`
|
||||
- Warnings: `0`
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1",
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094"
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15",
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -53,7 +53,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094",
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1"
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1`
|
||||
- Archive SHA256: `b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094`
|
||||
- Package SHA256: `81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15`
|
||||
- Archive SHA256: `55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404`
|
||||
- Install simulated: `True`
|
||||
|
||||
## Compatibility
|
||||
|
||||
@@ -292,12 +292,12 @@
|
||||
|
||||
<section>
|
||||
<h2>核心指标</h2>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Blueprint</span><strong>20/20</strong><p>2.0 coverage; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>80 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>539 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Blueprint</span><strong>20/20</strong><p>2.0 coverage; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>81 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>66 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>543 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>审查闸门</h2>
|
||||
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 80 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>世界证据</h3></div><p>4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true</p><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 81 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>世界证据</h3></div><p>4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true</p><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
</section>
|
||||
|
||||
<div class="twocol">
|
||||
@@ -307,7 +307,7 @@
|
||||
|
||||
<section id="actions">
|
||||
<h2>修复动作</h2>
|
||||
<div class="actions-grid"><article class='action-card warn'><div><span>关注</span><h3>输出实验</h3></div><p>补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。</p><small>没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。</small><dl><dt>修复位置</dt><dd>evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md</dd><dt>验证命令</dt><dd><code>python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review</code></dd></dl><ul class='source-ref-list'><li><a href='../evals/output/cases.jsonl'>evals/output/cases.jsonl:1</a><small>output eval cases · eval</small></li><li><a href='output_quality_scorecard.md'>reports/output_quality_scorecard.md:1</a><small>output scorecard · report</small></li><li><a href='output_execution_runs.md'>reports/output_execution_runs.md:1</a><small>output execution runs · report</small></li><li><a href='output_blind_review_pack.md'>reports/output_blind_review_pack.md:1</a><small>blind A/B review pack · report</small></li><li><a href='output_review_decisions.json'>reports/output_review_decisions.json:5</a><small>review decisions template · report</small></li><li><a href='output_review_adjudication.md'>reports/output_review_adjudication.md:1</a><small>review adjudication · report</small></li></ul><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>打开证据</a></footer></article><article class='action-card warn'><div><span>关注</span><h3>人工批准</h3></div><p>对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。</p><small>warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。</small><dl><dt>修复位置</dt><dd>reports/review_waivers.md</dd><dt>验证命令</dt><dd><code>python3 scripts/render_review_waivers.py .</code></dd></dl><ul class='source-ref-list'><li><a href='review_waivers.md'>reports/review_waivers.md:1</a><small>waiver ledger · report</small></li><li><a href='../references/review-waiver-method.md'>references/review-waiver-method.md:1</a><small>waiver method · method</small></li></ul><footer>reports/review_waivers.json <a href='review_waivers.md'>打开证据</a></footer></article><article class='action-card warn'><div><span>关注</span><h3>世界证据</h3></div><p>补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。</p><small>世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。</small><dl><dt>修复位置</dt><dd>reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md</dd><dt>验证命令</dt><dd><code>python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .</code></dd></dl><ul class='source-ref-list'><li><a href='world_class_evidence_ledger.md'>reports/world_class_evidence_ledger.md:1</a><small>world-class evidence ledger · report</small></li><li><a href='world_class_evidence_plan.md'>reports/world_class_evidence_plan.md:1</a><small>world-class evidence plan · report</small></li><li><a href='world_class_evidence_intake.md'>reports/world_class_evidence_intake.md:1</a><small>world-class evidence intake · report</small></li><li><a href='../evidence/world_class/intake.schema.json'>evidence/world_class/intake.schema.json:4</a><small>evidence intake schema · schema</small></li><li><a href='../evidence/world_class/templates/provider-holdout.intake.json'>evidence/world_class/templates/provider-holdout.intake.json:3</a><small>provider intake template · template</small></li><li><a href='../evidence/world_class/templates/human-adjudication.intake.json'>evidence/world_class/templates/human-adjudication.intake.json:3</a><small>human intake template · template</small></li><li><a href='skill_os2_audit.md'>reports/skill_os2_audit.md:1</a><small>Skill OS 2.0 audit · report</small></li><li><a href='output_review_decisions.json'>reports/output_review_decisions.json:6</a><small>human review decisions · report</small></li><li><a href='runtime_permission_probes.md'>reports/runtime_permission_probes.md:1</a><small>runtime permission probes · report</small></li><li><a href='adoption_drift_report.md'>reports/adoption_drift_report.md:1</a><small>adoption drift · report</small></li></ul><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>打开证据</a></footer></article></div>
|
||||
<div class="actions-grid"><article class='action-card warn'><div><span>关注</span><h3>输出实验</h3></div><p>补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。</p><small>没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。</small><dl><dt>修复位置</dt><dd>evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md</dd><dt>验证命令</dt><dd><code>python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review</code></dd></dl><ul class='source-ref-list'><li><a href='../evals/output/cases.jsonl'>evals/output/cases.jsonl:1</a><small>output eval cases · eval</small></li><li><a href='output_quality_scorecard.md'>reports/output_quality_scorecard.md:1</a><small>output scorecard · report</small></li><li><a href='output_execution_runs.md'>reports/output_execution_runs.md:1</a><small>output execution runs · report</small></li><li><a href='output_blind_review_pack.md'>reports/output_blind_review_pack.md:1</a><small>blind A/B review pack · report</small></li><li><a href='output_review_decisions.json'>reports/output_review_decisions.json:5</a><small>review decisions template · report</small></li><li><a href='output_review_adjudication.md'>reports/output_review_adjudication.md:1</a><small>review adjudication · report</small></li></ul><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>打开证据</a></footer></article><article class='action-card warn'><div><span>关注</span><h3>人工批准</h3></div><p>对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。</p><small>warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。</small><dl><dt>修复位置</dt><dd>reports/review_waivers.md</dd><dt>验证命令</dt><dd><code>python3 scripts/render_review_waivers.py .</code></dd></dl><ul class='source-ref-list'><li><a href='review_waivers.md'>reports/review_waivers.md:1</a><small>waiver ledger · report</small></li><li><a href='../references/review-waiver-method.md'>references/review-waiver-method.md:1</a><small>waiver method · method</small></li></ul><footer>reports/review_waivers.json <a href='review_waivers.md'>打开证据</a></footer></article><article class='action-card warn'><div><span>关注</span><h3>世界证据</h3></div><p>补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。</p><small>世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。</small><dl><dt>修复位置</dt><dd>reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md</dd><dt>验证命令</dt><dd><code>python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .</code></dd></dl><ul class='source-ref-list'><li><a href='world_class_evidence_ledger.md'>reports/world_class_evidence_ledger.md:1</a><small>world-class evidence ledger · report</small></li><li><a href='world_class_evidence_plan.md'>reports/world_class_evidence_plan.md:1</a><small>world-class evidence plan · report</small></li><li><a href='world_class_evidence_intake.md'>reports/world_class_evidence_intake.md:1</a><small>world-class evidence intake · report</small></li><li><a href='world_class_claim_guard.md'>reports/world_class_claim_guard.md:1</a><small>world-class claim guard · report</small></li><li><a href='../evidence/world_class/intake.schema.json'>evidence/world_class/intake.schema.json:4</a><small>evidence intake schema · schema</small></li><li><a href='../evidence/world_class/templates/provider-holdout.intake.json'>evidence/world_class/templates/provider-holdout.intake.json:3</a><small>provider intake template · template</small></li><li><a href='../evidence/world_class/templates/human-adjudication.intake.json'>evidence/world_class/templates/human-adjudication.intake.json:3</a><small>human intake template · template</small></li><li><a href='skill_os2_audit.md'>reports/skill_os2_audit.md:1</a><small>Skill OS 2.0 audit · report</small></li><li><a href='output_review_decisions.json'>reports/output_review_decisions.json:6</a><small>human review decisions · report</small></li><li><a href='runtime_permission_probes.md'>reports/runtime_permission_probes.md:1</a><small>runtime permission probes · report</small></li><li><a href='adoption_drift_report.md'>reports/adoption_drift_report.md:1</a><small>adoption drift · report</small></li></ul><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>打开证据</a></footer></article></div>
|
||||
</section>
|
||||
|
||||
<section id="annotations">
|
||||
@@ -323,7 +323,7 @@
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h2>证据路径</h2>
|
||||
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>output_execution</strong><span>reports/output_execution_runs.md</span></li><li><strong>output_blind_review</strong><span>reports/output_blind_review_pack.md</span></li><li><strong>output_review_decisions</strong><span>reports/output_review_decisions.json</span></li><li><strong>output_review_adjudication</strong><span>reports/output_review_adjudication.md</span></li><li><strong>benchmark_reproducibility</strong><span>reports/benchmark_reproducibility.md</span></li><li><strong>skill_os2_coverage</strong><span>reports/skill_os2_coverage.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>permission_policy</strong><span>security/permission_policy.md</span></li><li><strong>runtime_permissions</strong><span>reports/runtime_permission_probes.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>compiled_targets</strong><span>reports/compiled_targets.md</span></li><li><strong>adoption_drift</strong><span>reports/adoption_drift_report.md</span></li><li><strong>review_waivers</strong><span>reports/review_waivers.md</span></li><li><strong>review_annotations</strong><span>reports/review_annotations.md</span></li><li><strong>world_class_evidence_plan</strong><span>reports/world_class_evidence_plan.md</span></li><li><strong>world_class_evidence_ledger</strong><span>reports/world_class_evidence_ledger.md</span></li><li><strong>world_class_evidence_intake</strong><span>reports/world_class_evidence_intake.md</span></li><li><strong>registry_audit</strong><span>reports/registry_audit.md</span></li><li><strong>package_verification</strong><span>reports/package_verification.md</span></li><li><strong>install_simulation</strong><span>reports/install_simulation.md</span></li><li><strong>upgrade_check</strong><span>reports/upgrade_check.md</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
|
||||
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>output_execution</strong><span>reports/output_execution_runs.md</span></li><li><strong>output_blind_review</strong><span>reports/output_blind_review_pack.md</span></li><li><strong>output_review_decisions</strong><span>reports/output_review_decisions.json</span></li><li><strong>output_review_adjudication</strong><span>reports/output_review_adjudication.md</span></li><li><strong>benchmark_reproducibility</strong><span>reports/benchmark_reproducibility.md</span></li><li><strong>skill_os2_coverage</strong><span>reports/skill_os2_coverage.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>permission_policy</strong><span>security/permission_policy.md</span></li><li><strong>runtime_permissions</strong><span>reports/runtime_permission_probes.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>compiled_targets</strong><span>reports/compiled_targets.md</span></li><li><strong>adoption_drift</strong><span>reports/adoption_drift_report.md</span></li><li><strong>review_waivers</strong><span>reports/review_waivers.md</span></li><li><strong>review_annotations</strong><span>reports/review_annotations.md</span></li><li><strong>world_class_evidence_plan</strong><span>reports/world_class_evidence_plan.md</span></li><li><strong>world_class_evidence_ledger</strong><span>reports/world_class_evidence_ledger.md</span></li><li><strong>world_class_evidence_intake</strong><span>reports/world_class_evidence_intake.md</span></li><li><strong>world_class_claim_guard</strong><span>reports/world_class_claim_guard.md</span></li><li><strong>registry_audit</strong><span>reports/registry_audit.md</span></li><li><strong>package_verification</strong><span>reports/package_verification.md</span></li><li><strong>install_simulation</strong><span>reports/install_simulation.md</span></li><li><strong>upgrade_check</strong><span>reports/upgrade_check.md</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
@@ -363,7 +363,7 @@
|
||||
</section>
|
||||
|
||||
<section id="trust" class="twocol">
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>80</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>81</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
|
||||
</section>
|
||||
|
||||
@@ -413,14 +413,19 @@
|
||||
<div class="panel"><h2>入口边界</h2><p>intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。</p></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>声明守卫</h2><dl class='kv-grid'><div><dt>台账可声明</dt><dd>否</dd></div><div><dt>台账待补</dt><dd>4</dd></div><div><dt>声明面</dt><dd>66</dd></div><div><dt>违规数</dt><dd>0</dd></div><div><dt>Overclaim Guard Active</dt><dd>是</dd></div></dl></div>
|
||||
<div class="panel"><h2>声明边界</h2><p>claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="registry" class="twocol">
|
||||
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section id="release" class="twocol">
|
||||
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>539</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>543</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
|
||||
+471
-110
File diff suppressed because it is too large
Load Diff
@@ -2,8 +2,8 @@
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"scanned_files": 164,
|
||||
"script_count": 80,
|
||||
"scanned_files": 165,
|
||||
"script_count": 81,
|
||||
"internal_module_count": 10,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
@@ -12,18 +12,18 @@
|
||||
"network_script_count": 3,
|
||||
"network_policy_covered_count": 3,
|
||||
"network_policy_missing_count": 0,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"permission_required_count": 3,
|
||||
"permission_approved_count": 3,
|
||||
"permission_missing_count": 0,
|
||||
"permission_invalid_count": 0,
|
||||
"permission_expired_count": 0,
|
||||
"help_smoke_checked_count": 70,
|
||||
"help_smoke_checked_count": 71,
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 164,
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1"
|
||||
"package_hash_file_count": 165,
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
@@ -774,6 +774,20 @@
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_world_class_claim_guard.py",
|
||||
"interface": "cli",
|
||||
"interface_declared": true,
|
||||
"interface_reason": "Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists.",
|
||||
"has_argparse": true,
|
||||
"has_main_guard": true,
|
||||
"uses_input": false,
|
||||
"uses_network": false,
|
||||
"uses_file_write": true,
|
||||
"uses_subprocess": false,
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_world_class_evidence_intake.py",
|
||||
"interface": "cli",
|
||||
@@ -1197,9 +1211,9 @@
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"timeout_seconds": 5.0,
|
||||
"candidate_count": 70,
|
||||
"checked_count": 70,
|
||||
"passed_count": 70,
|
||||
"candidate_count": 71,
|
||||
"checked_count": 71,
|
||||
"passed_count": 71,
|
||||
"failed_count": 0,
|
||||
"skipped_count": 10,
|
||||
"failed_scripts": [],
|
||||
@@ -1724,6 +1738,16 @@
|
||||
"stdout_excerpt": "usage: render_telemetry_hook_recipes.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--output-jsonl OUTPUT_JSONL]\n ",
|
||||
"stderr_excerpt": ""
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_world_class_claim_guard.py",
|
||||
"command": "python3 scripts/render_world_class_claim_guard.py --help",
|
||||
"returncode": 0,
|
||||
"timed_out": false,
|
||||
"passed": true,
|
||||
"has_help_text": true,
|
||||
"stdout_excerpt": "usage: render_world_class_claim_guard.py [-h] [--claim-surface CLAIM_SURFACE]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ",
|
||||
"stderr_excerpt": ""
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_world_class_evidence_intake.py",
|
||||
"command": "python3 scripts/render_world_class_evidence_intake.py --help",
|
||||
@@ -2044,6 +2068,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
# Security Trust Report
|
||||
|
||||
- OK: `True`
|
||||
- Scanned files: `164`
|
||||
- Scripts: `80`
|
||||
- Scanned files: `165`
|
||||
- Scripts: `81`
|
||||
- Internal script modules: `10`
|
||||
- Secret findings: `0`
|
||||
- Network-capable scripts: `3`
|
||||
- Network policy covered scripts: `3`
|
||||
- Network policy missing scripts: `0`
|
||||
- File-write scripts: `57`
|
||||
- File-write scripts: `58`
|
||||
- Permission approvals: `3 / 3`
|
||||
- Permission approval gaps: `0`
|
||||
- CLI help smoke checked: `70`
|
||||
- CLI help smoke checked: `71`
|
||||
- CLI help smoke failures: `0`
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `164`
|
||||
- Package SHA256: `9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1`
|
||||
- Package hash files: `165`
|
||||
- Package SHA256: `81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15`
|
||||
|
||||
## Failures
|
||||
|
||||
@@ -54,8 +54,8 @@
|
||||
|
||||
- Enabled: `True`
|
||||
- Timeout seconds: `5.0`
|
||||
- Checked scripts: `70`
|
||||
- Passed scripts: `70`
|
||||
- Checked scripts: `71`
|
||||
- Passed scripts: `71`
|
||||
- Failed scripts: `none`
|
||||
|
||||
## Script Surface
|
||||
@@ -114,6 +114,7 @@
|
||||
| scripts/render_social_preview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/render_system_model.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/render_telemetry_hook_recipes.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/render_world_class_claim_guard.py | cli | True | True | True | False | False | True | False | Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists. |
|
||||
| scripts/render_world_class_evidence_intake.py | cli | True | True | True | False | False | True | False | Validates world-class human and external evidence intake packets before ledger review. |
|
||||
| scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. |
|
||||
| scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
|
||||
@@ -771,7 +771,7 @@
|
||||
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-0.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-1.7" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-2.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="27.6 72.4" stroke-dashoffset="-3.4" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="69.0 31.0" stroke-dashoffset="-31.0" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">333项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-0.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-1.7" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-2.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="27.4 72.6" stroke-dashoffset="-3.4" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="69.2 30.8" stroke-dashoffset="-30.8" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">336项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
|
||||
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
|
||||
|
||||
+25
-25
@@ -436,7 +436,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 80
|
||||
"file_count": 81
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -448,10 +448,10 @@
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 189
|
||||
"file_count": 191
|
||||
}
|
||||
],
|
||||
"file_count": 333,
|
||||
"file_count": 336,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -476,7 +476,7 @@
|
||||
},
|
||||
{
|
||||
"label": "scripts",
|
||||
"value": 80
|
||||
"value": 81
|
||||
},
|
||||
{
|
||||
"label": "evals",
|
||||
@@ -484,7 +484,7 @@
|
||||
},
|
||||
{
|
||||
"label": "reports",
|
||||
"value": 189
|
||||
"value": 191
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -605,7 +605,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 80
|
||||
"file_count": 81
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -617,7 +617,7 @@
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 189
|
||||
"file_count": 191
|
||||
}
|
||||
],
|
||||
"strengths": [
|
||||
@@ -889,11 +889,11 @@
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 18,
|
||||
"required_artifact_count": 19,
|
||||
"missing_artifact_count": 0,
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 17,
|
||||
"command_count": 18,
|
||||
"command_executed_count": 10,
|
||||
"timing_observed_count": 10,
|
||||
"model_executed_count": 0,
|
||||
@@ -905,9 +905,9 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 56
|
||||
"changed_file_count": 42
|
||||
},
|
||||
"commit": "60995ed569b1c99aa6f6ae2b4406e04327202beb",
|
||||
"commit": "c1c85093d1ea67954d414789f96c87bbc3964868",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
@@ -982,8 +982,8 @@
|
||||
"failures": []
|
||||
},
|
||||
"trust_security": {
|
||||
"scanned_files": 164,
|
||||
"script_count": 80,
|
||||
"scanned_files": 165,
|
||||
"script_count": 81,
|
||||
"internal_module_count": 10,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
@@ -992,18 +992,18 @@
|
||||
"network_script_count": 3,
|
||||
"network_policy_covered_count": 3,
|
||||
"network_policy_missing_count": 0,
|
||||
"file_write_script_count": 57,
|
||||
"file_write_script_count": 58,
|
||||
"permission_required_count": 3,
|
||||
"permission_approved_count": 3,
|
||||
"permission_missing_count": 0,
|
||||
"permission_invalid_count": 0,
|
||||
"permission_expired_count": 0,
|
||||
"help_smoke_checked_count": 70,
|
||||
"help_smoke_checked_count": 71,
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 164,
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1"
|
||||
"package_hash_file_count": 165,
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1041,8 +1041,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1",
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094"
|
||||
"package_sha256": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15",
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1073,7 +1073,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094",
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1089,8 +1089,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094",
|
||||
"archive_entry_count": 539,
|
||||
"archive_sha256": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404",
|
||||
"archive_entry_count": 543,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -1101,7 +1101,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 539,
|
||||
"archive_entry_count": 543,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -1168,12 +1168,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094"
|
||||
"to": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1"
|
||||
"to": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -116,6 +116,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
|
||||
@@ -145,7 +145,7 @@
|
||||
"key": "benchmark-reproducibility",
|
||||
"label": "Benchmark Reproducibility",
|
||||
"status": "pass",
|
||||
"current": "artifacts 18; missing 0; failures 3",
|
||||
"current": "artifacts 19; missing 0; failures 3",
|
||||
"target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable",
|
||||
"evidence": [
|
||||
{
|
||||
@@ -197,7 +197,7 @@
|
||||
"key": "trust-security",
|
||||
"label": "Trust Security",
|
||||
"status": "pass",
|
||||
"current": "secrets 0; scripts 80; help failures 0",
|
||||
"current": "secrets 0; scripts 81; help failures 0",
|
||||
"target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable",
|
||||
"evidence": [
|
||||
{
|
||||
@@ -281,7 +281,7 @@
|
||||
"key": "registry-distribution",
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"current": "zip entries 539; install failures 0; permission failures 0",
|
||||
"current": "zip entries 543; install failures 0; permission failures 0",
|
||||
"target": "Package metadata, archive checksum, package verification, and install simulation pass",
|
||||
"evidence": [
|
||||
{
|
||||
|
||||
@@ -21,13 +21,13 @@ Generated at: `2026-06-14`
|
||||
| Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. |
|
||||
| Provider Holdout | external-required | model-executed 0; token-observed 0 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
|
||||
| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. |
|
||||
| Benchmark Reproducibility | pass | artifacts 18; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
|
||||
| Benchmark Reproducibility | pass | artifacts 19; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
|
||||
| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. |
|
||||
| Trust Security | pass | secrets 0; scripts 80; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
|
||||
| Trust Security | pass | secrets 0; scripts 81; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
|
||||
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4 | Packaged adapters expose explicit permission metadata and residual risks | Preserve residual-risk notes until real native enforcement exists. |
|
||||
| Native Permission Enforcement | external-required | native-enforced targets 0 | At least one target/client enforces approved permissions at runtime | Integrate a real client or installer runtime guard before claiming native permission enforcement. |
|
||||
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
|
||||
| Registry Distribution | pass | zip entries 539; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Registry Distribution | pass | zip entries 543; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Review Studio | pass | decision review; warnings 3; score 90 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
|
||||
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
|
||||
| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
|
||||
|
||||
@@ -122,7 +122,7 @@
|
||||
"label": "Trust Security",
|
||||
"status": "pass",
|
||||
"objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
|
||||
"current": "80 scripts; secrets 0; help failures 0",
|
||||
"current": "81 scripts; secrets 0; help failures 0",
|
||||
"command": "python3 scripts/yao.py trust .",
|
||||
"test": "python3 tests/verify_trust_check.py",
|
||||
"evidence": [
|
||||
@@ -196,7 +196,7 @@
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
|
||||
"current": "archive entries 539; install failures 0",
|
||||
"current": "archive entries 543; install failures 0",
|
||||
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
|
||||
"test": "python3 tests/verify_registry_audit.py",
|
||||
"evidence": [
|
||||
@@ -305,7 +305,7 @@
|
||||
"label": "Benchmark Methodology",
|
||||
"status": "pass",
|
||||
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
|
||||
"current": "18 required artifacts checked",
|
||||
"current": "19 required artifacts checked",
|
||||
"command": "make ci-test",
|
||||
"test": "tests/verify_benchmark_reproducibility.py",
|
||||
"evidence": [
|
||||
|
||||
@@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
|
||||
| Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` |
|
||||
| Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` |
|
||||
| Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` |
|
||||
| Trust Security | `pass` | 80 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
|
||||
| Trust Security | `pass` | 81 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
|
||||
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
|
||||
| Registry Distribution | `pass` | archive entries 539; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
|
||||
| Registry Distribution | `pass` | archive entries 543; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
|
||||
| Review Studio | `pass` | 14 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
|
||||
| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` |
|
||||
|
||||
@@ -31,7 +31,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
|
||||
|
||||
| Item | Status | Current | Command | Test |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Benchmark Methodology | `pass` | 18 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` |
|
||||
| Benchmark Methodology | `pass` | 19 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` |
|
||||
| Output Eval Schema | `pass` | 5 output cases | `make ci-test` | `tests/verify_output_eval_lab.py` |
|
||||
| Output Eval Runner | `pass` | delta 100.0 | `make ci-test` | `tests/verify_output_eval_lab.py` |
|
||||
| Output Quality Scorecard | `pass` | gate pass True | `make ci-test` | `tests/verify_output_eval_lab.py` |
|
||||
|
||||
@@ -70,12 +70,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "b68dd2aaecfa1f585fb5b380d89a04adde672e7996b7750b3ecceb8ba2e3d094"
|
||||
"to": "55066438e0934145506d04883e617e340cf09cea824e8764a0d65f54bab6c404"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "9bbb47aaed8b2b003c68176ab3a8a4c3926493ded7c80be004881254cc113ee1"
|
||||
"to": "81b0f6007b71b949e1fba7079ffd8156fe906c6784eced95fd1f9a30f7f5bc15"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -0,0 +1,316 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": false,
|
||||
"ledger_pending_count": 4,
|
||||
"claim_surface_count": 66,
|
||||
"violation_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"decision": "claim-guard-pass-evidence-pending"
|
||||
},
|
||||
"rules": [
|
||||
{
|
||||
"key": "ready-to-claim-true",
|
||||
"reason": "ready_to_claim_world_class cannot be true while ledger evidence is pending.",
|
||||
"pattern": "ready[\\s_-]*to[\\s_-]*claim[\\s_-]*world[\\s_-]*class\\s*[:=]\\s*`?true`?"
|
||||
},
|
||||
{
|
||||
"key": "world-class-ready-true",
|
||||
"reason": "world-class readiness cannot be claimed before accepted external and human evidence exists.",
|
||||
"pattern": "(public\\s+)?world[\\s_-]*class[\\s_-]*ready\\s*[:=]\\s*`?true`?"
|
||||
},
|
||||
{
|
||||
"key": "json-world-class-ready-true",
|
||||
"reason": "machine-readable claim fields must remain false until the ledger is ready.",
|
||||
"pattern": "\"(?:ready_to_claim_world_class|world_class_ready|public_world_class_ready)\"\\s*:\\s*true"
|
||||
},
|
||||
{
|
||||
"key": "completion-phrase",
|
||||
"reason": "completion language is blocked until the world-class ledger is accepted.",
|
||||
"pattern": "(public\\s+)?world[\\s_-]*class\\s+(?:complete|completed|achieved|certified)"
|
||||
},
|
||||
{
|
||||
"key": "zh-completion-phrase",
|
||||
"reason": "中文完成态表述必须等到 ledger ready 后才能出现。",
|
||||
"pattern": "世界级(?:已完成|完成|已达成|达成|准备就绪|可声明|认证通过)"
|
||||
}
|
||||
],
|
||||
"scanned_surfaces": [
|
||||
{
|
||||
"path": "README.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "SKILL.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "docs/README.fr-FR.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "docs/README.ja-JP.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "docs/README.ru-RU.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "docs/README.zh-CN.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "docs/migration-v2.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/adoption_drift_report.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/artifact-design-profile.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/baseline-compare.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/benchmark_reproducibility.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/candidate_registry.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/compiled_targets.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/conformance_matrix.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/context_budget.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/description_drift_history.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/description_optimization.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/description_optimization_suite.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/family_summary.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/feishu-ranking-pattern-analysis-2026-04-01.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/github-benchmark-scan.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/install_simulation.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/intent-confidence.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/intent-dialogue.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration-directions.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_bundles/governed-incident-command/bundle.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_bundles/governed-incident-command/review.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_bundles/team-frontend-review/bundle.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_bundles/team-frontend-review/review.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_bundles/yao-meta-skill/bundle.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_bundles/yao-meta-skill/review.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/iteration_ledger.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output-risk-profile.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_blind_review_pack.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_execution_runs.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_quality_scorecard.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_adjudication.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/package_verification.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/portability_score.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/promotion_decisions.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/prompt-quality-profile.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/reference-scan.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/reference-synthesis.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/registry_audit.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/regression_history.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/review-studio.html",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/review-viewer.html",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/review_annotations.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/review_waivers.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/route_scorecard.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/runtime_permission_probes.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/security_trust_report.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/skill-os-2-review.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/skill-overview-v2-plan.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/skill-overview.html",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/skill_atlas.html",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/skill_os2_audit.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/skill_os2_coverage.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/system-model.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/telemetry_hook_recipes.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/upgrade_check.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/world_class_evidence_intake.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/world_class_evidence_ledger.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/world_class_evidence_plan.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/yao-deploy-bridge-skill-research-plan.md",
|
||||
"violation_count": 0
|
||||
}
|
||||
],
|
||||
"violations": [],
|
||||
"source_ledger": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
"markdown": "reports/world_class_evidence_ledger.md"
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_claim_guard.json",
|
||||
"markdown": "reports/world_class_claim_guard.md"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
# World-Class Claim Guard
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
|
||||
## Summary
|
||||
|
||||
- decision: `claim-guard-pass-evidence-pending`
|
||||
- ledger ready to claim world-class: `false`
|
||||
- ledger pending evidence: `4`
|
||||
- claim surfaces scanned: `66`
|
||||
- violations: `0`
|
||||
- overclaim guard active: `true`
|
||||
|
||||
This guard scans public claim surfaces for completion language that would contradict the world-class evidence ledger. It allows evidence planning and pending-state language, but blocks completion claims until the ledger is ready.
|
||||
|
||||
## Violations
|
||||
|
||||
| Path | Line | Rule | Excerpt |
|
||||
| --- | ---: | --- | --- |
|
||||
| `none` | 0 | `none` | none |
|
||||
|
||||
## Rules
|
||||
|
||||
| Rule | Reason |
|
||||
| --- | --- |
|
||||
| `ready-to-claim-true` | ready_to_claim_world_class cannot be true while ledger evidence is pending. |
|
||||
| `world-class-ready-true` | world-class readiness cannot be claimed before accepted external and human evidence exists. |
|
||||
| `json-world-class-ready-true` | machine-readable claim fields must remain false until the ledger is ready. |
|
||||
| `completion-phrase` | completion language is blocked until the world-class ledger is accepted. |
|
||||
| `zh-completion-phrase` | 中文完成态表述必须等到 ledger ready 后才能出现。 |
|
||||
@@ -53,6 +53,7 @@ DEFAULT_TARGETS = [
|
||||
"world-class-evidence-check",
|
||||
"world-class-ledger-check",
|
||||
"world-class-intake-check",
|
||||
"world-class-claim-guard-check",
|
||||
"benchmark-reproducibility-check",
|
||||
"feedback-check",
|
||||
"adoption-drift-check",
|
||||
|
||||
@@ -40,6 +40,7 @@ REQUIRED_ARTIFACTS = [
|
||||
("world_class_evidence_plan", "reports/world_class_evidence_plan.json"),
|
||||
("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"),
|
||||
("world_class_evidence_intake", "reports/world_class_evidence_intake.json"),
|
||||
("world_class_claim_guard", "reports/world_class_claim_guard.json"),
|
||||
]
|
||||
|
||||
REPRODUCTION_COMMANDS = [
|
||||
@@ -123,6 +124,11 @@ REPRODUCTION_COMMANDS = [
|
||||
"command": "python3 scripts/yao.py world-class-intake .",
|
||||
"evidence": "reports/world_class_evidence_intake.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class claim guard",
|
||||
"command": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
"evidence": "reports/world_class_claim_guard.json",
|
||||
},
|
||||
{
|
||||
"label": "full ci",
|
||||
"command": "make ci-test",
|
||||
|
||||
@@ -138,6 +138,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]:
|
||||
"world_class_evidence_plan": "reports/world_class_evidence_plan.md",
|
||||
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.md",
|
||||
"world_class_evidence_intake": "reports/world_class_evidence_intake.md",
|
||||
"world_class_claim_guard": "reports/world_class_claim_guard.md",
|
||||
"registry_audit": "reports/registry_audit.md",
|
||||
"package_verification": "reports/package_verification.md",
|
||||
"install_simulation": "reports/install_simulation.md",
|
||||
@@ -172,6 +173,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
"review_annotations": load_json(reports / "review_annotations.json"),
|
||||
"world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"),
|
||||
"world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"),
|
||||
"world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"),
|
||||
"registry": load_json(reports / "registry_audit.json"),
|
||||
"package_verification": load_json(reports / "package_verification.json"),
|
||||
"install_simulation": load_json(reports / "install_simulation.json"),
|
||||
@@ -198,6 +200,7 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
waivers = data["review_waivers"].get("summary", {})
|
||||
annotations = data["review_annotations"].get("summary", {})
|
||||
intake = data["world_class_evidence_intake"].get("summary", {})
|
||||
claim_guard = data["world_class_claim_guard"].get("summary", {})
|
||||
registry = data["registry"].get("package", {})
|
||||
package_verification = data["package_verification"].get("summary", {})
|
||||
install_simulation = data["install_simulation"].get("summary", {})
|
||||
@@ -280,6 +283,11 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
f"{intake.get('invalid_submission_count', 0)} invalid"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Claim Guard",
|
||||
"value": str(claim_guard.get("violation_count", 0)),
|
||||
"detail": f"{claim_guard.get('claim_surface_count', 0)} public surfaces scanned",
|
||||
},
|
||||
{
|
||||
"label": "Notes",
|
||||
"value": f"{annotations.get('open_count', 0)}/{annotations.get('annotation_count', 0)}",
|
||||
@@ -485,6 +493,7 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = {
|
||||
{"path": "reports/world_class_evidence_ledger.md", "label": "world-class evidence ledger", "kind": "report", "patterns": ["# World-Class Evidence Ledger"]},
|
||||
{"path": "reports/world_class_evidence_plan.md", "label": "world-class evidence plan", "kind": "report", "patterns": ["# World-Class Evidence Plan"]},
|
||||
{"path": "reports/world_class_evidence_intake.md", "label": "world-class evidence intake", "kind": "report", "patterns": ["# World-Class Evidence Intake"]},
|
||||
{"path": "reports/world_class_claim_guard.md", "label": "world-class claim guard", "kind": "report", "patterns": ["# World-Class Claim Guard"]},
|
||||
{"path": "evidence/world_class/intake.schema.json", "label": "evidence intake schema", "kind": "schema", "patterns": ["Yao World-Class Evidence Intake"]},
|
||||
{"path": "evidence/world_class/templates/provider-holdout.intake.json", "label": "provider intake template", "kind": "template", "patterns": ["provider-holdout"]},
|
||||
{"path": "evidence/world_class/templates/human-adjudication.intake.json", "label": "human intake template", "kind": "template", "patterns": ["human-adjudication"]},
|
||||
@@ -710,6 +719,7 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
world_class_ledger = report["data"].get("world_class_evidence_ledger", {})
|
||||
world_class_summary = world_class_ledger.get("summary", {})
|
||||
world_class_intake_summary = report["data"].get("world_class_evidence_intake", {}).get("summary", {})
|
||||
world_class_claim_summary = report["data"].get("world_class_claim_guard", {}).get("summary", {})
|
||||
world_class_entries_html = render_world_class_evidence_entries(world_class_ledger)
|
||||
annotation_summary = report["data"]["review_annotations"].get("summary", {})
|
||||
annotation_caption = (
|
||||
@@ -844,6 +854,17 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
],
|
||||
"world-class evidence intake missing",
|
||||
)
|
||||
world_class_claim_panel = render_kv_grid(
|
||||
world_class_claim_summary,
|
||||
[
|
||||
"ledger_ready_to_claim_world_class",
|
||||
"ledger_pending_count",
|
||||
"claim_surface_count",
|
||||
"violation_count",
|
||||
"overclaim_guard_active",
|
||||
],
|
||||
"world-class claim guard missing",
|
||||
)
|
||||
registry_panel = render_kv_grid(
|
||||
registry_package_summary(registry_package),
|
||||
[
|
||||
@@ -1032,6 +1053,11 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
<div class="panel"><h2>入口边界</h2><p>intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。</p></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>声明守卫</h2>{world_class_claim_panel}</div>
|
||||
<div class="panel"><h2>声明边界</h2><p>claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="registry" class="twocol">
|
||||
<div class="panel"><h2>注册审计</h2><p>{html.escape(gate_details.get('registry-audit', 'registry audit missing'))}</p></div>
|
||||
<div class="panel"><h2>包体元数据</h2>{registry_panel}</div>
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists."
|
||||
|
||||
FORBIDDEN_WHEN_PENDING = [
|
||||
{
|
||||
"key": "ready-to-claim-true",
|
||||
"pattern": re.compile(r"ready[\s_-]*to[\s_-]*claim[\s_-]*world[\s_-]*class\s*[:=]\s*`?true`?", re.IGNORECASE),
|
||||
"reason": "ready_to_claim_world_class cannot be true while ledger evidence is pending.",
|
||||
},
|
||||
{
|
||||
"key": "world-class-ready-true",
|
||||
"pattern": re.compile(r"(public\s+)?world[\s_-]*class[\s_-]*ready\s*[:=]\s*`?true`?", re.IGNORECASE),
|
||||
"reason": "world-class readiness cannot be claimed before accepted external and human evidence exists.",
|
||||
},
|
||||
{
|
||||
"key": "json-world-class-ready-true",
|
||||
"pattern": re.compile(r'"(?:ready_to_claim_world_class|world_class_ready|public_world_class_ready)"\s*:\s*true', re.IGNORECASE),
|
||||
"reason": "machine-readable claim fields must remain false until the ledger is ready.",
|
||||
},
|
||||
{
|
||||
"key": "completion-phrase",
|
||||
"pattern": re.compile(r"(public\s+)?world[\s_-]*class\s+(?:complete|completed|achieved|certified)", re.IGNORECASE),
|
||||
"reason": "completion language is blocked until the world-class ledger is accepted.",
|
||||
},
|
||||
{
|
||||
"key": "zh-completion-phrase",
|
||||
"pattern": re.compile(r"世界级(?:已完成|完成|已达成|达成|准备就绪|可声明|认证通过)"),
|
||||
"reason": "中文完成态表述必须等到 ledger ready 后才能出现。",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def resolve_claim_surface(raw_path: str, skill_dir: Path) -> Path:
|
||||
path = Path(raw_path)
|
||||
if not path.is_absolute():
|
||||
path = skill_dir / path
|
||||
return path.resolve()
|
||||
|
||||
|
||||
def default_claim_surfaces(skill_dir: Path) -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
for rel in ["README.md", "SKILL.md"]:
|
||||
path = skill_dir / rel
|
||||
if path.exists():
|
||||
paths.append(path)
|
||||
for directory, patterns in [
|
||||
(skill_dir / "docs", ["*.md"]),
|
||||
(skill_dir / "reports", ["*.md", "*.html"]),
|
||||
]:
|
||||
if not directory.exists():
|
||||
continue
|
||||
for pattern in patterns:
|
||||
paths.extend(
|
||||
path
|
||||
for path in directory.rglob(pattern)
|
||||
if path.is_file()
|
||||
and "release_snapshots" not in path.parts
|
||||
and path.name not in {"world_class_claim_guard.md", "world_class_claim_guard.html"}
|
||||
)
|
||||
return sorted(set(paths), key=lambda item: rel_path(item, skill_dir))
|
||||
|
||||
|
||||
def line_matches(path: Path, ledger_ready: bool) -> list[dict[str, Any]]:
|
||||
if ledger_ready:
|
||||
return []
|
||||
violations: list[dict[str, Any]] = []
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
return violations
|
||||
for lineno, line in enumerate(lines, start=1):
|
||||
for rule in FORBIDDEN_WHEN_PENDING:
|
||||
if rule["pattern"].search(line):
|
||||
violations.append(
|
||||
{
|
||||
"line": lineno,
|
||||
"rule": rule["key"],
|
||||
"reason": rule["reason"],
|
||||
"excerpt": line.strip()[:240],
|
||||
}
|
||||
)
|
||||
return violations
|
||||
|
||||
|
||||
def build_guard(skill_dir: Path, generated_at: str, claim_surfaces: list[Path] | None = None) -> dict[str, Any]:
|
||||
ledger = build_ledger(skill_dir, generated_at)
|
||||
ledger_ready = ledger.get("summary", {}).get("ready_to_claim_world_class") is True
|
||||
surfaces = claim_surfaces or default_claim_surfaces(skill_dir)
|
||||
scanned = []
|
||||
violations = []
|
||||
for path in surfaces:
|
||||
matches = line_matches(path, ledger_ready)
|
||||
rel = rel_path(path, skill_dir)
|
||||
scanned.append({"path": rel, "violation_count": len(matches)})
|
||||
for match in matches:
|
||||
violations.append({"path": rel, **match})
|
||||
ok = len(violations) == 0
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": ok,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": ledger_ready,
|
||||
"ledger_pending_count": ledger.get("summary", {}).get("pending_count", 0),
|
||||
"claim_surface_count": len(scanned),
|
||||
"violation_count": len(violations),
|
||||
"overclaim_guard_active": True,
|
||||
"decision": (
|
||||
"claim-ready"
|
||||
if ledger_ready and ok
|
||||
else ("claim-blocked-overclaim" if violations else "claim-guard-pass-evidence-pending")
|
||||
),
|
||||
},
|
||||
"rules": [
|
||||
{"key": item["key"], "reason": item["reason"], "pattern": item["pattern"].pattern}
|
||||
for item in FORBIDDEN_WHEN_PENDING
|
||||
],
|
||||
"scanned_surfaces": scanned,
|
||||
"violations": violations,
|
||||
"source_ledger": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
"markdown": "reports/world_class_evidence_ledger.md",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_claim_guard.json",
|
||||
"markdown": "reports/world_class_claim_guard.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# World-Class Claim Guard",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- ledger ready to claim world-class: `{str(summary['ledger_ready_to_claim_world_class']).lower()}`",
|
||||
f"- ledger pending evidence: `{summary['ledger_pending_count']}`",
|
||||
f"- claim surfaces scanned: `{summary['claim_surface_count']}`",
|
||||
f"- violations: `{summary['violation_count']}`",
|
||||
f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
|
||||
"",
|
||||
"This guard scans public claim surfaces for completion language that would contradict the world-class evidence ledger. It allows evidence planning and pending-state language, but blocks completion claims until the ledger is ready.",
|
||||
"",
|
||||
"## Violations",
|
||||
"",
|
||||
"| Path | Line | Rule | Excerpt |",
|
||||
"| --- | ---: | --- | --- |",
|
||||
]
|
||||
if report["violations"]:
|
||||
for item in report["violations"]:
|
||||
excerpt = str(item["excerpt"]).replace("|", "\\|")
|
||||
lines.append(f"| `{item['path']}` | {item['line']} | `{item['rule']}` | {excerpt} |")
|
||||
else:
|
||||
lines.append("| `none` | 0 | `none` | none |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Rules",
|
||||
"",
|
||||
"| Rule | Reason |",
|
||||
"| --- | --- |",
|
||||
]
|
||||
)
|
||||
for rule in report["rules"]:
|
||||
lines.append(f"| `{rule['key']}` | {rule['reason'].replace('|', '\\|')} |")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Guard public world-class completion claims against the evidence ledger.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--claim-surface", action="append", default=[])
|
||||
parser.add_argument("--output-json", default="reports/world_class_claim_guard.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_claim_guard.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
claim_surfaces = [resolve_claim_surface(item, skill_dir) for item in args.claim_surface] if args.claim_surface else None
|
||||
report = build_guard(skill_dir, args.generated_at, claim_surfaces)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -23,6 +23,7 @@ LABELS = {
|
||||
"baseline_pass_rate": "Baseline",
|
||||
"breaking_change_count": "破坏变更",
|
||||
"case_count": "案例数",
|
||||
"claim_surface_count": "声明面",
|
||||
"command_executed_count": "命令执行",
|
||||
"compatibility_pass_count": "兼容通过",
|
||||
"covered_gate_count": "覆盖 Gate",
|
||||
@@ -37,6 +38,8 @@ LABELS = {
|
||||
"installer_permission_failure_count": "安装权限失败",
|
||||
"invalid_submission_count": "无效提交",
|
||||
"item_count": "项目数",
|
||||
"ledger_pending_count": "台账待补",
|
||||
"ledger_ready_to_claim_world_class": "台账可声明",
|
||||
"license": "License",
|
||||
"local_blueprint_ready": "本地蓝图",
|
||||
"metadata_fallback_count": "Metadata fallback",
|
||||
@@ -82,6 +85,7 @@ LABELS = {
|
||||
"variant_run_count": "运行数",
|
||||
"valid_submission_count": "有效提交",
|
||||
"version": "版本",
|
||||
"violation_count": "违规数",
|
||||
"warning_count": "警告数",
|
||||
"with_skill_pass_rate": "With Skill",
|
||||
"world_class_evidence_pending_count": "待补证据",
|
||||
|
||||
@@ -468,6 +468,7 @@ def command_report(args: argparse.Namespace) -> int:
|
||||
run_script("render_world_class_evidence_plan.py", [str(ROOT)]),
|
||||
run_script("render_world_class_evidence_ledger.py", [str(ROOT)]),
|
||||
run_script("render_world_class_evidence_intake.py", [str(ROOT)]),
|
||||
run_script("render_world_class_claim_guard.py", [str(ROOT)]),
|
||||
run_script("render_benchmark_reproducibility.py", [str(ROOT)]),
|
||||
run_script("render_skill_os2_coverage.py", [str(ROOT)]),
|
||||
]
|
||||
@@ -499,6 +500,7 @@ def command_report(args: argparse.Namespace) -> int:
|
||||
"world_class_evidence_plan": "reports/world_class_evidence_plan.json",
|
||||
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.json",
|
||||
"world_class_evidence_intake": "reports/world_class_evidence_intake.json",
|
||||
"world_class_claim_guard": "reports/world_class_claim_guard.json",
|
||||
"benchmark_reproducibility": "reports/benchmark_reproducibility.json",
|
||||
"skill_os2_coverage": "reports/skill_os2_coverage.json",
|
||||
},
|
||||
@@ -615,6 +617,22 @@ def command_world_class_intake(args: argparse.Namespace) -> int:
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_world_class_claim_guard(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir]
|
||||
for surface in args.claim_surface:
|
||||
cmd.extend(["--claim-surface", surface])
|
||||
if args.output_json:
|
||||
cmd.extend(["--output-json", args.output_json])
|
||||
if args.output_md:
|
||||
cmd.extend(["--output-md", args.output_md])
|
||||
if args.generated_at:
|
||||
cmd.extend(["--generated-at", args.generated_at])
|
||||
result = run_script("render_world_class_claim_guard.py", cmd)
|
||||
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_benchmark_reproducibility(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir]
|
||||
@@ -1252,6 +1270,7 @@ def command_workspace_flow(args: argparse.Namespace) -> int:
|
||||
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])},
|
||||
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])},
|
||||
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_intake.py", [str(ROOT)])},
|
||||
{"phase": "report-refresh", "result": run_script("render_world_class_claim_guard.py", [str(ROOT)])},
|
||||
{"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])},
|
||||
{"phase": "report-refresh", "result": run_script("render_skill_os2_coverage.py", [str(ROOT)])},
|
||||
]
|
||||
|
||||
@@ -183,6 +183,17 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]
|
||||
world_class_intake_cmd.add_argument("--generated-at")
|
||||
world_class_intake_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_intake"))
|
||||
|
||||
world_class_claim_guard_cmd = subparsers.add_parser(
|
||||
"world-class-claim-guard",
|
||||
help="Scan public claim surfaces for premature world-class completion claims.",
|
||||
)
|
||||
world_class_claim_guard_cmd.add_argument("skill_dir", nargs="?", default=".")
|
||||
world_class_claim_guard_cmd.add_argument("--claim-surface", action="append", default=[])
|
||||
world_class_claim_guard_cmd.add_argument("--output-json")
|
||||
world_class_claim_guard_cmd.add_argument("--output-md")
|
||||
world_class_claim_guard_cmd.add_argument("--generated-at")
|
||||
world_class_claim_guard_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_claim_guard"))
|
||||
|
||||
benchmark_reproducibility_cmd = subparsers.add_parser(
|
||||
"benchmark-reproducibility",
|
||||
help="Render benchmark methodology, artifact, failure-disclosure, and reproduction-command evidence.",
|
||||
|
||||
@@ -160,6 +160,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
@@ -186,8 +187,7 @@
|
||||
"scripts/verify_package.py",
|
||||
"scripts/yao.py",
|
||||
"scripts/yao_cli_config.py",
|
||||
"scripts/yao_cli_parser.py",
|
||||
"scripts/yao_cli_telemetry.py"
|
||||
"scripts/yao_cli_parser.py"
|
||||
],
|
||||
"assets": [
|
||||
"templates/basic_skill.md.j2",
|
||||
|
||||
@@ -72,6 +72,7 @@
|
||||
"scripts/render_social_preview.py",
|
||||
"scripts/render_system_model.py",
|
||||
"scripts/render_telemetry_hook_recipes.py",
|
||||
"scripts/render_world_class_claim_guard.py",
|
||||
"scripts/render_world_class_evidence_intake.py",
|
||||
"scripts/render_world_class_evidence_ledger.py",
|
||||
"scripts/render_world_class_evidence_plan.py",
|
||||
|
||||
@@ -40,6 +40,19 @@ def main() -> None:
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "render_world_class_claim_guard.py"),
|
||||
str(ROOT),
|
||||
"--generated-at",
|
||||
"2026-06-13",
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
output_json = TMP / "benchmark_reproducibility.json"
|
||||
output_md = TMP / "benchmark_reproducibility.md"
|
||||
proc = subprocess.run(
|
||||
@@ -82,9 +95,11 @@ def main() -> None:
|
||||
assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts
|
||||
assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts
|
||||
assert artifacts["reports/world_class_evidence_intake.json"]["exists"], artifacts
|
||||
assert artifacts["reports/world_class_claim_guard.json"]["exists"], artifacts
|
||||
assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload
|
||||
assert any(command["command"] == "python3 scripts/yao.py world-class-ledger ." for command in payload["reproduction_commands"]), payload
|
||||
assert any(command["command"] == "python3 scripts/yao.py world-class-intake ." for command in payload["reproduction_commands"]), payload
|
||||
assert any(command["command"] == "python3 scripts/yao.py world-class-claim-guard ." for command in payload["reproduction_commands"]), payload
|
||||
assert any("provider-backed" in item for item in payload["limitations"]), payload["limitations"]
|
||||
markdown = output_md.read_text(encoding="utf-8")
|
||||
assert "Benchmark Reproducibility" in markdown, markdown
|
||||
|
||||
@@ -223,6 +223,7 @@ def main() -> None:
|
||||
"render_world_class_evidence_plan.py",
|
||||
"render_world_class_evidence_ledger.py",
|
||||
"render_world_class_evidence_intake.py",
|
||||
"render_world_class_claim_guard.py",
|
||||
"render_skill_os2_coverage.py",
|
||||
]:
|
||||
subprocess.run(
|
||||
@@ -335,6 +336,8 @@ def main() -> None:
|
||||
assert full_payload["evidence_paths"]["world_class_evidence_ledger"] == "reports/world_class_evidence_ledger.md", full_payload["evidence_paths"]
|
||||
if (ROOT / "reports" / "world_class_evidence_intake.md").exists():
|
||||
assert full_payload["evidence_paths"]["world_class_evidence_intake"] == "reports/world_class_evidence_intake.md", full_payload["evidence_paths"]
|
||||
if (ROOT / "reports" / "world_class_claim_guard.md").exists():
|
||||
assert full_payload["evidence_paths"]["world_class_claim_guard"] == "reports/world_class_claim_guard.md", full_payload["evidence_paths"]
|
||||
assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"]
|
||||
assert full_payload["data"]["output_execution"]["summary"]["command_executed_count"] == 10, full_payload["data"]["output_execution"]
|
||||
assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"]
|
||||
@@ -355,6 +358,7 @@ def main() -> None:
|
||||
"reports/world_class_evidence_ledger.md",
|
||||
"reports/world_class_evidence_plan.md",
|
||||
"reports/world_class_evidence_intake.md",
|
||||
"reports/world_class_claim_guard.md",
|
||||
"evidence/world_class/intake.schema.json",
|
||||
"evidence/world_class/templates/provider-holdout.intake.json",
|
||||
"evidence/world_class/templates/human-adjudication.intake.json",
|
||||
@@ -365,6 +369,9 @@ def main() -> None:
|
||||
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_evidence_intake"]
|
||||
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["template_pass_count"] == 4, full_payload["data"]["world_class_evidence_intake"]
|
||||
assert full_payload["data"]["world_class_evidence_intake"]["summary"]["ready_to_claim_world_class"] is False, full_payload["data"]["world_class_evidence_intake"]
|
||||
assert full_payload["data"]["world_class_claim_guard"]["summary"]["decision"] == "claim-guard-pass-evidence-pending", full_payload["data"]["world_class_claim_guard"]
|
||||
assert full_payload["data"]["world_class_claim_guard"]["summary"]["violation_count"] == 0, full_payload["data"]["world_class_claim_guard"]
|
||||
assert full_payload["data"]["world_class_claim_guard"]["summary"]["ledger_pending_count"] == 4, full_payload["data"]["world_class_claim_guard"]
|
||||
if full_payload["data"]["skill_os2_coverage"]:
|
||||
assert full_payload["data"]["skill_os2_coverage"]["summary"]["local_blueprint_ready"] is True, full_payload["data"]["skill_os2_coverage"]
|
||||
assert full_payload["data"]["skill_os2_coverage"]["summary"]["public_world_class_ready"] is False, full_payload["data"]["skill_os2_coverage"]
|
||||
@@ -427,8 +434,12 @@ def main() -> None:
|
||||
assert "证据台账" in html, html
|
||||
assert "证据入口" in html, html
|
||||
assert "入口边界" in html, html
|
||||
assert "声明守卫" in html, html
|
||||
assert "声明边界" in html, html
|
||||
assert "intake 只校验证据包格式、来源、隐私和反过度声明" in html, html
|
||||
assert "reports/world_class_evidence_intake.md" in html, html
|
||||
assert "reports/world_class_claim_guard.md" in html, html
|
||||
assert "英文完成断言、true 状态声明或中文完成态" in html, html
|
||||
assert "world-evidence-grid" in html, html
|
||||
assert "Provider Holdout" in html, html
|
||||
assert "Native Permission Enforcement" in html, html
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = ROOT / "scripts" / "render_world_class_claim_guard.py"
|
||||
TMP = ROOT / "tests" / "tmp_world_class_claim_guard"
|
||||
|
||||
|
||||
def run_guard(*extra: str, check: bool = True, cwd: Path = ROOT) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(ROOT),
|
||||
"--generated-at",
|
||||
"2026-06-14",
|
||||
*extra,
|
||||
],
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
shutil.rmtree(TMP, ignore_errors=True)
|
||||
TMP.mkdir(parents=True, exist_ok=True)
|
||||
output_json = TMP / "world_class_claim_guard.json"
|
||||
output_md = TMP / "world_class_claim_guard.md"
|
||||
proc = run_guard("--output-json", str(output_json), "--output-md", str(output_md))
|
||||
payload = json.loads(proc.stdout)
|
||||
assert payload["schema_version"] == "1.0", payload
|
||||
assert payload["ok"] is True, payload
|
||||
summary = payload["summary"]
|
||||
assert summary["decision"] == "claim-guard-pass-evidence-pending", summary
|
||||
assert summary["ledger_ready_to_claim_world_class"] is False, summary
|
||||
assert summary["ledger_pending_count"] == 4, summary
|
||||
assert summary["claim_surface_count"] >= 10, summary
|
||||
assert summary["violation_count"] == 0, summary
|
||||
assert summary["overclaim_guard_active"] is True, summary
|
||||
assert any(item["path"] == "README.md" for item in payload["scanned_surfaces"]), payload["scanned_surfaces"]
|
||||
markdown = output_md.read_text(encoding="utf-8")
|
||||
assert "World-Class Claim Guard" in markdown, markdown
|
||||
assert "claim-guard-pass-evidence-pending" in markdown, markdown
|
||||
assert "world-class evidence ledger" in markdown, markdown
|
||||
|
||||
safe_surface = TMP / "safe.md"
|
||||
safe_surface.write_text("ready to claim world-class: `false`\nworld-class evidence is pending.\n", encoding="utf-8")
|
||||
safe_proc = run_guard("--claim-surface", str(safe_surface), check=True)
|
||||
safe_payload = json.loads(safe_proc.stdout)
|
||||
assert safe_payload["summary"]["violation_count"] == 0, safe_payload
|
||||
|
||||
unsafe_surface = TMP / "unsafe.md"
|
||||
unsafe_surface.write_text("ready to claim world-class: `true`\n世界级已完成\n", encoding="utf-8")
|
||||
unsafe_proc = run_guard("--claim-surface", str(unsafe_surface), check=False)
|
||||
assert unsafe_proc.returncode == 2, unsafe_proc.stdout
|
||||
unsafe_payload = json.loads(unsafe_proc.stdout)
|
||||
assert unsafe_payload["ok"] is False, unsafe_payload
|
||||
assert unsafe_payload["summary"]["decision"] == "claim-blocked-overclaim", unsafe_payload
|
||||
assert unsafe_payload["summary"]["violation_count"] == 2, unsafe_payload
|
||||
rules = {item["rule"] for item in unsafe_payload["violations"]}
|
||||
assert {"ready-to-claim-true", "zh-completion-phrase"} <= rules, unsafe_payload["violations"]
|
||||
|
||||
relative_unsafe_surface = TMP / "relative-unsafe.md"
|
||||
relative_unsafe_surface.write_text("world-class completed\n", encoding="utf-8")
|
||||
relative_proc = run_guard(
|
||||
"--claim-surface",
|
||||
relative_unsafe_surface.relative_to(ROOT).as_posix(),
|
||||
check=False,
|
||||
cwd=TMP,
|
||||
)
|
||||
assert relative_proc.returncode == 2, relative_proc.stdout
|
||||
relative_payload = json.loads(relative_proc.stdout)
|
||||
assert relative_payload["summary"]["violation_count"] == 1, relative_payload
|
||||
assert relative_payload["violations"][0]["path"] == "tests/tmp_world_class_claim_guard/relative-unsafe.md", relative_payload
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -78,6 +78,7 @@ def main() -> None:
|
||||
assert "world-class-evidence" in parser_help, parser_help
|
||||
assert "world-class-ledger" in parser_help, parser_help
|
||||
assert "world-class-intake" in parser_help, parser_help
|
||||
assert "world-class-claim-guard" in parser_help, parser_help
|
||||
assert "benchmark-reproducibility" in parser_help, parser_help
|
||||
assert "telemetry-import" in parser_help, parser_help
|
||||
assert "telemetry-emit" in parser_help, parser_help
|
||||
@@ -226,6 +227,21 @@ def main() -> None:
|
||||
assert world_class_intake_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_intake_result
|
||||
assert world_class_ledger_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_ledger_result
|
||||
|
||||
world_class_claim_guard_result = run(
|
||||
"world-class-claim-guard",
|
||||
str(ROOT),
|
||||
"--output-json",
|
||||
str(tmp_root / "world_class_claim_guard.json"),
|
||||
"--output-md",
|
||||
str(tmp_root / "world_class_claim_guard.md"),
|
||||
"--generated-at",
|
||||
"2026-06-14",
|
||||
)
|
||||
assert world_class_claim_guard_result["ok"], world_class_claim_guard_result
|
||||
assert world_class_claim_guard_result["payload"]["summary"]["decision"] == "claim-guard-pass-evidence-pending", world_class_claim_guard_result
|
||||
assert world_class_claim_guard_result["payload"]["summary"]["violation_count"] == 0, world_class_claim_guard_result
|
||||
assert world_class_claim_guard_result["payload"]["summary"]["ledger_pending_count"] == 4, world_class_claim_guard_result
|
||||
|
||||
benchmark_reproducibility_result = run(
|
||||
"benchmark-reproducibility",
|
||||
str(ROOT),
|
||||
@@ -650,6 +666,7 @@ def main() -> None:
|
||||
assert "world_class_evidence_plan" in report_result["payload"]["artifacts"], report_result
|
||||
assert "world_class_evidence_ledger" in report_result["payload"]["artifacts"], report_result
|
||||
assert "world_class_evidence_intake" in report_result["payload"]["artifacts"], report_result
|
||||
assert "world_class_claim_guard" in report_result["payload"]["artifacts"], report_result
|
||||
assert "benchmark_reproducibility" in report_result["payload"]["artifacts"], report_result
|
||||
assert "skill_os2_coverage" in report_result["payload"]["artifacts"], report_result
|
||||
report_output_execution = json.loads((ROOT / "reports" / "output_execution_runs.json").read_text(encoding="utf-8"))
|
||||
|
||||
Reference in New Issue
Block a user