docs: refresh benchmark evidence bundle reports
This commit is contained in:
yaojingang
@@ -3,31 +3,23 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"skill_dir": ".",
|
||||
"commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308",
|
||||
"commit": "165ef11daf45ffff44169935c71de280348af936",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": true,
|
||||
"changed_file_count": 50,
|
||||
"sample": [
|
||||
" M AGENTS.md",
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/adoption_drift_report.md",
|
||||
" M reports/architecture_maintainability.json",
|
||||
" M reports/architecture_maintainability.md",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/install_simulation.json",
|
||||
" M reports/install_simulation.md"
|
||||
]
|
||||
"dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"sample": [],
|
||||
"scope": "generation-time status before this report is written"
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "de8906187bdb5ad75cfc4e606ed9cd38b88cd23232fce6a60912be45b16bce06",
|
||||
"source_contract_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 21,
|
||||
@@ -41,8 +33,22 @@
|
||||
"world_class_open_gap_count": 4,
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 50
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
},
|
||||
"release_lock": {
|
||||
"ready": true,
|
||||
"commit": "165ef11daf45ffff44169935c71de280348af936",
|
||||
"status_scope": "generation-time status before this report is written",
|
||||
"reason": "clean generation-time HEAD"
|
||||
},
|
||||
"evidence_bundle": {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
"artifact_count": 24,
|
||||
"existing_count": 24,
|
||||
"missing_count": 0,
|
||||
"missing_paths": [],
|
||||
"sha256": "de8906187bdb5ad75cfc4e606ed9cd38b88cd23232fce6a60912be45b16bce06"
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -115,8 +121,8 @@
|
||||
"label": "output_execution",
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "0ff38df2c18b25b79b9bf36e3055908a57d920ff45fa697f8281ce3a960f61b5"
|
||||
"bytes": 7964,
|
||||
"sha256": "c90136a4365e2db855de37883bff7266b581bb132717770e041f05e4b01c282e"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -158,7 +164,7 @@
|
||||
"path": "reports/python_compatibility.json",
|
||||
"exists": true,
|
||||
"bytes": 20499,
|
||||
"sha256": "4edb5722c888f2fe5fc6c3141535ab5d550b6b1c05e9b4230b77e6712a982fb4"
|
||||
"sha256": "512efa58cc6fa3e950b4d9dde0c941ceefcdae871f9379ca48591bcc2d819f1f"
|
||||
},
|
||||
{
|
||||
"label": "registry_audit",
|
||||
@@ -193,42 +199,42 @@
|
||||
"path": "reports/world_class_evidence_plan.json",
|
||||
"exists": true,
|
||||
"bytes": 19532,
|
||||
"sha256": "a9bd392cd45f3accabe9b5befb9090456f383b1edbc323cab7d20930179dc229"
|
||||
"sha256": "94abf1c53494bff48c4aca1de59281e075f67901f9c6e2d9bb7413315a99ae60"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_ledger",
|
||||
"path": "reports/world_class_evidence_ledger.json",
|
||||
"exists": true,
|
||||
"bytes": 11613,
|
||||
"sha256": "0bff14542475657a1261c1afcd87ffabc5d5c5e75bcd20afcfeacc44b16b7c14"
|
||||
"sha256": "0bdb4bc97f6f37ad1a6ec6df953d4d682780a5a813ac6abff4a003159378d7fb"
|
||||
},
|
||||
{
|
||||
"label": "world_class_evidence_intake",
|
||||
"path": "reports/world_class_evidence_intake.json",
|
||||
"exists": true,
|
||||
"bytes": 13356,
|
||||
"sha256": "2cb7e6beb565eacf4d4d636f9488d6b88b41c6cb2cbcbcc30eac82e7a3160fbf"
|
||||
"sha256": "fe89d51331e2acf22b9049f1b423ff66cac972179c1a8525e9b96ba1bdc88a6b"
|
||||
},
|
||||
{
|
||||
"label": "world_class_submission_review",
|
||||
"path": "reports/world_class_submission_review.json",
|
||||
"exists": true,
|
||||
"bytes": 7263,
|
||||
"sha256": "4f03edb2ef28d9dfd27a393f8058d6e27d2b10fa8cbfe35f473e003556b9ffe0"
|
||||
"sha256": "18e3f112099ab9f1c89eff5765d6b2b9c28bb1c6b3c11df11cc9633f853da2e7"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook",
|
||||
"path": "reports/world_class_operator_runbook.json",
|
||||
"exists": true,
|
||||
"bytes": 15002,
|
||||
"sha256": "614742ea666c64cdb68e10fc77022c31e68abe82ed88325aea21acb2f277063e"
|
||||
"sha256": "05f8163361207059529749abd5a9d90a465231b0eb8f19846bec8dc97e9f664e"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_markdown",
|
||||
"path": "reports/world_class_operator_runbook.md",
|
||||
"exists": true,
|
||||
"bytes": 9206,
|
||||
"sha256": "f67ca3d0aa9c02953e967d9657ca4cfe41ee883c73950bce16d6c25a1e206e10"
|
||||
"sha256": "058ed307e18c12a9f94c4fd17c4caf483e6ab3a5e13b29072957a8ad0500c496"
|
||||
},
|
||||
{
|
||||
"label": "world_class_operator_runbook_html",
|
||||
@@ -242,7 +248,7 @@
|
||||
"path": "reports/world_class_claim_guard.json",
|
||||
"exists": true,
|
||||
"bytes": 8463,
|
||||
"sha256": "250d616b028cf046e2033f9e2c5648c8d95c110d0d8990b35e4d481e14cfc558"
|
||||
"sha256": "44a278d19978f36117cb681cde84b4d2b23e76642d11960457b22700c9ba1847"
|
||||
}
|
||||
],
|
||||
"missing_artifacts": [],
|
||||
@@ -359,6 +365,7 @@
|
||||
"policy": "Keep representative failures visible and tied to regression checks."
|
||||
},
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
"Pending blind-review decisions are visible but do not count as human adjudication.",
|
||||
"World-class readiness remains false until external and human evidence gaps close."
|
||||
|
||||
@@ -1,24 +1,40 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Commit: `48f2826b8c9ebad3a9cc90e6166f4fd253f98308`
|
||||
Working tree dirty at generation: `true`
|
||||
Commit: `165ef11daf45ffff44169935c71de280348af936`
|
||||
Working tree dirty at generation: `false`
|
||||
Evidence bundle SHA256: `de8906187bdb5ad75cfc4e606ed9cd38b88cd23232fce6a60912be45b16bce06`
|
||||
|
||||
## Summary
|
||||
|
||||
- reproducibility ready: `true`
|
||||
- release lock ready: `true`
|
||||
- methodology complete: `true`
|
||||
- required artifacts: `24`
|
||||
- missing artifacts: `0`
|
||||
- source contract sha256: `c42643ea854d`
|
||||
- archive sha256: `76793bd367e9`
|
||||
- output cases: `5`
|
||||
- disclosed failure cases: `3`
|
||||
- reproduction commands: `21`
|
||||
- provider evidence complete: `false`
|
||||
- human review complete: `false`
|
||||
- world-class ready: `false`
|
||||
- changed files at generation: `50`
|
||||
- changed files at generation: `0`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.
|
||||
|
||||
## Release Lock
|
||||
|
||||
- ready: `true`
|
||||
- reason: clean generation-time HEAD
|
||||
- status scope: generation-time status before this report is written
|
||||
|
||||
## Evidence Bundle
|
||||
|
||||
- algorithm: `sha256(path,label,exists,artifact_sha256)`
|
||||
- artifacts: `24` / `24`
|
||||
- sha256: `de8906187bdb5ad75cfc4e606ed9cd38b88cd23232fce6a60912be45b16bce06`
|
||||
|
||||
## Methodology Sections
|
||||
|
||||
@@ -40,25 +56,25 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `0ff38df2c18b` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `c90136a4365e` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `3c95ea60b1c9` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `4edb5722c888` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `512efa58cc6f` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `ad7900024ed6` |
|
||||
| package_verification | `reports/package_verification.json` | present | `dd12856ce830` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `aabe72c3f5bd` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `0c74deea906e` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `a9bd392cd45f` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0bff14542475` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `2cb7e6beb565` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `4f03edb2ef28` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `614742ea666c` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `f67ca3d0aa9c` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `94abf1c53494` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0bdb4bc97f6f` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `fe89d51331e2` |
|
||||
| world_class_submission_review | `reports/world_class_submission_review.json` | present | `18e3f112099a` |
|
||||
| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `05f816336120` |
|
||||
| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `058ed307e18c` |
|
||||
| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `963026f551dc` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `250d616b028c` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `44a278d19978` |
|
||||
|
||||
## Reproduction Commands
|
||||
|
||||
@@ -113,6 +129,7 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
## Limits
|
||||
|
||||
- The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.
|
||||
- Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.
|
||||
- Pending blind-review decisions are visible but do not count as human adjudication.
|
||||
- World-class readiness remains false until external and human evidence gaps close.
|
||||
|
||||
+19
-14
@@ -1332,9 +1332,13 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "de8906187bdb5ad75cfc4e606ed9cd38b88cd23232fce6a60912be45b16bce06",
|
||||
"source_contract_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 21,
|
||||
@@ -1348,12 +1352,13 @@
|
||||
"world_class_open_gap_count": 4,
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 50
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
},
|
||||
"commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308",
|
||||
"commit": "165ef11daf45ffff44169935c71de280348af936",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
"Pending blind-review decisions are visible but do not count as human adjudication.",
|
||||
"World-class readiness remains false until external and human evidence gaps close."
|
||||
@@ -3804,7 +3809,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.38,
|
||||
"duration_ms": 30.68,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3832,7 +3837,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.21,
|
||||
"duration_ms": 28.53,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3855,7 +3860,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.7,
|
||||
"duration_ms": 29.59,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3883,7 +3888,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.35,
|
||||
"duration_ms": 31.84,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3906,7 +3911,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.38,
|
||||
"duration_ms": 30.95,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3934,7 +3939,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.65,
|
||||
"duration_ms": 27.6,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3957,7 +3962,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.6,
|
||||
"duration_ms": 26.76,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3984,7 +3989,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.75,
|
||||
"duration_ms": 26.87,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4007,7 +4012,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.86,
|
||||
"duration_ms": 26.9,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4036,7 +4041,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.32,
|
||||
"duration_ms": 27.3,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -15615,7 +15620,7 @@
|
||||
"adoption_drift": {
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-14T00:50:30Z",
|
||||
"generated_at": "2026-06-14T00:52:22Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
|
||||
@@ -918,9 +918,13 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "de8906187bdb5ad75cfc4e606ed9cd38b88cd23232fce6a60912be45b16bce06",
|
||||
"source_contract_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb",
|
||||
"archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe",
|
||||
"output_case_count": 5,
|
||||
"failure_disclosure_count": 3,
|
||||
"command_count": 21,
|
||||
@@ -934,12 +938,13 @@
|
||||
"world_class_open_gap_count": 4,
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 50
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
},
|
||||
"commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308",
|
||||
"commit": "165ef11daf45ffff44169935c71de280348af936",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
"Pending blind-review decisions are visible but do not count as human adjudication.",
|
||||
"World-class readiness remains false until external and human evidence gaps close."
|
||||
|
||||
Reference in New Issue
Block a user