wehub-resource-sync
313 lines
8.3 KiB
JSON
313 lines
8.3 KiB
JSON
{
|
|
"schema_version": "1.0",
|
|
"ok": true,
|
|
"cases": "evals/output/cases.jsonl",
|
|
"runner": {
|
|
"mode": "command",
|
|
"command": [
|
|
"python3",
|
|
"scripts/provider_output_eval_runner.py",
|
|
"--provider",
|
|
"deepseek",
|
|
"--model",
|
|
"deepseek-v4-flash",
|
|
"--api-format",
|
|
"chat-completions",
|
|
"--thinking",
|
|
"disabled",
|
|
"--api-key-env",
|
|
"DEEPSEEK_API_KEY"
|
|
],
|
|
"timeout_seconds": 180.0
|
|
},
|
|
"summary": {
|
|
"case_count": 5,
|
|
"variant_run_count": 10,
|
|
"command_executed_count": 10,
|
|
"model_executed_count": 10,
|
|
"recorded_fixture_count": 0,
|
|
"timing_observed_count": 10,
|
|
"token_observed_count": 10,
|
|
"token_estimated_count": 0,
|
|
"baseline_pass_rate": 23.67,
|
|
"with_skill_pass_rate": 43.67,
|
|
"delta": 20.0,
|
|
"regression_count": 0,
|
|
"failure_count": 0,
|
|
"gate_pass": true
|
|
},
|
|
"runs": [
|
|
{
|
|
"case_id": "skill-package-contract",
|
|
"variant": "baseline",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 5980.01,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 100,
|
|
"output_tokens": 384,
|
|
"total_tokens": 484,
|
|
"estimated": false
|
|
},
|
|
"score": 0.0,
|
|
"passed_count": 0,
|
|
"failed_count": 4,
|
|
"failed_assertions": [
|
|
"has-entrypoint",
|
|
"has-interface",
|
|
"has-report",
|
|
"has-resource-boundary"
|
|
],
|
|
"output_sha256": "1aa12d65d55863581eef67aaa173b3a9962524a91533693044cb340a342b760b",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "skill-package-contract",
|
|
"variant": "with_skill",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 11217.19,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 850,
|
|
"output_tokens": 998,
|
|
"total_tokens": 1848,
|
|
"estimated": false
|
|
},
|
|
"score": 75.0,
|
|
"passed_count": 3,
|
|
"failed_count": 1,
|
|
"failed_assertions": [
|
|
"has-report"
|
|
],
|
|
"output_sha256": "ac2cb822520b3099150ea1df07de59b9c6df382b7251792e0662631a60989d89",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "output-eval-expectation",
|
|
"variant": "baseline",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 3178.95,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 107,
|
|
"output_tokens": 129,
|
|
"total_tokens": 236,
|
|
"estimated": false
|
|
},
|
|
"score": 0.0,
|
|
"passed_count": 0,
|
|
"failed_count": 4,
|
|
"failed_assertions": [
|
|
"has-baseline-comparison",
|
|
"has-assertion-grading",
|
|
"has-scorecard",
|
|
"has-failure-taxonomy"
|
|
],
|
|
"output_sha256": "a128ff4dbd7870e287ac651b6cab6b7887e347d6477e5bd8c6f40cd9479539f0",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "output-eval-expectation",
|
|
"variant": "with_skill",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 4611.78,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 857,
|
|
"output_tokens": 264,
|
|
"total_tokens": 1121,
|
|
"estimated": false
|
|
},
|
|
"score": 25.0,
|
|
"passed_count": 1,
|
|
"failed_count": 3,
|
|
"failed_assertions": [
|
|
"has-baseline-comparison",
|
|
"has-assertion-grading",
|
|
"has-failure-taxonomy"
|
|
],
|
|
"output_sha256": "4df98d20e2199e56dae2618b6a64e59d5c96484a4f71ae0ebc674b755cf533c3",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "ir-before-packaging",
|
|
"variant": "baseline",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 6772.59,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 104,
|
|
"output_tokens": 661,
|
|
"total_tokens": 765,
|
|
"estimated": false
|
|
},
|
|
"score": 25.0,
|
|
"passed_count": 1,
|
|
"failed_count": 3,
|
|
"failed_assertions": [
|
|
"has-ir-first",
|
|
"has-core-ir-fields",
|
|
"has-semantic-preservation"
|
|
],
|
|
"output_sha256": "1696075b4f884ff8b478786b5565aff73ad73235936eb39c2ed615c0746861d2",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "ir-before-packaging",
|
|
"variant": "with_skill",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 17471.19,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 854,
|
|
"output_tokens": 2132,
|
|
"total_tokens": 2986,
|
|
"estimated": false
|
|
},
|
|
"score": 25.0,
|
|
"passed_count": 1,
|
|
"failed_count": 3,
|
|
"failed_assertions": [
|
|
"has-ir-first",
|
|
"has-core-ir-fields",
|
|
"has-semantic-preservation"
|
|
],
|
|
"output_sha256": "4f87633e53f360948d6403126d49ad3dabcb1b47aff596918650719b7840abcf",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "near-neighbor-boundary",
|
|
"variant": "baseline",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 3091.33,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 105,
|
|
"output_tokens": 93,
|
|
"total_tokens": 198,
|
|
"estimated": false
|
|
},
|
|
"score": 33.33,
|
|
"passed_count": 1,
|
|
"failed_count": 2,
|
|
"failed_assertions": [
|
|
"declines-skill",
|
|
"requires-reuse-signal"
|
|
],
|
|
"output_sha256": "56c8f48f3b2b9963adad3afdb1b6a2bed6b162067506edee1e0f1acd32668a91",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "near-neighbor-boundary",
|
|
"variant": "with_skill",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 4152.85,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 855,
|
|
"output_tokens": 160,
|
|
"total_tokens": 1015,
|
|
"estimated": false
|
|
},
|
|
"score": 33.33,
|
|
"passed_count": 1,
|
|
"failed_count": 2,
|
|
"failed_assertions": [
|
|
"declines-skill",
|
|
"requires-reuse-signal"
|
|
],
|
|
"output_sha256": "d2d85a2aae5db5c4c7e6e0db5bdd865157a90018afe302b0a703940cc84ba68c",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "file-backed-governed-package",
|
|
"variant": "baseline",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 4750.01,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 200,
|
|
"output_tokens": 302,
|
|
"total_tokens": 502,
|
|
"estimated": false
|
|
},
|
|
"score": 60.0,
|
|
"passed_count": 3,
|
|
"failed_count": 2,
|
|
"failed_assertions": [
|
|
"uses-file-backed-evidence",
|
|
"has-trust-and-scorecard"
|
|
],
|
|
"output_sha256": "d41880b0c2f7e60ee6acb2cb1f4acb9377fd63de72b8e026e6259a962b89af0b",
|
|
"failure": ""
|
|
},
|
|
{
|
|
"case_id": "file-backed-governed-package",
|
|
"variant": "with_skill",
|
|
"status": "pass",
|
|
"execution_mode": "model",
|
|
"model_executed": true,
|
|
"command_executed": true,
|
|
"duration_ms": 9406.25,
|
|
"provider": "deepseek",
|
|
"model": "deepseek-v4-flash",
|
|
"usage": {
|
|
"input_tokens": 950,
|
|
"output_tokens": 660,
|
|
"total_tokens": 1610,
|
|
"estimated": false
|
|
},
|
|
"score": 60.0,
|
|
"passed_count": 3,
|
|
"failed_count": 2,
|
|
"failed_assertions": [
|
|
"has-governance",
|
|
"has-output-contract"
|
|
],
|
|
"output_sha256": "c670ae2ef81a819a9a0f1b0c046e82ba4c649f56dcbe96a45a6938c7f214fba4",
|
|
"failure": ""
|
|
}
|
|
],
|
|
"failures": [],
|
|
"artifacts": {
|
|
"json": "reports/output_execution_runs.json",
|
|
"markdown": "reports/output_execution_runs.md"
|
|
}
|
|
}
|