{
  "schema_version": "1.0",
  "ok": true,
  "cases": "evals/output/cases.jsonl",
  "runner": {
    "mode": "command",
    "command": [
      "python3",
      "scripts/provider_output_eval_runner.py",
      "--provider",
      "deepseek",
      "--model",
      "deepseek-v4-flash",
      "--api-format",
      "chat-completions",
      "--thinking",
      "disabled",
      "--api-key-env",
      "DEEPSEEK_API_KEY"
    ],
    "timeout_seconds": 180.0
  },
  "summary": {
    "case_count": 5,
    "variant_run_count": 10,
    "command_executed_count": 10,
    "model_executed_count": 10,
    "recorded_fixture_count": 0,
    "timing_observed_count": 10,
    "token_observed_count": 10,
    "token_estimated_count": 0,
    "baseline_pass_rate": 23.67,
    "with_skill_pass_rate": 43.67,
    "delta": 20.0,
    "regression_count": 0,
    "failure_count": 0,
    "gate_pass": true
  },
  "runs": [
    {
      "case_id": "skill-package-contract",
      "variant": "baseline",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 5980.01,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 100,
        "output_tokens": 384,
        "total_tokens": 484,
        "estimated": false
      },
      "score": 0.0,
      "passed_count": 0,
      "failed_count": 4,
      "failed_assertions": [
        "has-entrypoint",
        "has-interface",
        "has-report",
        "has-resource-boundary"
      ],
      "output_sha256": "1aa12d65d55863581eef67aaa173b3a9962524a91533693044cb340a342b760b",
      "failure": ""
    },
    {
      "case_id": "skill-package-contract",
      "variant": "with_skill",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 11217.19,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 850,
        "output_tokens": 998,
        "total_tokens": 1848,
        "estimated": false
      },
      "score": 75.0,
      "passed_count": 3,
      "failed_count": 1,
      "failed_assertions": [
        "has-report"
      ],
      "output_sha256": "ac2cb822520b3099150ea1df07de59b9c6df382b7251792e0662631a60989d89",
      "failure": ""
    },
    {
      "case_id": "output-eval-expectation",
      "variant": "baseline",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 3178.95,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 107,
        "output_tokens": 129,
        "total_tokens": 236,
        "estimated": false
      },
      "score": 0.0,
      "passed_count": 0,
      "failed_count": 4,
      "failed_assertions": [
        "has-baseline-comparison",
        "has-assertion-grading",
        "has-scorecard",
        "has-failure-taxonomy"
      ],
      "output_sha256": "a128ff4dbd7870e287ac651b6cab6b7887e347d6477e5bd8c6f40cd9479539f0",
      "failure": ""
    },
    {
      "case_id": "output-eval-expectation",
      "variant": "with_skill",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 4611.78,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 857,
        "output_tokens": 264,
        "total_tokens": 1121,
        "estimated": false
      },
      "score": 25.0,
      "passed_count": 1,
      "failed_count": 3,
      "failed_assertions": [
        "has-baseline-comparison",
        "has-assertion-grading",
        "has-failure-taxonomy"
      ],
      "output_sha256": "4df98d20e2199e56dae2618b6a64e59d5c96484a4f71ae0ebc674b755cf533c3",
      "failure": ""
    },
    {
      "case_id": "ir-before-packaging",
      "variant": "baseline",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 6772.59,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 104,
        "output_tokens": 661,
        "total_tokens": 765,
        "estimated": false
      },
      "score": 25.0,
      "passed_count": 1,
      "failed_count": 3,
      "failed_assertions": [
        "has-ir-first",
        "has-core-ir-fields",
        "has-semantic-preservation"
      ],
      "output_sha256": "1696075b4f884ff8b478786b5565aff73ad73235936eb39c2ed615c0746861d2",
      "failure": ""
    },
    {
      "case_id": "ir-before-packaging",
      "variant": "with_skill",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 17471.19,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 854,
        "output_tokens": 2132,
        "total_tokens": 2986,
        "estimated": false
      },
      "score": 25.0,
      "passed_count": 1,
      "failed_count": 3,
      "failed_assertions": [
        "has-ir-first",
        "has-core-ir-fields",
        "has-semantic-preservation"
      ],
      "output_sha256": "4f87633e53f360948d6403126d49ad3dabcb1b47aff596918650719b7840abcf",
      "failure": ""
    },
    {
      "case_id": "near-neighbor-boundary",
      "variant": "baseline",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 3091.33,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 105,
        "output_tokens": 93,
        "total_tokens": 198,
        "estimated": false
      },
      "score": 33.33,
      "passed_count": 1,
      "failed_count": 2,
      "failed_assertions": [
        "declines-skill",
        "requires-reuse-signal"
      ],
      "output_sha256": "56c8f48f3b2b9963adad3afdb1b6a2bed6b162067506edee1e0f1acd32668a91",
      "failure": ""
    },
    {
      "case_id": "near-neighbor-boundary",
      "variant": "with_skill",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 4152.85,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 855,
        "output_tokens": 160,
        "total_tokens": 1015,
        "estimated": false
      },
      "score": 33.33,
      "passed_count": 1,
      "failed_count": 2,
      "failed_assertions": [
        "declines-skill",
        "requires-reuse-signal"
      ],
      "output_sha256": "d2d85a2aae5db5c4c7e6e0db5bdd865157a90018afe302b0a703940cc84ba68c",
      "failure": ""
    },
    {
      "case_id": "file-backed-governed-package",
      "variant": "baseline",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 4750.01,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 200,
        "output_tokens": 302,
        "total_tokens": 502,
        "estimated": false
      },
      "score": 60.0,
      "passed_count": 3,
      "failed_count": 2,
      "failed_assertions": [
        "uses-file-backed-evidence",
        "has-trust-and-scorecard"
      ],
      "output_sha256": "d41880b0c2f7e60ee6acb2cb1f4acb9377fd63de72b8e026e6259a962b89af0b",
      "failure": ""
    },
    {
      "case_id": "file-backed-governed-package",
      "variant": "with_skill",
      "status": "pass",
      "execution_mode": "model",
      "model_executed": true,
      "command_executed": true,
      "duration_ms": 9406.25,
      "provider": "deepseek",
      "model": "deepseek-v4-flash",
      "usage": {
        "input_tokens": 950,
        "output_tokens": 660,
        "total_tokens": 1610,
        "estimated": false
      },
      "score": 60.0,
      "passed_count": 3,
      "failed_count": 2,
      "failed_assertions": [
        "has-governance",
        "has-output-contract"
      ],
      "output_sha256": "c670ae2ef81a819a9a0f1b0c046e82ba4c649f56dcbe96a45a6938c7f214fba4",
      "failure": ""
    }
  ],
  "failures": [],
  "artifacts": {
    "json": "reports/output_execution_runs.json",
    "markdown": "reports/output_execution_runs.md"
  }
}