yaojingang
86 lines
2.1 KiB
JSON
86 lines
2.1 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://yao.example/output-eval-case.schema.json",
|
|
"title": "Yao Output Eval Case",
|
|
"description": "Output-eval case for comparing without-skill and with-skill outputs, with optional execution metadata.",
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["id", "prompt", "baseline_output", "with_skill_output", "assertions"],
|
|
"properties": {
|
|
"id": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"prompt": {
|
|
"type": "string",
|
|
"minLength": 1
|
|
},
|
|
"input_files": {
|
|
"type": "array",
|
|
"items": {"type": "string"}
|
|
},
|
|
"baseline_output": {
|
|
"type": "string"
|
|
},
|
|
"with_skill_output": {
|
|
"type": "string"
|
|
},
|
|
"assertions": {
|
|
"type": "array",
|
|
"minItems": 1,
|
|
"items": {
|
|
"type": "object",
|
|
"additionalProperties": false,
|
|
"required": ["id", "description"],
|
|
"properties": {
|
|
"id": {"type": "string", "minLength": 1},
|
|
"description": {"type": "string", "minLength": 1},
|
|
"required": {
|
|
"type": "array",
|
|
"items": {"type": "string"}
|
|
},
|
|
"forbidden": {
|
|
"type": "array",
|
|
"items": {"type": "string"}
|
|
},
|
|
"weight": {
|
|
"type": "number",
|
|
"minimum": 0
|
|
},
|
|
"failure_type": {
|
|
"type": "string"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"human_review": {
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
},
|
|
"execution": {
|
|
"type": "object",
|
|
"description": "Optional case-level execution metadata for fixture, command, or provider-backed output runs.",
|
|
"additionalProperties": true,
|
|
"properties": {
|
|
"mode": {
|
|
"type": "string",
|
|
"enum": ["recorded_fixture", "command", "model"]
|
|
},
|
|
"provider": {
|
|
"type": "string"
|
|
},
|
|
"model": {
|
|
"type": "string"
|
|
},
|
|
"holdout": {
|
|
"type": "boolean"
|
|
}
|
|
}
|
|
},
|
|
"metadata": {
|
|
"type": "object",
|
|
"additionalProperties": true
|
|
}
|
|
}
|
|
}
|