yaojingang
591 lines
17 KiB
JSON
591 lines
17 KiB
JSON
{
|
|
"summary": {
|
|
"total_cases": 13,
|
|
"correct_cases": 13,
|
|
"accuracy": 1.0,
|
|
"ambiguous_case_count": 0,
|
|
"margin_warning_threshold": 0.08,
|
|
"misroute_count": 0,
|
|
"no_route_accuracy": 1.0
|
|
},
|
|
"route_stats": {
|
|
"yao-meta-skill": {
|
|
"expected_count": 3,
|
|
"predicted_count": 3,
|
|
"correct_count": 3,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": 0.611
|
|
},
|
|
"team-frontend-review": {
|
|
"expected_count": 3,
|
|
"predicted_count": 3,
|
|
"correct_count": 3,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": 0.805
|
|
},
|
|
"governed-incident-command": {
|
|
"expected_count": 3,
|
|
"predicted_count": 3,
|
|
"correct_count": 3,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": 0.659
|
|
},
|
|
"no_route": {
|
|
"expected_count": 4,
|
|
"predicted_count": 4,
|
|
"correct_count": 4,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": null
|
|
}
|
|
},
|
|
"confusion_matrix": {
|
|
"yao-meta-skill": {
|
|
"yao-meta-skill": 3,
|
|
"team-frontend-review": 0,
|
|
"governed-incident-command": 0,
|
|
"no_route": 0
|
|
},
|
|
"team-frontend-review": {
|
|
"yao-meta-skill": 0,
|
|
"team-frontend-review": 3,
|
|
"governed-incident-command": 0,
|
|
"no_route": 0
|
|
},
|
|
"governed-incident-command": {
|
|
"yao-meta-skill": 0,
|
|
"team-frontend-review": 0,
|
|
"governed-incident-command": 3,
|
|
"no_route": 0
|
|
},
|
|
"no_route": {
|
|
"yao-meta-skill": 0,
|
|
"team-frontend-review": 0,
|
|
"governed-incident-command": 0,
|
|
"no_route": 4
|
|
}
|
|
},
|
|
"ambiguous_cases": [],
|
|
"misroutes": [],
|
|
"cases": [
|
|
{
|
|
"family": "meta_create",
|
|
"prompt": "Create a reusable skill from this workflow and add evals before sharing it with the team.",
|
|
"expected_route": "yao-meta-skill",
|
|
"predicted_route": "yao-meta-skill",
|
|
"correct": true,
|
|
"margin": 0.741,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.741,
|
|
"passed_threshold": true,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package",
|
|
"team_operationalize",
|
|
"transform_workflow"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.01,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.01,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "meta_iterate",
|
|
"prompt": "Refactor this existing skill description, tighten the trigger boundary, and package it for reuse.",
|
|
"expected_route": "yao-meta-skill",
|
|
"predicted_route": "yao-meta-skill",
|
|
"correct": true,
|
|
"margin": 0.535,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.535,
|
|
"passed_threshold": true,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.011,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.01,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "meta_operationalize",
|
|
"prompt": "Turn these prompts, notes, and transcripts into a maintained skill package with references and scripts.",
|
|
"expected_route": "yao-meta-skill",
|
|
"predicted_route": "yao-meta-skill",
|
|
"correct": true,
|
|
"margin": 0.558,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.558,
|
|
"passed_threshold": true,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.269,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.011,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "frontend_premerge",
|
|
"prompt": "Run a pre-merge frontend review for keyboard navigation issues and unsafe rendering.",
|
|
"expected_route": "team-frontend-review",
|
|
"predicted_route": "team-frontend-review",
|
|
"correct": true,
|
|
"margin": 0.893,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.893,
|
|
"passed_threshold": true,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"accessibility",
|
|
"frontend_scope",
|
|
"review_frontend",
|
|
"ui_security",
|
|
"ux_regression"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.018,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.012,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "frontend_states",
|
|
"prompt": "Run a pre-merge frontend review on this React diff for missing empty states, form handling, and UX regressions.",
|
|
"expected_route": "team-frontend-review",
|
|
"predicted_route": "team-frontend-review",
|
|
"correct": true,
|
|
"margin": 0.704,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.704,
|
|
"passed_threshold": true,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"frontend_scope",
|
|
"review_frontend",
|
|
"ui_security",
|
|
"ux_regression"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.013,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.008,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "frontend_a11y",
|
|
"prompt": "Review React components for a11y regressions, unsafe rendering, and UI behavior before merge.",
|
|
"expected_route": "team-frontend-review",
|
|
"predicted_route": "team-frontend-review",
|
|
"correct": true,
|
|
"margin": 0.819,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.819,
|
|
"passed_threshold": true,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"accessibility",
|
|
"frontend_scope",
|
|
"review_frontend",
|
|
"ui_security"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.012,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.006,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "incident_packet",
|
|
"prompt": "Build a governed incident command packet from alerts, responder notes, incident communication needs, and severity findings.",
|
|
"expected_route": "governed-incident-command",
|
|
"predicted_route": "governed-incident-command",
|
|
"correct": true,
|
|
"margin": 1.0,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 1.0,
|
|
"passed_threshold": true,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.021,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.011,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "incident_severity",
|
|
"prompt": "Run severity assessment and package incident communication from this outage review.",
|
|
"expected_route": "governed-incident-command",
|
|
"predicted_route": "governed-incident-command",
|
|
"correct": true,
|
|
"margin": 0.805,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.805,
|
|
"passed_threshold": true,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"severity_assessment"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.189,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.022,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "incident_governed_workflow",
|
|
"prompt": "Standardize this incident review into a reusable governed workflow with owner actions.",
|
|
"expected_route": "governed-incident-command",
|
|
"predicted_route": "governed-incident-command",
|
|
"correct": true,
|
|
"margin": 0.172,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.62,
|
|
"passed_threshold": true,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.448,
|
|
"passed_threshold": true,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"team_operationalize",
|
|
"transform_workflow"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.013,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "no_route_explain",
|
|
"prompt": "Explain keyboard accessibility basics in plain English.",
|
|
"expected_route": "no_route",
|
|
"predicted_route": "no_route",
|
|
"correct": true,
|
|
"margin": null,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.195,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"accessibility"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.0,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
]
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.0,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "no_route_build",
|
|
"prompt": "Build a React form page with loading and empty states.",
|
|
"expected_route": "no_route",
|
|
"predicted_route": "no_route",
|
|
"correct": true,
|
|
"margin": null,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.016,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.008,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.0,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [
|
|
"frontend_scope"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"build_ui"
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "no_route_single_update",
|
|
"prompt": "Write one short outage update for customers from these notes.",
|
|
"expected_route": "no_route",
|
|
"predicted_route": "no_route",
|
|
"correct": true,
|
|
"margin": null,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.024,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.008,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.0,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"family": "no_route_document_only",
|
|
"prompt": "Summarize this workflow and keep it as documentation only.",
|
|
"expected_route": "no_route",
|
|
"predicted_route": "no_route",
|
|
"correct": true,
|
|
"margin": null,
|
|
"ambiguous": false,
|
|
"ranked_routes": [
|
|
{
|
|
"name": "governed-incident-command",
|
|
"score": 0.01,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"score": 0.009,
|
|
"passed_threshold": false,
|
|
"threshold": 0.48,
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"score": 0.0,
|
|
"passed_threshold": false,
|
|
"threshold": 0.33,
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|