fix: surface pending output review in studio
This commit is contained in:
@@ -352,9 +352,9 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = {
|
||||
"verification": "python3 scripts/build_confusion_matrix.py",
|
||||
},
|
||||
"output-lab": {
|
||||
"summary": "补足 output eval 的 case 数、file-backed case、near-neighbor case 和 boundary case。",
|
||||
"why": "没有输出质量证据时,Skill 只能证明会触发,不能证明输出真的更好。",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md",
|
||||
"summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。",
|
||||
"why": "没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_adjudication.md",
|
||||
"source_paths": [
|
||||
{"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ["case_id"]},
|
||||
{"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]},
|
||||
|
||||
@@ -148,8 +148,11 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
|
||||
execution_recorded_count = int(output_execution_summary.get("recorded_fixture_count", 0) or 0)
|
||||
review_pair_count = int(output_review_summary.get("pair_count", 0) or 0)
|
||||
review_judgment_count = int(output_review_summary.get("judgment_count", 0) or 0)
|
||||
review_pending_count = int(output_review_summary.get("pending_count", 0) or 0)
|
||||
review_invalid_count = int(output_review_summary.get("invalid_decision_count", 0) or 0)
|
||||
blind_missing = maturity in {"production", "library", "governed"} and (not output_blind or blind_pair_count < case_count)
|
||||
review_missing = maturity in {"production", "library", "governed"} and case_count > 0 and not output_review
|
||||
review_pending = maturity in {"production", "library", "governed"} and bool(output_review) and review_pending_count > 0
|
||||
execution_failed = bool(output_execution) and (not output_execution.get("ok", True) or int(output_execution_summary.get("failure_count", 0) or 0) > 0)
|
||||
review_invalid = bool(output_review) and (not output_review.get("ok", True) or review_invalid_count > 0)
|
||||
output_blocked = (
|
||||
@@ -159,7 +162,7 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
|
||||
or execution_failed
|
||||
or review_invalid
|
||||
)
|
||||
output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing
|
||||
output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing or review_missing or review_pending
|
||||
if not output:
|
||||
output_status = "warn"
|
||||
output_detail = "output eval scorecard is missing; generate it before production review"
|
||||
@@ -176,6 +179,8 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
|
||||
else ""
|
||||
)
|
||||
+ (f"; reviewed {review_judgment_count}/{review_pair_count}" if output_review else "")
|
||||
+ (f"; review pending {review_pending_count}" if review_pending else "")
|
||||
+ ("; review adjudication missing" if review_missing else "")
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
@@ -509,4 +514,3 @@ def weighted_score(gates: list[dict[str, str]]) -> int:
|
||||
elif item["status"] == "warn":
|
||||
earned += weight * 0.6
|
||||
return int(round(earned / total * 100)) if total else 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user