Surface SkillOps curator in Review Studio

This commit is contained in:
yaojingang
2026-06-16 05:25:58 +08:00
parent 0051dad489
commit 36160e9466
5 changed files with 149 additions and 1 deletions
+37
View File
@@ -142,6 +142,8 @@ def render_html(report: dict[str, Any]) -> str:
runtime_permissions_summary = report["data"]["runtime_permissions"].get("summary", {})
atlas_summary = report["data"]["atlas"].get("summary", {})
adoption_summary = report["data"]["adoption_drift"].get("summary", {})
daily_skillops_summary = report["data"]["daily_skillops"].get("summary", {})
weekly_curator_summary = report["data"]["weekly_curator"].get("summary", {})
waiver_summary = report["data"]["review_waivers"].get("summary", {})
world_class_ledger = report["data"].get("world_class_evidence_ledger", {})
world_class_summary = world_class_ledger.get("summary", {})
@@ -291,6 +293,36 @@ def render_html(report: dict[str, Any]) -> str:
["event_count", "adoption_rate", "missed_trigger_count", "bad_output_count", "risk_band"],
"no adoption drift summary",
)
daily_skillops_panel = render_kv_grid(
daily_skillops_summary,
[
"decision",
"proposal_count",
"approval_count",
"pending_review_count",
"release_lock_ready",
"public_world_class_ready",
"writes_source_files",
"auto_patch_enabled",
],
"no daily SkillOps summary",
)
weekly_curator_panel = render_kv_grid(
weekly_curator_summary,
[
"decision",
"week_id",
"daily_report_count",
"unique_opportunity_count",
"ready_for_approval_review_count",
"proposal_review_count",
"top_score",
"release_lock_ready",
"writes_source_files",
"auto_patch_enabled",
],
"no weekly curator summary",
)
waiver_panel = render_kv_grid(
waiver_summary,
["waiver_count", "active_count", "expired_count", "invalid_count", "covered_gate_count"],
@@ -504,6 +536,11 @@ def render_html(report: dict[str, Any]) -> str:
<div class="panel"><h2>漂移信号</h2>{adoption_panel}</div>
</section>
<section class="twocol">
<div class="panel"><h2>日常运维</h2>{daily_skillops_panel}</div>
<div class="panel"><h2>周度队列</h2>{weekly_curator_panel}</div>
</section>
<section id="waivers">
<h2>人工批准</h2>
<p class="muted">warning 可以被有边界地接受,但必须写入 reviewer、理由、范围和到期时间;blocker 与 world-class 完成证据不能通过 waiver 变成通过。</p>
+55
View File
@@ -29,6 +29,33 @@ def load_json(path: Path) -> dict[str, Any]:
return payload if isinstance(payload, dict) else {}
def latest_json_report(directory: Path) -> dict[str, Any]:
if not directory.exists():
return {}
candidates = sorted(path for path in directory.glob("*.json") if path.is_file())
if not candidates:
return {}
payload = load_json(candidates[-1])
if payload:
try:
payload.setdefault("source_path", str(candidates[-1].resolve().relative_to(directory.parent.parent.parent.resolve())))
except ValueError:
payload.setdefault("source_path", str(candidates[-1]))
return payload
def latest_report_path(skill_dir: Path, directory: Path, suffix: str) -> str:
if not directory.exists():
return ""
candidates = sorted(path for path in directory.glob(f"*{suffix}") if path.is_file())
if not candidates:
return ""
try:
return str(candidates[-1].resolve().relative_to(skill_dir.resolve()))
except ValueError:
return str(candidates[-1])
def load_yaml(path: Path) -> dict[str, Any]:
if not path.exists() or yaml is None:
return {}
@@ -107,6 +134,12 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]:
skill_ir_path = find_skill_ir_path(skill_dir)
if skill_ir_path:
rels["skill_ir"] = skill_ir_path
daily_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "daily", ".md")
weekly_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "weekly", ".md")
if daily_md:
rels["daily_skillops"] = daily_md
if weekly_md:
rels["weekly_curator"] = weekly_md
return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()}
@@ -134,6 +167,8 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
"promotion": load_json(reports / "promotion_decisions.json"),
"atlas": load_json(reports / "skill_atlas.json"),
"adoption_drift": load_json(reports / "adoption_drift_report.json"),
"daily_skillops": latest_json_report(reports / "skillops" / "daily"),
"weekly_curator": latest_json_report(reports / "skillops" / "weekly"),
"review_waivers": load_json(reports / "review_waivers.json"),
"review_annotations": load_json(reports / "review_annotations.json"),
"adaptation_proposals": load_json(reports / "adaptation_proposals.json"),
@@ -173,6 +208,8 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
architecture = data["architecture_maintainability"].get("summary", {})
atlas = data["atlas"].get("summary", {})
adoption = data["adoption_drift"].get("summary", {})
daily_skillops = data["daily_skillops"].get("summary", {})
weekly_curator = data["weekly_curator"].get("summary", {})
waivers = data["review_waivers"].get("summary", {})
annotations = data["review_annotations"].get("summary", {})
intake = data["world_class_evidence_intake"].get("summary", {})
@@ -277,6 +314,24 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
"value": str(adoption.get("risk_band", "n/a")),
"detail": f"{adoption.get('event_count', 0)} metadata events; {adoption.get('missed_trigger_count', 0)} missed triggers",
},
{
"label": "Daily Ops",
"value": str(daily_skillops.get("proposal_count", 0)),
"detail": (
f"{daily_skillops.get('decision', 'missing')}; "
f"approval {daily_skillops.get('approval_count', 0)}; "
f"release lock {str(daily_skillops.get('release_lock_ready', False)).lower()}"
),
},
{
"label": "Weekly Queue",
"value": str(weekly_curator.get("unique_opportunity_count", 0)),
"detail": (
f"{weekly_curator.get('decision', 'missing')}; "
f"ready {weekly_curator.get('ready_for_approval_review_count', 0)}; "
f"top score {weekly_curator.get('top_score', 0)}"
),
},
{
"label": "Waivers",
"value": str(waivers.get("active_count", 0)),
+6
View File
@@ -27,7 +27,9 @@ LABELS = {
"command_executed_count": "命令执行",
"compatibility_pass_count": "兼容通过",
"covered_gate_count": "覆盖 Gate",
"daily_report_count": "日报数",
"declared_bump": "声明版本",
"decision": "决策",
"delta": "增益",
"event_count": "事件数",
"failure_count": "失败数",
@@ -73,8 +75,11 @@ LABELS = {
"public_claim_blocker_count": "声明阻断",
"public_claim_ready": "可公开声明",
"public_world_class_ready": "世界级",
"proposal_count": "提案数",
"proposal_review_count": "提案复核",
"reproducibility_ready": "本地复现",
"recorded_fixture_count": "记录样本",
"ready_for_approval_review_count": "待批准复核",
"ready_for_external_collection": "可收证据",
"ready_for_ledger_review": "可审台账",
"recommended_pr_count": "建议 PR",
@@ -98,6 +103,7 @@ LABELS = {
"token_estimated_count": "估算 Token",
"token_observed_count": "真实 Token",
"trust_level": "信任级别",
"unique_opportunity_count": "唯一机会",
"variant_run_count": "运行数",
"valid_submission_count": "有效提交",
"version": "版本",
+35 -1
View File
@@ -438,6 +438,10 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
adoption = data["adoption_drift"]
adoption_summary = adoption.get("summary", {})
daily_skillops = data.get("daily_skillops", {})
weekly_curator = data.get("weekly_curator", {})
daily_summary = daily_skillops.get("summary", {}) if isinstance(daily_skillops, dict) else {}
weekly_summary = weekly_curator.get("summary", {}) if isinstance(weekly_curator, dict) else {}
if not adoption:
adoption_status = "warn"
adoption_detail = "adoption drift report is missing; real usage impact is unknown"
@@ -454,13 +458,43 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
f"bad-output {adoption_summary.get('bad_output_count', 0)}; "
f"risk {risk_band}"
)
skillops_parts = []
skillops_blocked = False
for label, report, summary in [
("daily", daily_skillops, daily_summary),
("weekly", weekly_curator, weekly_summary),
]:
if not report:
continue
writes_source = bool(summary.get("writes_source_files") or report.get("writes_source_files"))
auto_patch = bool(summary.get("auto_patch_enabled") or report.get("auto_patch_enabled"))
failure_count = int(summary.get("failure_count", 0) or 0)
if writes_source or auto_patch or failure_count:
skillops_blocked = True
if label == "daily":
skillops_parts.append(
f"daily proposals {summary.get('proposal_count', 0)}; "
f"daily decision {summary.get('decision', report.get('decision', 'unknown'))}; "
f"daily release lock {str(summary.get('release_lock_ready', False)).lower()}"
)
else:
skillops_parts.append(
f"weekly queue {summary.get('unique_opportunity_count', 0)} unique; "
f"weekly ready {summary.get('ready_for_approval_review_count', 0)}; "
f"weekly top {summary.get('top_score', 0)}; "
f"weekly release lock {str(summary.get('release_lock_ready', False)).lower()}"
)
if skillops_blocked:
adoption_status = "block"
if skillops_parts:
adoption_detail = adoption_detail + "; " + "; ".join(skillops_parts)
gates.append(
gate(
"operations-loop",
"运营回路",
adoption_status,
adoption_detail,
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly",
_report_link(output_html, skill_dir, "reports/adoption_drift_report.md"),
)
)
+16
View File
@@ -370,7 +370,11 @@ def main() -> None:
assert operations_gate["status"] == "pass", operations_gate
assert "metadata events" in operations_gate["detail"], operations_gate
assert "risk low" in operations_gate["detail"], operations_gate
assert "daily proposals" in operations_gate["detail"], operations_gate
assert "weekly queue" in operations_gate["detail"], operations_gate
assert "reports/adoption_drift_report.json" in operations_gate["evidence"], operations_gate
assert "reports/skillops/daily" in operations_gate["evidence"], operations_gate
assert "reports/skillops/weekly" in operations_gate["evidence"], operations_gate
waivers_gate = next(item for item in payload["gates"] if item["key"] == "review-waivers")
assert waivers_gate["status"] == "warn", waivers_gate
assert "1 warning gates still need reviewer decision" in waivers_gate["detail"], waivers_gate
@@ -408,6 +412,8 @@ def main() -> None:
assert full_payload["evidence_paths"]["benchmark_reproducibility"] == "reports/benchmark_reproducibility.md", full_payload["evidence_paths"]
if (ROOT / "reports" / "skill_os2_coverage.md").exists():
assert full_payload["evidence_paths"]["skill_os2_coverage"] == "reports/skill_os2_coverage.md", full_payload["evidence_paths"]
assert full_payload["evidence_paths"]["daily_skillops"].startswith("reports/skillops/daily/"), full_payload["evidence_paths"]
assert full_payload["evidence_paths"]["weekly_curator"].startswith("reports/skillops/weekly/"), full_payload["evidence_paths"]
assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"]
assert full_payload["evidence_paths"]["adaptation_proposals"] == "reports/adaptation_proposals.md", full_payload["evidence_paths"]
assert full_payload["evidence_paths"]["adaptation_approval_ledger"] == "reports/adaptation_approval_ledger.json", full_payload["evidence_paths"]
@@ -451,6 +457,13 @@ def main() -> None:
assert all(not item["answer_key_visible"] for item in output_review_checklist), output_review_checklist
assert output_review_checklist[0]["commands"]["adjudicate"] == "python3 scripts/yao.py output-review", output_review_checklist[0]
assert full_payload["data"]["review_annotations"]["summary"]["annotation_count"] == 0, full_payload["data"]["review_annotations"]
daily_skillops_summary = full_payload["data"]["daily_skillops"]["summary"]
assert daily_skillops_summary["writes_source_files"] is False, daily_skillops_summary
assert daily_skillops_summary["auto_patch_enabled"] is False, daily_skillops_summary
weekly_curator_summary = full_payload["data"]["weekly_curator"]["summary"]
assert weekly_curator_summary["unique_opportunity_count"] >= 1, weekly_curator_summary
assert weekly_curator_summary["writes_source_files"] is False, weekly_curator_summary
assert weekly_curator_summary["auto_patch_enabled"] is False, weekly_curator_summary
assert full_payload["data"]["adaptation_proposals"]["report_contract"]["contract"] == "adaptation-proposals", full_payload["data"]["adaptation_proposals"]
assert full_payload["data"]["adaptation_proposals"]["proposal_count"] == full_payload["data"]["adaptation_proposals"]["summary"]["proposal_count"], full_payload["data"]["adaptation_proposals"]
assert full_payload["data"]["adaptation_approval_ledger"]["report_contract"]["contract"] == "adaptation-approval-ledger", full_payload["data"]["adaptation_approval_ledger"]
@@ -651,6 +664,9 @@ def main() -> None:
assert "reports/world_class_evidence_intake.md" in html, html
assert "reports/world_class_operator_runbook.html" in html, html
assert "reports/world_class_claim_guard.md" in html, html
assert "日常运维" in html, html
assert "周度队列" in html, html
assert "Weekly Queue" in html, html
assert "英文完成断言、true 状态声明或中文完成态" in html, html
assert "world-evidence-grid" in html, html
assert "Provider Holdout" in html, html