Surface SkillOps curator in Review Studio
This commit is contained in:
@@ -142,6 +142,8 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
runtime_permissions_summary = report["data"]["runtime_permissions"].get("summary", {})
|
||||
atlas_summary = report["data"]["atlas"].get("summary", {})
|
||||
adoption_summary = report["data"]["adoption_drift"].get("summary", {})
|
||||
daily_skillops_summary = report["data"]["daily_skillops"].get("summary", {})
|
||||
weekly_curator_summary = report["data"]["weekly_curator"].get("summary", {})
|
||||
waiver_summary = report["data"]["review_waivers"].get("summary", {})
|
||||
world_class_ledger = report["data"].get("world_class_evidence_ledger", {})
|
||||
world_class_summary = world_class_ledger.get("summary", {})
|
||||
@@ -291,6 +293,36 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
["event_count", "adoption_rate", "missed_trigger_count", "bad_output_count", "risk_band"],
|
||||
"no adoption drift summary",
|
||||
)
|
||||
daily_skillops_panel = render_kv_grid(
|
||||
daily_skillops_summary,
|
||||
[
|
||||
"decision",
|
||||
"proposal_count",
|
||||
"approval_count",
|
||||
"pending_review_count",
|
||||
"release_lock_ready",
|
||||
"public_world_class_ready",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
],
|
||||
"no daily SkillOps summary",
|
||||
)
|
||||
weekly_curator_panel = render_kv_grid(
|
||||
weekly_curator_summary,
|
||||
[
|
||||
"decision",
|
||||
"week_id",
|
||||
"daily_report_count",
|
||||
"unique_opportunity_count",
|
||||
"ready_for_approval_review_count",
|
||||
"proposal_review_count",
|
||||
"top_score",
|
||||
"release_lock_ready",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
],
|
||||
"no weekly curator summary",
|
||||
)
|
||||
waiver_panel = render_kv_grid(
|
||||
waiver_summary,
|
||||
["waiver_count", "active_count", "expired_count", "invalid_count", "covered_gate_count"],
|
||||
@@ -504,6 +536,11 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
<div class="panel"><h2>漂移信号</h2>{adoption_panel}</div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>日常运维</h2>{daily_skillops_panel}</div>
|
||||
<div class="panel"><h2>周度队列</h2>{weekly_curator_panel}</div>
|
||||
</section>
|
||||
|
||||
<section id="waivers">
|
||||
<h2>人工批准</h2>
|
||||
<p class="muted">warning 可以被有边界地接受,但必须写入 reviewer、理由、范围和到期时间;blocker 与 world-class 完成证据不能通过 waiver 变成通过。</p>
|
||||
|
||||
@@ -29,6 +29,33 @@ def load_json(path: Path) -> dict[str, Any]:
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def latest_json_report(directory: Path) -> dict[str, Any]:
|
||||
if not directory.exists():
|
||||
return {}
|
||||
candidates = sorted(path for path in directory.glob("*.json") if path.is_file())
|
||||
if not candidates:
|
||||
return {}
|
||||
payload = load_json(candidates[-1])
|
||||
if payload:
|
||||
try:
|
||||
payload.setdefault("source_path", str(candidates[-1].resolve().relative_to(directory.parent.parent.parent.resolve())))
|
||||
except ValueError:
|
||||
payload.setdefault("source_path", str(candidates[-1]))
|
||||
return payload
|
||||
|
||||
|
||||
def latest_report_path(skill_dir: Path, directory: Path, suffix: str) -> str:
|
||||
if not directory.exists():
|
||||
return ""
|
||||
candidates = sorted(path for path in directory.glob(f"*{suffix}") if path.is_file())
|
||||
if not candidates:
|
||||
return ""
|
||||
try:
|
||||
return str(candidates[-1].resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
return str(candidates[-1])
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict[str, Any]:
|
||||
if not path.exists() or yaml is None:
|
||||
return {}
|
||||
@@ -107,6 +134,12 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]:
|
||||
skill_ir_path = find_skill_ir_path(skill_dir)
|
||||
if skill_ir_path:
|
||||
rels["skill_ir"] = skill_ir_path
|
||||
daily_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "daily", ".md")
|
||||
weekly_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "weekly", ".md")
|
||||
if daily_md:
|
||||
rels["daily_skillops"] = daily_md
|
||||
if weekly_md:
|
||||
rels["weekly_curator"] = weekly_md
|
||||
return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()}
|
||||
|
||||
|
||||
@@ -134,6 +167,8 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
"promotion": load_json(reports / "promotion_decisions.json"),
|
||||
"atlas": load_json(reports / "skill_atlas.json"),
|
||||
"adoption_drift": load_json(reports / "adoption_drift_report.json"),
|
||||
"daily_skillops": latest_json_report(reports / "skillops" / "daily"),
|
||||
"weekly_curator": latest_json_report(reports / "skillops" / "weekly"),
|
||||
"review_waivers": load_json(reports / "review_waivers.json"),
|
||||
"review_annotations": load_json(reports / "review_annotations.json"),
|
||||
"adaptation_proposals": load_json(reports / "adaptation_proposals.json"),
|
||||
@@ -173,6 +208,8 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
architecture = data["architecture_maintainability"].get("summary", {})
|
||||
atlas = data["atlas"].get("summary", {})
|
||||
adoption = data["adoption_drift"].get("summary", {})
|
||||
daily_skillops = data["daily_skillops"].get("summary", {})
|
||||
weekly_curator = data["weekly_curator"].get("summary", {})
|
||||
waivers = data["review_waivers"].get("summary", {})
|
||||
annotations = data["review_annotations"].get("summary", {})
|
||||
intake = data["world_class_evidence_intake"].get("summary", {})
|
||||
@@ -277,6 +314,24 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
"value": str(adoption.get("risk_band", "n/a")),
|
||||
"detail": f"{adoption.get('event_count', 0)} metadata events; {adoption.get('missed_trigger_count', 0)} missed triggers",
|
||||
},
|
||||
{
|
||||
"label": "Daily Ops",
|
||||
"value": str(daily_skillops.get("proposal_count", 0)),
|
||||
"detail": (
|
||||
f"{daily_skillops.get('decision', 'missing')}; "
|
||||
f"approval {daily_skillops.get('approval_count', 0)}; "
|
||||
f"release lock {str(daily_skillops.get('release_lock_ready', False)).lower()}"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Weekly Queue",
|
||||
"value": str(weekly_curator.get("unique_opportunity_count", 0)),
|
||||
"detail": (
|
||||
f"{weekly_curator.get('decision', 'missing')}; "
|
||||
f"ready {weekly_curator.get('ready_for_approval_review_count', 0)}; "
|
||||
f"top score {weekly_curator.get('top_score', 0)}"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Waivers",
|
||||
"value": str(waivers.get("active_count", 0)),
|
||||
|
||||
@@ -27,7 +27,9 @@ LABELS = {
|
||||
"command_executed_count": "命令执行",
|
||||
"compatibility_pass_count": "兼容通过",
|
||||
"covered_gate_count": "覆盖 Gate",
|
||||
"daily_report_count": "日报数",
|
||||
"declared_bump": "声明版本",
|
||||
"decision": "决策",
|
||||
"delta": "增益",
|
||||
"event_count": "事件数",
|
||||
"failure_count": "失败数",
|
||||
@@ -73,8 +75,11 @@ LABELS = {
|
||||
"public_claim_blocker_count": "声明阻断",
|
||||
"public_claim_ready": "可公开声明",
|
||||
"public_world_class_ready": "世界级",
|
||||
"proposal_count": "提案数",
|
||||
"proposal_review_count": "提案复核",
|
||||
"reproducibility_ready": "本地复现",
|
||||
"recorded_fixture_count": "记录样本",
|
||||
"ready_for_approval_review_count": "待批准复核",
|
||||
"ready_for_external_collection": "可收证据",
|
||||
"ready_for_ledger_review": "可审台账",
|
||||
"recommended_pr_count": "建议 PR",
|
||||
@@ -98,6 +103,7 @@ LABELS = {
|
||||
"token_estimated_count": "估算 Token",
|
||||
"token_observed_count": "真实 Token",
|
||||
"trust_level": "信任级别",
|
||||
"unique_opportunity_count": "唯一机会",
|
||||
"variant_run_count": "运行数",
|
||||
"valid_submission_count": "有效提交",
|
||||
"version": "版本",
|
||||
|
||||
@@ -438,6 +438,10 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
|
||||
|
||||
adoption = data["adoption_drift"]
|
||||
adoption_summary = adoption.get("summary", {})
|
||||
daily_skillops = data.get("daily_skillops", {})
|
||||
weekly_curator = data.get("weekly_curator", {})
|
||||
daily_summary = daily_skillops.get("summary", {}) if isinstance(daily_skillops, dict) else {}
|
||||
weekly_summary = weekly_curator.get("summary", {}) if isinstance(weekly_curator, dict) else {}
|
||||
if not adoption:
|
||||
adoption_status = "warn"
|
||||
adoption_detail = "adoption drift report is missing; real usage impact is unknown"
|
||||
@@ -454,13 +458,43 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An
|
||||
f"bad-output {adoption_summary.get('bad_output_count', 0)}; "
|
||||
f"risk {risk_band}"
|
||||
)
|
||||
skillops_parts = []
|
||||
skillops_blocked = False
|
||||
for label, report, summary in [
|
||||
("daily", daily_skillops, daily_summary),
|
||||
("weekly", weekly_curator, weekly_summary),
|
||||
]:
|
||||
if not report:
|
||||
continue
|
||||
writes_source = bool(summary.get("writes_source_files") or report.get("writes_source_files"))
|
||||
auto_patch = bool(summary.get("auto_patch_enabled") or report.get("auto_patch_enabled"))
|
||||
failure_count = int(summary.get("failure_count", 0) or 0)
|
||||
if writes_source or auto_patch or failure_count:
|
||||
skillops_blocked = True
|
||||
if label == "daily":
|
||||
skillops_parts.append(
|
||||
f"daily proposals {summary.get('proposal_count', 0)}; "
|
||||
f"daily decision {summary.get('decision', report.get('decision', 'unknown'))}; "
|
||||
f"daily release lock {str(summary.get('release_lock_ready', False)).lower()}"
|
||||
)
|
||||
else:
|
||||
skillops_parts.append(
|
||||
f"weekly queue {summary.get('unique_opportunity_count', 0)} unique; "
|
||||
f"weekly ready {summary.get('ready_for_approval_review_count', 0)}; "
|
||||
f"weekly top {summary.get('top_score', 0)}; "
|
||||
f"weekly release lock {str(summary.get('release_lock_ready', False)).lower()}"
|
||||
)
|
||||
if skillops_blocked:
|
||||
adoption_status = "block"
|
||||
if skillops_parts:
|
||||
adoption_detail = adoption_detail + "; " + "; ".join(skillops_parts)
|
||||
gates.append(
|
||||
gate(
|
||||
"operations-loop",
|
||||
"运营回路",
|
||||
adoption_status,
|
||||
adoption_detail,
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly",
|
||||
_report_link(output_html, skill_dir, "reports/adoption_drift_report.md"),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -370,7 +370,11 @@ def main() -> None:
|
||||
assert operations_gate["status"] == "pass", operations_gate
|
||||
assert "metadata events" in operations_gate["detail"], operations_gate
|
||||
assert "risk low" in operations_gate["detail"], operations_gate
|
||||
assert "daily proposals" in operations_gate["detail"], operations_gate
|
||||
assert "weekly queue" in operations_gate["detail"], operations_gate
|
||||
assert "reports/adoption_drift_report.json" in operations_gate["evidence"], operations_gate
|
||||
assert "reports/skillops/daily" in operations_gate["evidence"], operations_gate
|
||||
assert "reports/skillops/weekly" in operations_gate["evidence"], operations_gate
|
||||
waivers_gate = next(item for item in payload["gates"] if item["key"] == "review-waivers")
|
||||
assert waivers_gate["status"] == "warn", waivers_gate
|
||||
assert "1 warning gates still need reviewer decision" in waivers_gate["detail"], waivers_gate
|
||||
@@ -408,6 +412,8 @@ def main() -> None:
|
||||
assert full_payload["evidence_paths"]["benchmark_reproducibility"] == "reports/benchmark_reproducibility.md", full_payload["evidence_paths"]
|
||||
if (ROOT / "reports" / "skill_os2_coverage.md").exists():
|
||||
assert full_payload["evidence_paths"]["skill_os2_coverage"] == "reports/skill_os2_coverage.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["daily_skillops"].startswith("reports/skillops/daily/"), full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["weekly_curator"].startswith("reports/skillops/weekly/"), full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["adaptation_proposals"] == "reports/adaptation_proposals.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["adaptation_approval_ledger"] == "reports/adaptation_approval_ledger.json", full_payload["evidence_paths"]
|
||||
@@ -451,6 +457,13 @@ def main() -> None:
|
||||
assert all(not item["answer_key_visible"] for item in output_review_checklist), output_review_checklist
|
||||
assert output_review_checklist[0]["commands"]["adjudicate"] == "python3 scripts/yao.py output-review", output_review_checklist[0]
|
||||
assert full_payload["data"]["review_annotations"]["summary"]["annotation_count"] == 0, full_payload["data"]["review_annotations"]
|
||||
daily_skillops_summary = full_payload["data"]["daily_skillops"]["summary"]
|
||||
assert daily_skillops_summary["writes_source_files"] is False, daily_skillops_summary
|
||||
assert daily_skillops_summary["auto_patch_enabled"] is False, daily_skillops_summary
|
||||
weekly_curator_summary = full_payload["data"]["weekly_curator"]["summary"]
|
||||
assert weekly_curator_summary["unique_opportunity_count"] >= 1, weekly_curator_summary
|
||||
assert weekly_curator_summary["writes_source_files"] is False, weekly_curator_summary
|
||||
assert weekly_curator_summary["auto_patch_enabled"] is False, weekly_curator_summary
|
||||
assert full_payload["data"]["adaptation_proposals"]["report_contract"]["contract"] == "adaptation-proposals", full_payload["data"]["adaptation_proposals"]
|
||||
assert full_payload["data"]["adaptation_proposals"]["proposal_count"] == full_payload["data"]["adaptation_proposals"]["summary"]["proposal_count"], full_payload["data"]["adaptation_proposals"]
|
||||
assert full_payload["data"]["adaptation_approval_ledger"]["report_contract"]["contract"] == "adaptation-approval-ledger", full_payload["data"]["adaptation_approval_ledger"]
|
||||
@@ -651,6 +664,9 @@ def main() -> None:
|
||||
assert "reports/world_class_evidence_intake.md" in html, html
|
||||
assert "reports/world_class_operator_runbook.html" in html, html
|
||||
assert "reports/world_class_claim_guard.md" in html, html
|
||||
assert "日常运维" in html, html
|
||||
assert "周度队列" in html, html
|
||||
assert "Weekly Queue" in html, html
|
||||
assert "英文完成断言、true 状态声明或中文完成态" in html, html
|
||||
assert "world-evidence-grid" in html, html
|
||||
assert "Provider Holdout" in html, html
|
||||
|
||||
Reference in New Issue
Block a user