frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: expose source checks in world-class ledger

Browse Source
This commit is contained in:
yaojingang
2026-06-14 13:49:13 +08:00
parent a619a17ea2
commit 2abc2252e0
3 changed files with 49 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/render_world_class_evidence_ledger.py
+33 -2
View File
@@ -7,6 +7,7 @@ from typing import Any
from render_world_class_evidence_plan import build_plan
from world_class_evidence_contract import load_json, load_json_with_status, rel_path, validate_payload
from world_class_source_checks import build_source_checklist, summarize_source_checklist
ROOT = Path(__file__).resolve().parent.parent
@@ -129,7 +130,9 @@ def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Pat
def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]:
state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir)
submission = submission_state(skill_dir, task, submissions_dir)
source_accepted = bool(state.get("accepted"))
source_checklist = build_source_checklist([{"key": task["key"], "observed_state": state}])
source_summary = summarize_source_checklist(source_checklist)
source_accepted = bool(source_checklist) and source_summary["source_blocked_count"] == 0
accepted = source_accepted and submission.get("status") == "submitted"
return {
"key": task["key"],
@@ -146,6 +149,8 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) ->
"evidence_artifacts": task["evidence_artifacts"],
"privacy_contract": task["privacy_contract"],
"observed_state": state,
"source_checklist": source_checklist,
**source_summary,
"submission_state": submission,
"anti_overclaim": {
"planned_work_counts_as_evidence": False,
@@ -162,6 +167,8 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
evidence_requirements = plan.get("evidence_requirements") or plan.get("tasks", [])
entries = [build_entry(skill_dir, task, submissions_dir) for task in evidence_requirements]
source_rows = [row for entry in entries for row in entry.get("source_checklist", [])]
source_summary = summarize_source_checklist(source_rows)
source_accepted_count = sum(1 for entry in entries if entry.get("source_accepted") is True)
accepted_count = sum(1 for entry in entries if entry["status"] == "accepted")
pending_count = len(entries) - accepted_count
@@ -194,6 +201,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
"submitted_entry_count": submitted_entry_count,
"missing_submission_count": missing_submission_count,
"invalid_submission_count": invalid_submission_count,
**source_summary,
"submitted_but_pending_count": submitted_but_pending_count,
"source_accepted_without_valid_submission_count": source_accepted_without_valid_submission_count,
"overclaim_guard_active": True,
@@ -233,6 +241,8 @@ def render_markdown(ledger: dict[str, Any]) -> str:
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
f"- entries: `{summary['ledger_entry_count']}`",
f"- source accepted: `{summary.get('source_accepted_count', 0)}`",
f"- source checks: `{summary.get('source_pass_count', 0)}` pass / `{summary.get('source_check_count', 0)}` total",
f"- source blocked: `{summary.get('source_blocked_count', 0)}`",
f"- accepted: `{summary['accepted_count']}`",
f"- pending: `{summary['pending_count']}`",
f"- human pending: `{summary['human_pending_count']}`",
@@ -262,10 +272,31 @@ def render_markdown(ledger: dict[str, Any]) -> str:
lines.append(f"- objective: {entry['objective']}")
lines.append(f"- source status: `{entry['source_status']}`")
lines.append(f"- observed state: `{json.dumps(entry['observed_state'], ensure_ascii=False)}`")
lines.append(
f"- source checks: `{entry.get('source_pass_count', 0)}` pass / "
f"`{entry.get('source_check_count', 0)}` total"
)
lines.append(f"- submission state: `{json.dumps(entry.get('submission_state', {}), ensure_ascii=False)}`")
lines.extend(["", "### Provenance Requirements", ""])
lines.extend(f"- {item}" for item in entry["provenance_requirements"])
lines.extend(["", "### Success Checks", ""])
lines.extend(
[
"",
"### Source Evidence Checks",
"",
"| Check | Current | Expected | Status |",
"| --- | --- | --- | --- |",
]
)
source_checks = entry.get("source_checklist", [])
if source_checks:
for row in source_checks:
lines.append(
f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` |"
)
else:
lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` |")
lines.extend(["", "### Completion Assertions", ""])
lines.extend(f"- {item}" for item in entry["success_checks"])
lines.extend(["", "### Privacy Contract", ""])
lines.extend(f"- {item}" for item in entry["privacy_contract"])
scripts/review_studio_world_class.py
+3 -1
View File
@@ -18,7 +18,9 @@ def render_inline_list(items: list[Any], empty_label: str) -> str:
def render_source_checks(entry: dict[str, Any]) -> str:
rows = build_source_checklist([entry])
rows = entry.get("source_checklist")
if not isinstance(rows, list):
rows = build_source_checklist([entry])
if not rows:
return "<p class='muted'>暂无源证据检查。</p>"
items = []
tests/verify_world_class_evidence_ledger.py
+13
View File
@@ -98,6 +98,9 @@ def main() -> None:
assert summary["submitted_entry_count"] == 0, summary
assert summary["missing_submission_count"] == 4, summary
assert summary["invalid_submission_count"] == 0, summary
assert summary["source_check_count"] >= 13, summary
assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary
assert summary["source_blocked_count"] >= 6, summary
assert summary["submitted_but_pending_count"] == 0, summary
assert summary["overclaim_guard_active"] is True, summary
assert summary["ready_to_claim_world_class"] is False, summary
@@ -110,6 +113,10 @@ def main() -> None:
"native-client-telemetry",
}, entries
assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"]
provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]}
assert provider_source["model_executed_count"]["status"] == "blocked", provider_source
assert provider_source["timing_observed_count"]["status"] == "pass", provider_source
assert provider_source["token_observed_count"]["status"] == "blocked", provider_source
assert entries["provider-holdout"]["submission_state"]["status"] == "missing", entries["provider-holdout"]
assert entries["provider-holdout"]["submission_state"]["ledger_counts_as_completion"] is False, entries["provider-holdout"]
assert entries["human-adjudication"]["observed_state"]["pending_count"] == 5, entries["human-adjudication"]
@@ -128,6 +135,9 @@ def main() -> None:
assert "World-Class Evidence Ledger" in markdown, markdown
assert "overclaim guard active: `true`" in markdown, markdown
assert "submitted entries: `0`" in markdown, markdown
assert "source checks:" in markdown, markdown
assert "Source Evidence Checks" in markdown, markdown
assert "| Provider model run | `0` | `>0` | `blocked` |" in markdown, markdown
assert "`provider-holdout`" in markdown, markdown
submissions = TMP / "submissions"
@@ -160,6 +170,7 @@ def main() -> None:
assert submitted_summary["submitted_entry_count"] == 1, submitted_summary
assert submitted_summary["submitted_but_pending_count"] == 1, submitted_summary
assert submitted_summary["accepted_count"] == 0, submitted_summary
assert submitted_summary["source_blocked_count"] >= 6, submitted_summary
submitted_provider = {entry["key"]: entry for entry in submitted_payload["entries"]}["provider-holdout"]
assert submitted_provider["status"] == "pending", submitted_provider
assert submitted_provider["submission_state"]["status"] == "submitted", submitted_provider
@@ -206,12 +217,14 @@ def main() -> None:
assert accepted_source_summary["source_accepted_count"] == 1, accepted_source_summary
assert accepted_source_summary["accepted_count"] == 0, accepted_source_summary
assert accepted_source_summary["source_accepted_without_valid_submission_count"] == 1, accepted_source_summary
assert accepted_source_summary["source_check_count"] >= 13, accepted_source_summary
accepted_source_provider = {
entry["key"]: entry for entry in accepted_source_payload["entries"]
}["provider-holdout"]
assert accepted_source_provider["source_accepted"] is True, accepted_source_provider
assert accepted_source_provider["status"] == "pending", accepted_source_provider
assert accepted_source_provider["submission_state"]["status"] == "missing", accepted_source_provider
assert all(item["status"] == "pass" for item in accepted_source_provider["source_checklist"]), accepted_source_provider
(accepted_source_skill / "reports" / "context_budget.json").write_text(
json.dumps({"summary": {"unrelated": True}}, ensure_ascii=False, indent=2) + "\n",