Split meta skill CLI and review gates
Merge the beta-ready Yao Meta Skill architecture, report, evidence gate, and release-boundary updates.\n\nRelease boundary: beta/public testing is allowed; formal world-class, fully reviewed, or superiority claims remain blocked until the pending evidence gates are accepted.
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
"""Patch target and baseline hash checks for approval-gated adaptation."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Shared adaptation patch target parsing and target-file baseline verification helpers."
|
||||
|
||||
BLOCKED_PATH_PARTS = {".git", "__pycache__", ".pytest_cache", "dist"}
|
||||
ABSENT_FILE_SHA256 = "__absent__"
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def target_file_sha256(skill_dir: Path, target_files: list[str]) -> dict[str, str]:
|
||||
observed: dict[str, str] = {}
|
||||
for target in target_files:
|
||||
path = skill_dir / target
|
||||
observed[target] = sha256_file(path) if path.exists() else ABSENT_FILE_SHA256
|
||||
return observed
|
||||
|
||||
|
||||
def normalize_patch_path(raw: str) -> str | None:
|
||||
token = raw.strip().split("\t", 1)[0].split(" ", 1)[0]
|
||||
if token == "/dev/null":
|
||||
return None
|
||||
if token.startswith("a/") or token.startswith("b/"):
|
||||
token = token[2:]
|
||||
path = Path(token)
|
||||
if path.is_absolute() or ".." in path.parts or any(part in BLOCKED_PATH_PARTS for part in path.parts):
|
||||
raise ValueError(f"Unsafe patch path: {raw}")
|
||||
if not token or token == ".":
|
||||
raise ValueError(f"Empty patch path: {raw}")
|
||||
return token
|
||||
|
||||
|
||||
def patch_target_files(patch_text: str) -> list[str]:
|
||||
targets: set[str] = set()
|
||||
for line in patch_text.splitlines():
|
||||
if line.startswith("--- ") or line.startswith("+++ "):
|
||||
raw = line[4:].strip()
|
||||
path = normalize_patch_path(raw)
|
||||
if path:
|
||||
targets.add(path)
|
||||
return sorted(targets)
|
||||
|
||||
|
||||
def validate_target_file_sha256(
|
||||
skill_dir: Path,
|
||||
target_files: list[str],
|
||||
expected_sha256: dict[str, Any],
|
||||
) -> tuple[list[str], dict[str, str]]:
|
||||
failures: list[str] = []
|
||||
observed: dict[str, str] = {}
|
||||
for target in target_files:
|
||||
path = skill_dir / target
|
||||
expected = str(expected_sha256.get(target, ""))
|
||||
if not expected:
|
||||
failures.append(f"Approval target_file_sha256 missing target: {target}")
|
||||
continue
|
||||
if path.exists() and not path.is_file():
|
||||
failures.append(f"Patch target is not a file: {target}")
|
||||
continue
|
||||
current = sha256_file(path) if path.exists() else ABSENT_FILE_SHA256
|
||||
observed[target] = current
|
||||
if current != expected:
|
||||
failures.append(f"Target file baseline sha256 does not match approval ledger: {target}")
|
||||
return failures, observed
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Shared report contracts for approval-gated skill adaptation."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Shared adaptation approval and regression report contract decoration helpers."
|
||||
|
||||
APPROVAL_SUMMARY_FIELDS = [
|
||||
"approval_count",
|
||||
"active_approval_count",
|
||||
"pending_review_count",
|
||||
"applied_count",
|
||||
"rollback_count",
|
||||
]
|
||||
APPROVAL_CONTRACT_FIELDS = [
|
||||
"approval_required",
|
||||
"patch_sha256_required",
|
||||
"allowlisted_targets_required",
|
||||
"target_file_sha256_required",
|
||||
"approval_draft_supported",
|
||||
"dry_run_default",
|
||||
"writes_repository_files_only_with_apply",
|
||||
"rollback_required",
|
||||
]
|
||||
REGRESSION_SUMMARY_FIELDS = [
|
||||
"apply_supported",
|
||||
"attempt_count",
|
||||
"approval_draft_count",
|
||||
"applied_count",
|
||||
"dry_run_count",
|
||||
"rollback_count",
|
||||
"regression_run_count",
|
||||
"regression_pass_count",
|
||||
"failure_count",
|
||||
]
|
||||
APPLY_CONTRACT_FIELDS = [
|
||||
*APPROVAL_CONTRACT_FIELDS,
|
||||
"safe_regression_commands_only",
|
||||
"rollback_on_failure_default",
|
||||
]
|
||||
APPROVAL_CONTRACT = {
|
||||
"approval_required": True,
|
||||
"patch_sha256_required": True,
|
||||
"allowlisted_targets_required": True,
|
||||
"target_file_sha256_required": True,
|
||||
"approval_draft_supported": True,
|
||||
"dry_run_default": True,
|
||||
"writes_repository_files_only_with_apply": True,
|
||||
"rollback_required": True,
|
||||
}
|
||||
APPLY_CONTRACT = {
|
||||
**APPROVAL_CONTRACT,
|
||||
"safe_regression_commands_only": True,
|
||||
"rollback_on_failure_default": True,
|
||||
}
|
||||
|
||||
|
||||
def top_level_mirrors(
|
||||
summary: dict[str, Any],
|
||||
contract: dict[str, Any],
|
||||
summary_fields: list[str],
|
||||
contract_fields: list[str],
|
||||
) -> dict[str, Any]:
|
||||
mirrored = {key: summary[key] for key in summary_fields if key in summary}
|
||||
mirrored.update({key: contract[key] for key in contract_fields if key in contract})
|
||||
return mirrored
|
||||
|
||||
|
||||
def report_contract(name: str, contract_key: str, summary_fields: list[str], contract_fields: list[str]) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"contract": name,
|
||||
"top_level_mirrors_summary": True,
|
||||
f"top_level_mirrors_{contract_key}": True,
|
||||
"summary_fields": summary_fields,
|
||||
f"{contract_key}_fields": contract_fields,
|
||||
"source_of_truth": ["summary", contract_key],
|
||||
}
|
||||
|
||||
|
||||
def decorate_approval_ledger(ledger: dict[str, Any]) -> dict[str, Any]:
|
||||
summary = ledger.get("summary", {}) if isinstance(ledger.get("summary"), dict) else {}
|
||||
approval_contract = (
|
||||
ledger.get("approval_contract", {}) if isinstance(ledger.get("approval_contract"), dict) else {}
|
||||
) or dict(APPROVAL_CONTRACT)
|
||||
ledger.update(top_level_mirrors(summary, approval_contract, APPROVAL_SUMMARY_FIELDS, APPROVAL_CONTRACT_FIELDS))
|
||||
ledger["approval_contract"] = approval_contract
|
||||
ledger["report_contract"] = report_contract(
|
||||
"adaptation-approval-ledger",
|
||||
"approval_contract",
|
||||
APPROVAL_SUMMARY_FIELDS,
|
||||
APPROVAL_CONTRACT_FIELDS,
|
||||
)
|
||||
return ledger
|
||||
|
||||
|
||||
def decorate_regression_report(report: dict[str, Any]) -> dict[str, Any]:
|
||||
summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {}
|
||||
apply_contract = (
|
||||
report.get("apply_contract", {}) if isinstance(report.get("apply_contract"), dict) else {}
|
||||
) or dict(APPLY_CONTRACT)
|
||||
report.update(top_level_mirrors(summary, apply_contract, REGRESSION_SUMMARY_FIELDS, APPLY_CONTRACT_FIELDS))
|
||||
report["apply_contract"] = apply_contract
|
||||
report["report_contract"] = report_contract(
|
||||
"adaptation-regression-report",
|
||||
"apply_contract",
|
||||
REGRESSION_SUMMARY_FIELDS,
|
||||
APPLY_CONTRACT_FIELDS,
|
||||
)
|
||||
return report
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -59,6 +60,45 @@ def confidence_value(value: Any) -> tuple[float | None, str | None]:
|
||||
return round(parsed, 3), None
|
||||
|
||||
|
||||
def prompt_sha256(pair: dict[str, Any]) -> str:
|
||||
return hashlib.sha256(str(pair.get("prompt", "")).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def canonical_sha256(value: Any) -> str:
|
||||
encoded = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
|
||||
def review_integrity(blind_pack: dict[str, Any]) -> dict[str, Any]:
|
||||
pairs = blind_pack.get("pairs", [])
|
||||
case_ids: list[str] = []
|
||||
prompt_hashes: dict[str, str] = {}
|
||||
if isinstance(pairs, list):
|
||||
for pair in pairs:
|
||||
if not isinstance(pair, dict):
|
||||
continue
|
||||
case_id = str(pair.get("case_id", "")).strip()
|
||||
if not case_id:
|
||||
continue
|
||||
case_ids.append(case_id)
|
||||
prompt_hashes[case_id] = prompt_sha256(pair)
|
||||
return {
|
||||
"blind_pack_sha256": canonical_sha256(blind_pack),
|
||||
"case_count": len(case_ids),
|
||||
"case_ids": case_ids,
|
||||
"prompt_sha256_by_case": prompt_hashes,
|
||||
}
|
||||
|
||||
|
||||
def default_reviewer_attestation() -> dict[str, Any]:
|
||||
return {
|
||||
"blind_review_completed_before_answer_key": False,
|
||||
"answer_key_not_opened_before_decisions": False,
|
||||
"raw_content_excluded": True,
|
||||
"reviewer_reason_required": True,
|
||||
}
|
||||
|
||||
|
||||
def answer_index(answer_key: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
answers = answer_key.get("answers", [])
|
||||
if not isinstance(answers, list):
|
||||
@@ -113,10 +153,13 @@ def build_decision_template(blind_pack: dict[str, Any]) -> dict[str, Any]:
|
||||
"schema_version": "1.0",
|
||||
"reviewer": "",
|
||||
"reviewed_at": "",
|
||||
"review_integrity": review_integrity(blind_pack),
|
||||
"reviewer_attestation": default_reviewer_attestation(),
|
||||
"decision_contract": {
|
||||
"winner_variant": "Use A or B after reading the blind review pack. Leave blank when pending.",
|
||||
"confidence": "Optional number from 0 to 1.",
|
||||
"reason": "Short reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.",
|
||||
"reason": "Required reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.",
|
||||
"reviewer_attestation": "Set blind_review_completed_before_answer_key and answer_key_not_opened_before_decisions to true only after a real blind review.",
|
||||
},
|
||||
"decisions": template_decisions,
|
||||
}
|
||||
@@ -127,6 +170,7 @@ def adjudicate_pair(
|
||||
pair: dict[str, Any],
|
||||
answer: dict[str, Any] | None,
|
||||
decision: dict[str, Any] | None,
|
||||
reviewer_metadata_present: bool = True,
|
||||
) -> tuple[dict[str, Any], list[str]]:
|
||||
failures: list[str] = []
|
||||
expected = normalize_variant(answer.get("expected_winner_variant", "") if answer else "")
|
||||
@@ -137,11 +181,12 @@ def adjudicate_pair(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"status": "pending",
|
||||
"expected_winner_variant": expected,
|
||||
"expected_winner_variant": "",
|
||||
"expected_revealed": False,
|
||||
"reviewer_winner_variant": "",
|
||||
"confidence": None,
|
||||
"reason": "",
|
||||
"prompt": str(pair.get("prompt", "")),
|
||||
"prompt_sha256": prompt_sha256(pair),
|
||||
},
|
||||
failures,
|
||||
)
|
||||
@@ -156,43 +201,77 @@ def adjudicate_pair(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"status": "pending",
|
||||
"expected_winner_variant": expected,
|
||||
"expected_winner_variant": "",
|
||||
"expected_revealed": False,
|
||||
"reviewer_winner_variant": "",
|
||||
"confidence": confidence,
|
||||
"reason": reason,
|
||||
"prompt": str(pair.get("prompt", "")),
|
||||
"prompt_sha256": prompt_sha256(pair),
|
||||
},
|
||||
failures,
|
||||
)
|
||||
if reviewer not in {"A", "B"}:
|
||||
if expected not in {"A", "B"}:
|
||||
status = "invalid"
|
||||
elif reviewer not in {"A", "B"}:
|
||||
failures.append(f"{case_id}: winner_variant must be A or B")
|
||||
status = "invalid"
|
||||
elif confidence_failure:
|
||||
failures.append(f"{case_id}: {confidence_failure}")
|
||||
status = "invalid"
|
||||
elif not reason:
|
||||
failures.append(f"{case_id}: reason is required before answer key can be revealed")
|
||||
status = "invalid"
|
||||
elif not reviewer_metadata_present:
|
||||
failures.append(f"{case_id}: reviewer and reviewed_at are required before answer key can be revealed")
|
||||
status = "invalid"
|
||||
else:
|
||||
status = "match" if reviewer == expected else "disagree"
|
||||
expected_revealed = status in {"match", "disagree"}
|
||||
return (
|
||||
{
|
||||
"case_id": case_id,
|
||||
"status": status,
|
||||
"expected_winner_variant": expected,
|
||||
"expected_winner_variant": expected if expected_revealed else "",
|
||||
"expected_revealed": expected_revealed,
|
||||
"reviewer_winner_variant": reviewer,
|
||||
"confidence": confidence,
|
||||
"reason": reason,
|
||||
"prompt": str(pair.get("prompt", "")),
|
||||
"prompt_sha256": prompt_sha256(pair),
|
||||
},
|
||||
failures,
|
||||
)
|
||||
|
||||
|
||||
def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]:
|
||||
def reviewer_attestation_state(decisions_payload: dict[str, Any]) -> dict[str, bool]:
|
||||
attestation = decisions_payload.get("reviewer_attestation", {})
|
||||
if not isinstance(attestation, dict):
|
||||
attestation = {}
|
||||
blind_review_completed = attestation.get("blind_review_completed_before_answer_key") is True
|
||||
answer_key_not_opened = attestation.get("answer_key_not_opened_before_decisions") is True
|
||||
raw_content_excluded = attestation.get("raw_content_excluded") is True
|
||||
reviewer_reason_required = attestation.get("reviewer_reason_required") is True
|
||||
return {
|
||||
"blind_review_attested": blind_review_completed and answer_key_not_opened,
|
||||
"blind_review_completed_before_answer_key": blind_review_completed,
|
||||
"answer_key_not_opened_before_decisions": answer_key_not_opened,
|
||||
"raw_content_excluded_attested": raw_content_excluded,
|
||||
"reviewer_reason_required_attested": reviewer_reason_required,
|
||||
}
|
||||
|
||||
|
||||
def build_summary(
|
||||
pairs: list[dict[str, Any]],
|
||||
failures: list[str],
|
||||
reviewer_metadata_present: bool,
|
||||
attestation_state: dict[str, bool],
|
||||
) -> dict[str, Any]:
|
||||
pair_count = len(pairs)
|
||||
judgment_count = sum(1 for item in pairs if item["status"] in {"match", "disagree"})
|
||||
pending_count = sum(1 for item in pairs if item["status"] == "pending")
|
||||
agreement_count = sum(1 for item in pairs if item["status"] == "match")
|
||||
disagreement_count = sum(1 for item in pairs if item["status"] == "disagree")
|
||||
invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid")
|
||||
answer_revealed_count = sum(1 for item in pairs if item.get("expected_revealed"))
|
||||
agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None
|
||||
return {
|
||||
"pair_count": pair_count,
|
||||
@@ -201,12 +280,122 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str,
|
||||
"agreement_count": agreement_count,
|
||||
"disagreement_count": disagreement_count,
|
||||
"invalid_decision_count": invalid_decision_count,
|
||||
"answer_revealed_count": answer_revealed_count,
|
||||
"pending_answer_hidden_count": sum(1 for item in pairs if item["status"] in {"pending", "invalid"} and not item.get("expected_revealed")),
|
||||
"agreement_rate": agreement_rate,
|
||||
"needs_review": pending_count > 0,
|
||||
"reviewer_metadata_present": reviewer_metadata_present,
|
||||
"reason_required": True,
|
||||
**attestation_state,
|
||||
"ready_for_human_evidence": bool(pair_count)
|
||||
and judgment_count == pair_count
|
||||
and pending_count == 0
|
||||
and invalid_decision_count == 0
|
||||
and reviewer_metadata_present
|
||||
and attestation_state["blind_review_attested"]
|
||||
and attestation_state["raw_content_excluded_attested"]
|
||||
and attestation_state["reviewer_reason_required_attested"]
|
||||
and len(failures) == 0,
|
||||
"failure_count": len(failures),
|
||||
}
|
||||
|
||||
|
||||
def checklist_readiness(pair: dict[str, Any]) -> tuple[str, str]:
|
||||
status = pair.get("status")
|
||||
if status in {"match", "disagree"}:
|
||||
return "adjudicated", "Reviewer decision is valid; answer key is revealed for this case."
|
||||
if status == "invalid":
|
||||
return "fix-decision", "Reviewer decision exists but failed validation; answer key remains hidden."
|
||||
return "awaiting-decision", "Reviewer has not selected A or B yet; answer key remains hidden."
|
||||
|
||||
|
||||
def build_reviewer_checklist(
|
||||
pairs: list[dict[str, Any]],
|
||||
blind_pack_path: Path,
|
||||
decisions_path: Path,
|
||||
) -> list[dict[str, Any]]:
|
||||
checklist = []
|
||||
for pair in pairs:
|
||||
readiness, blocking_reason = checklist_readiness(pair)
|
||||
checklist.append(
|
||||
{
|
||||
"case_id": pair.get("case_id", ""),
|
||||
"readiness": readiness,
|
||||
"blocking_reason": blocking_reason,
|
||||
"status": pair.get("status", "pending"),
|
||||
"reviewer_winner_variant": pair.get("reviewer_winner_variant", ""),
|
||||
"answer_key_visible": bool(pair.get("expected_revealed")),
|
||||
"prompt_sha256": pair.get("prompt_sha256", ""),
|
||||
"blind_pack_path": display_path(blind_pack_path),
|
||||
"decisions_path": display_path(decisions_path),
|
||||
"commands": {
|
||||
"prepare_review_kit": "python3 scripts/yao.py output-review-kit",
|
||||
"write_template": "python3 scripts/adjudicate_output_review.py --write-template",
|
||||
"import_decisions": "python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
||||
"adjudicate": "python3 scripts/yao.py output-review",
|
||||
"refresh_review_studio": "python3 scripts/yao.py review-studio .",
|
||||
},
|
||||
"required_fields": {
|
||||
"winner_variant": "A or B after reading only the blind review pack.",
|
||||
"confidence": "Optional number from 0 to 1.",
|
||||
"reason": "Required rationale; do not reveal baseline or with-skill labels before adjudication.",
|
||||
"reviewer": "Human reviewer name or review group at the decision-file top level.",
|
||||
"reviewed_at": "Review date or timestamp at the decision-file top level.",
|
||||
"reviewer_attestation.blind_review_completed_before_answer_key": "True only after the reviewer has completed choices before opening the answer key.",
|
||||
"reviewer_attestation.answer_key_not_opened_before_decisions": "True only when the answer key was not opened before decisions were recorded.",
|
||||
},
|
||||
"privacy_contract": [
|
||||
"Do not paste raw private user data into the decision reason.",
|
||||
"Do not open the answer key before reviewer choices are recorded.",
|
||||
"Leave winner_variant blank when the reviewer is not ready to decide.",
|
||||
],
|
||||
}
|
||||
)
|
||||
return checklist
|
||||
|
||||
|
||||
def add_checklist_summary(summary: dict[str, Any], checklist: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
enriched = dict(summary)
|
||||
enriched["reviewer_checklist_count"] = len(checklist)
|
||||
enriched["reviewer_checklist_pending_count"] = sum(1 for item in checklist if item["readiness"] == "awaiting-decision")
|
||||
enriched["reviewer_checklist_invalid_count"] = sum(1 for item in checklist if item["readiness"] == "fix-decision")
|
||||
enriched["reviewer_checklist_ready_count"] = sum(1 for item in checklist if item["readiness"] == "adjudicated")
|
||||
return enriched
|
||||
|
||||
|
||||
def render_reviewer_checklist(checklist: list[dict[str, Any]]) -> list[str]:
|
||||
lines = [
|
||||
"## Reviewer Checklist",
|
||||
"",
|
||||
"| Case | Readiness | Answer key | Decision file |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
if not checklist:
|
||||
lines.append("| `none` | `adjudicated` | n/a | none |")
|
||||
return lines
|
||||
for item in checklist:
|
||||
answer_key = "visible" if item.get("answer_key_visible") else "hidden"
|
||||
lines.append(
|
||||
f"| `{item['case_id']}` | `{item['readiness']}` | `{answer_key}` | `{item['decisions_path']}` |"
|
||||
)
|
||||
for item in checklist:
|
||||
lines.extend(["", f"### {item['case_id']}", ""])
|
||||
lines.append(f"- readiness: `{item['readiness']}`")
|
||||
lines.append(f"- blocking reason: {item['blocking_reason']}")
|
||||
lines.append(f"- answer key visible: `{str(item['answer_key_visible']).lower()}`")
|
||||
lines.append(f"- blind pack: `{item['blind_pack_path']}`")
|
||||
lines.append(f"- decisions: `{item['decisions_path']}`")
|
||||
lines.extend(["", "#### Commands", ""])
|
||||
for label, command in item.get("commands", {}).items():
|
||||
lines.append(f"- {label}: `{command}`")
|
||||
lines.extend(["", "#### Required Fields", ""])
|
||||
for label, description in item.get("required_fields", {}).items():
|
||||
lines.append(f"- {label}: {description}")
|
||||
lines.extend(["", "#### Privacy Contract", ""])
|
||||
lines.extend(f"- {contract}" for contract in item.get("privacy_contract", []))
|
||||
return lines
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
summary = payload["summary"]
|
||||
lines = [
|
||||
@@ -219,6 +408,13 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
f"- Pending: `{summary['pending_count']}`",
|
||||
f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`",
|
||||
f"- Invalid decisions: `{summary['invalid_decision_count']}`",
|
||||
f"- Answer keys revealed: `{summary['answer_revealed_count']}`",
|
||||
f"- Pending/invalid answers hidden: `{summary['pending_answer_hidden_count']}`",
|
||||
f"- Reviewer checklist: `{summary['reviewer_checklist_ready_count']}` ready / `{summary['reviewer_checklist_count']}` total",
|
||||
f"- Reviewer metadata present: `{str(summary['reviewer_metadata_present']).lower()}`",
|
||||
f"- Blind review attested: `{str(summary['blind_review_attested']).lower()}`",
|
||||
f"- Raw content excluded: `{str(summary['raw_content_excluded_attested']).lower()}`",
|
||||
f"- Ready for human evidence: `{str(summary['ready_for_human_evidence']).lower()}`",
|
||||
"",
|
||||
]
|
||||
if summary["judgment_count"] == 0:
|
||||
@@ -227,6 +423,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
"No reviewer decisions recorded yet.",
|
||||
"",
|
||||
"Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.",
|
||||
"Expected winners stay hidden until a valid reviewer decision is recorded.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
@@ -241,14 +438,16 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
for item in payload["pairs"]:
|
||||
confidence = "" if item.get("confidence") is None else str(item["confidence"])
|
||||
reason = str(item.get("reason", "")).replace("|", "\\|") or ""
|
||||
expected = item.get("expected_winner_variant", "") if item.get("expected_revealed") else "hidden"
|
||||
lines.append(
|
||||
f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | "
|
||||
f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |"
|
||||
f"{expected} | {item['status']} | {confidence} | {reason} |"
|
||||
)
|
||||
if payload.get("failures"):
|
||||
lines.extend(["", "## Failures", ""])
|
||||
for failure in payload["failures"]:
|
||||
lines.append(f"- {failure}")
|
||||
lines.extend(["", *render_reviewer_checklist(payload.get("reviewer_checklist", []))])
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
@@ -289,6 +488,11 @@ def adjudicate_output_review(
|
||||
answers_by_id = answer_index(answer_key)
|
||||
decisions_by_id, index_failures = decision_index(decisions_payload)
|
||||
failures.extend(index_failures)
|
||||
reviewer_metadata_present = bool(
|
||||
str(decisions_payload.get("reviewer", "")).strip()
|
||||
and str(decisions_payload.get("reviewed_at", "")).strip()
|
||||
)
|
||||
attestation_state = reviewer_attestation_state(decisions_payload)
|
||||
|
||||
for case_id in decisions_by_id:
|
||||
if case_id not in pairs_by_id:
|
||||
@@ -296,17 +500,27 @@ def adjudicate_output_review(
|
||||
|
||||
adjudicated_pairs: list[dict[str, Any]] = []
|
||||
for case_id, pair in pairs_by_id.items():
|
||||
adjudicated, pair_failures = adjudicate_pair(case_id, pair, answers_by_id.get(case_id), decisions_by_id.get(case_id))
|
||||
adjudicated, pair_failures = adjudicate_pair(
|
||||
case_id,
|
||||
pair,
|
||||
answers_by_id.get(case_id),
|
||||
decisions_by_id.get(case_id),
|
||||
reviewer_metadata_present=reviewer_metadata_present,
|
||||
)
|
||||
adjudicated_pairs.append(adjudicated)
|
||||
failures.extend(pair_failures)
|
||||
|
||||
summary = build_summary(adjudicated_pairs, failures)
|
||||
summary = build_summary(adjudicated_pairs, failures, reviewer_metadata_present, attestation_state)
|
||||
reviewer_checklist = build_reviewer_checklist(adjudicated_pairs, blind_pack_path, decisions_path)
|
||||
summary = add_checklist_summary(summary, reviewer_checklist)
|
||||
payload = {
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
"summary": summary,
|
||||
"reviewer": decisions_payload.get("reviewer", ""),
|
||||
"reviewed_at": decisions_payload.get("reviewed_at", ""),
|
||||
"review_integrity": review_integrity(blind_pack),
|
||||
"reviewer_attestation": decisions_payload.get("reviewer_attestation", {}),
|
||||
"artifacts": {
|
||||
"blind_pack": display_path(blind_pack_path),
|
||||
"answer_key": display_path(answer_key_path),
|
||||
@@ -316,6 +530,7 @@ def adjudicate_output_review(
|
||||
},
|
||||
"template_written": template_written,
|
||||
"pairs": adjudicated_pairs,
|
||||
"reviewer_checklist": reviewer_checklist,
|
||||
"failures": failures,
|
||||
}
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply an approved adaptation patch with allowlisted targets and regression evidence."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from adaptation_patch_safety import patch_target_files, sha256_file, target_file_sha256, validate_target_file_sha256
|
||||
from adaptation_report_contracts import APPLY_CONTRACT, APPROVAL_CONTRACT, decorate_approval_ledger, decorate_regression_report
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Approval-gated adaptive patch application with dry-run, allowlist, regression, and rollback evidence."
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def display_path(path: Path, skill_dir: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def resolve_path(skill_dir: Path, value: str) -> Path:
|
||||
path = Path(value)
|
||||
return path if path.is_absolute() else skill_dir / path
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def approved_entries(ledger: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
entries = ledger.get("entries")
|
||||
return [item for item in entries if isinstance(item, dict)] if isinstance(entries, list) else []
|
||||
|
||||
|
||||
def find_proposal(proposals: dict[str, Any], proposal_id: str) -> dict[str, Any]:
|
||||
for item in proposals.get("proposals", []):
|
||||
if isinstance(item, dict) and item.get("proposal_id") == proposal_id:
|
||||
return item
|
||||
return {}
|
||||
|
||||
|
||||
def find_approval(ledger: dict[str, Any], proposal_id: str) -> dict[str, Any]:
|
||||
for item in approved_entries(ledger):
|
||||
if item.get("proposal_id") == proposal_id and item.get("decision") == "approved":
|
||||
return item
|
||||
return {}
|
||||
|
||||
|
||||
def refresh_ledger_summary(ledger: dict[str, Any]) -> None:
|
||||
entries = approved_entries(ledger)
|
||||
approved = [item for item in entries if item.get("decision") == "approved"]
|
||||
pending = [item for item in entries if item.get("decision") == "pending-review"]
|
||||
ledger["summary"] = {
|
||||
"approval_count": len(approved),
|
||||
"active_approval_count": len(approved),
|
||||
"pending_review_count": len(pending),
|
||||
"applied_count": 0,
|
||||
"rollback_count": 0,
|
||||
}
|
||||
decorate_approval_ledger(ledger)
|
||||
|
||||
|
||||
def upsert_ledger_entry(ledger: dict[str, Any], entry: dict[str, Any]) -> None:
|
||||
entries = approved_entries(ledger)
|
||||
proposal_id = entry.get("proposal_id")
|
||||
kept = [item for item in entries if item.get("proposal_id") != proposal_id]
|
||||
ledger["entries"] = [*kept, entry]
|
||||
refresh_ledger_summary(ledger)
|
||||
|
||||
|
||||
def parse_date(value: str) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def validate_approval(approval: dict[str, Any], today: date) -> list[str]:
|
||||
failures = []
|
||||
required = [
|
||||
"reviewer",
|
||||
"reason",
|
||||
"approved_at",
|
||||
"patch_sha256",
|
||||
"target_files",
|
||||
"target_file_sha256",
|
||||
"verification_commands",
|
||||
"rollback_plan",
|
||||
]
|
||||
for key in required:
|
||||
if not approval.get(key):
|
||||
failures.append(f"Approval entry missing required field: {key}")
|
||||
expires_at = parse_date(str(approval.get("expires_at", "")))
|
||||
if expires_at and expires_at < today:
|
||||
failures.append(f"Approval entry is expired: {approval.get('expires_at')}")
|
||||
if not isinstance(approval.get("target_files"), list):
|
||||
failures.append("Approval target_files must be a list.")
|
||||
if not isinstance(approval.get("target_file_sha256"), dict):
|
||||
failures.append("Approval target_file_sha256 must be an object.")
|
||||
if not isinstance(approval.get("verification_commands"), list):
|
||||
failures.append("Approval verification_commands must be a list.")
|
||||
return failures
|
||||
|
||||
|
||||
def safe_command(command: str) -> tuple[bool, list[str], str]:
|
||||
try:
|
||||
parts = shlex.split(command)
|
||||
except ValueError as exc:
|
||||
return False, [], f"cannot parse command: {exc}"
|
||||
if not parts:
|
||||
return False, [], "empty command"
|
||||
if parts[0] == "make" and len(parts) == 2 and all(ch.isalnum() or ch in {"-", "_"} for ch in parts[1]):
|
||||
return True, parts, "make target"
|
||||
if parts[0] in {"python3", sys.executable} and len(parts) >= 2:
|
||||
script = Path(parts[1])
|
||||
if not script.is_absolute() and script.parts and script.parts[0] in {"tests", "scripts"} and script.suffix == ".py":
|
||||
return True, parts, "local python verifier"
|
||||
return False, parts, "command is not in the safe regression allowlist"
|
||||
|
||||
|
||||
def run_command(command: str, skill_dir: Path) -> dict[str, Any]:
|
||||
allowed, parts, reason = safe_command(command)
|
||||
if not allowed:
|
||||
return {"command": command, "ok": False, "returncode": None, "stdout": "", "stderr": reason}
|
||||
proc = subprocess.run(parts, cwd=skill_dir, capture_output=True, text=True)
|
||||
return {
|
||||
"command": command,
|
||||
"ok": proc.returncode == 0,
|
||||
"returncode": proc.returncode,
|
||||
"stdout": proc.stdout[-4000:],
|
||||
"stderr": proc.stderr[-4000:],
|
||||
}
|
||||
|
||||
|
||||
def git_apply_check(skill_dir: Path, patch_file: Path) -> dict[str, Any]:
|
||||
env = dict(os.environ)
|
||||
env["GIT_CEILING_DIRECTORIES"] = str(skill_dir.parent)
|
||||
proc = subprocess.run(["git", "apply", "--check", str(patch_file)], cwd=skill_dir, capture_output=True, text=True, env=env)
|
||||
return {"ok": proc.returncode == 0, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr}
|
||||
|
||||
|
||||
def git_apply(skill_dir: Path, patch_file: Path) -> dict[str, Any]:
|
||||
env = dict(os.environ)
|
||||
env["GIT_CEILING_DIRECTORIES"] = str(skill_dir.parent)
|
||||
proc = subprocess.run(["git", "apply", str(patch_file)], cwd=skill_dir, capture_output=True, text=True, env=env)
|
||||
return {"ok": proc.returncode == 0, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr}
|
||||
|
||||
|
||||
def git_apply_reverse(skill_dir: Path, patch_file: Path) -> dict[str, Any]:
|
||||
env = dict(os.environ)
|
||||
env["GIT_CEILING_DIRECTORIES"] = str(skill_dir.parent)
|
||||
proc = subprocess.run(["git", "apply", "-R", str(patch_file)], cwd=skill_dir, capture_output=True, text=True, env=env)
|
||||
return {"ok": proc.returncode == 0, "returncode": proc.returncode, "stdout": proc.stdout, "stderr": proc.stderr}
|
||||
|
||||
|
||||
def empty_approval_ledger(generated_at: str) -> dict[str, Any]:
|
||||
return decorate_approval_ledger({
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
"generated_at": generated_at,
|
||||
"summary": {
|
||||
"approval_count": 0,
|
||||
"active_approval_count": 0,
|
||||
"pending_review_count": 0,
|
||||
"applied_count": 0,
|
||||
"rollback_count": 0,
|
||||
},
|
||||
"approval_contract": dict(APPROVAL_CONTRACT),
|
||||
"entries": [],
|
||||
})
|
||||
|
||||
|
||||
def empty_regression_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
return decorate_regression_report({
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
"summary": {
|
||||
"apply_supported": True,
|
||||
"attempt_count": 0,
|
||||
"approval_draft_count": 0,
|
||||
"applied_count": 0,
|
||||
"dry_run_count": 0,
|
||||
"rollback_count": 0,
|
||||
"regression_run_count": 0,
|
||||
"regression_pass_count": 0,
|
||||
"failure_count": 0,
|
||||
},
|
||||
"apply_contract": dict(APPLY_CONTRACT),
|
||||
"attempts": [],
|
||||
"failures": [],
|
||||
"artifacts": {
|
||||
"json": "reports/adaptation_regression_report.json",
|
||||
"approval_ledger": "reports/adaptation_approval_ledger.json",
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Adaptation Regression Report",
|
||||
"",
|
||||
f"- generated_at: `{report['generated_at']}`",
|
||||
f"- apply_supported: `{str(summary['apply_supported']).lower()}`",
|
||||
f"- attempts: `{summary['attempt_count']}`",
|
||||
f"- applied: `{summary['applied_count']}`",
|
||||
f"- dry runs: `{summary['dry_run_count']}`",
|
||||
f"- rollbacks: `{summary.get('rollback_count', 0)}`",
|
||||
f"- failures: `{summary['failure_count']}`",
|
||||
"",
|
||||
"This report proves the adaptive apply harness behavior. It does not count proposals as applied changes.",
|
||||
"",
|
||||
]
|
||||
for attempt in report.get("attempts", []):
|
||||
lines.extend(
|
||||
[
|
||||
f"## {attempt['proposal_id']}",
|
||||
"",
|
||||
f"- mode: `{attempt['mode']}`",
|
||||
f"- status: `{attempt['status']}`",
|
||||
f"- patch: `{attempt['patch']}`",
|
||||
f"- patch sha256: `{attempt['patch_sha256']}`",
|
||||
f"- targets: `{', '.join(attempt['target_files'])}`",
|
||||
f"- rollback: {attempt['rollback']['plan']}",
|
||||
]
|
||||
)
|
||||
if attempt.get("regression_runs"):
|
||||
lines.append("- regression:")
|
||||
lines.extend(f" - `{item['command']}`: `{str(item['ok']).lower()}`" for item in attempt["regression_runs"])
|
||||
lines.append("")
|
||||
if report.get("failures"):
|
||||
lines.extend(["## Failures", ""])
|
||||
lines.extend(f"- {failure}" for failure in report["failures"])
|
||||
draft = report.get("approval_draft")
|
||||
if isinstance(draft, dict):
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Approval Draft",
|
||||
"",
|
||||
f"- proposal: `{draft.get('proposal_id', '')}`",
|
||||
f"- decision: `{draft.get('decision', '')}`",
|
||||
f"- patch sha256: `{draft.get('patch_sha256', '')}`",
|
||||
f"- targets: `{', '.join(draft.get('target_files', []))}`",
|
||||
"",
|
||||
"A human reviewer must set `decision` to `approved` and fill reviewer, reason, and approval date before apply.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def build_report(args: argparse.Namespace) -> dict[str, Any]:
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
generated_at = args.generated_at or utc_now()
|
||||
ledger_path = resolve_path(skill_dir, args.approval_ledger)
|
||||
output_json = resolve_path(skill_dir, args.output_json)
|
||||
output_md = resolve_path(skill_dir, args.output_md)
|
||||
if args.write_template:
|
||||
ledger = empty_approval_ledger(generated_at)
|
||||
report = empty_regression_report(skill_dir, generated_at)
|
||||
write_json(ledger_path, ledger)
|
||||
write_json(output_json, report)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
return report
|
||||
|
||||
failures: list[str] = []
|
||||
proposals_path = resolve_path(skill_dir, args.proposals_json)
|
||||
patch_path = resolve_path(skill_dir, args.patch_file) if args.patch_file else Path("")
|
||||
proposals = load_json(proposals_path)
|
||||
ledger = load_json(ledger_path)
|
||||
if not proposals:
|
||||
failures.append(f"Proposal report missing or invalid: {display_path(proposals_path, skill_dir)}")
|
||||
if args.prepare_approval and not ledger and not ledger_path.exists():
|
||||
ledger = empty_approval_ledger(generated_at)
|
||||
if not args.prepare_approval and not ledger:
|
||||
failures.append(f"Approval ledger missing or invalid: {display_path(ledger_path, skill_dir)}")
|
||||
if args.prepare_approval and ledger_path.exists() and not ledger:
|
||||
failures.append(f"Approval ledger exists but is invalid: {display_path(ledger_path, skill_dir)}")
|
||||
if not args.proposal_id:
|
||||
failures.append("--proposal-id is required unless --write-template is used.")
|
||||
if not args.patch_file or not patch_path.exists():
|
||||
failures.append("--patch-file must point to an existing unified diff.")
|
||||
|
||||
today = parse_date(str(args.today or date.today().isoformat()))
|
||||
if today is None:
|
||||
failures.append("--today must be an ISO date when provided.")
|
||||
|
||||
proposal = find_proposal(proposals, args.proposal_id) if not failures else {}
|
||||
if not failures and not proposal:
|
||||
failures.append(f"Proposal id is not present in proposal report: {args.proposal_id}")
|
||||
|
||||
patch_sha = sha256_file(patch_path) if patch_path.exists() else ""
|
||||
target_files: list[str] = []
|
||||
observed_target_file_sha256: dict[str, str] = {}
|
||||
if not failures:
|
||||
try:
|
||||
target_files = patch_target_files(patch_path.read_text(encoding="utf-8", errors="replace"))
|
||||
except ValueError as exc:
|
||||
failures.append(str(exc))
|
||||
|
||||
if args.prepare_approval:
|
||||
if not failures:
|
||||
proposal_targets = set(str(item) for item in proposal.get("target_files", []))
|
||||
patch_targets = set(target_files)
|
||||
if not patch_targets:
|
||||
failures.append("Patch does not declare any target files.")
|
||||
if not patch_targets <= proposal_targets:
|
||||
failures.append("Patch touches files outside proposal target_files.")
|
||||
check = git_apply_check(skill_dir, patch_path) if not failures else {"ok": False, "returncode": None, "stdout": "", "stderr": ""}
|
||||
if not failures and not check["ok"]:
|
||||
failures.append(f"git apply --check failed: {check['stderr'].strip()}")
|
||||
draft: dict[str, Any] = {}
|
||||
if not failures:
|
||||
observed_target_file_sha256 = target_file_sha256(skill_dir, target_files)
|
||||
draft = {
|
||||
"proposal_id": args.proposal_id,
|
||||
"decision": "pending-review",
|
||||
"reviewer": "",
|
||||
"reason": "",
|
||||
"approved_at": "",
|
||||
"expires_at": "",
|
||||
"created_at": generated_at,
|
||||
"patch": display_path(patch_path, skill_dir),
|
||||
"patch_sha256": patch_sha,
|
||||
"target_files": target_files,
|
||||
"target_file_sha256": observed_target_file_sha256,
|
||||
"verification_commands": [
|
||||
str(command) for command in proposal.get("verification_commands", []) if command
|
||||
],
|
||||
"rollback_plan": proposal.get("rollback_plan", f"git apply -R {display_path(patch_path, skill_dir)}"),
|
||||
}
|
||||
upsert_ledger_entry(ledger, draft)
|
||||
write_json(ledger_path, ledger)
|
||||
report = decorate_regression_report({
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
"summary": {
|
||||
"apply_supported": True,
|
||||
"attempt_count": 0,
|
||||
"approval_draft_count": 1 if draft else 0,
|
||||
"applied_count": 0,
|
||||
"dry_run_count": 0,
|
||||
"rollback_count": 0,
|
||||
"regression_run_count": 0,
|
||||
"regression_pass_count": 0,
|
||||
"failure_count": len(failures),
|
||||
},
|
||||
"apply_contract": dict(APPLY_CONTRACT),
|
||||
"approval_draft": draft,
|
||||
"attempts": [],
|
||||
"failures": failures,
|
||||
"artifacts": {
|
||||
"json": display_path(output_json, skill_dir),
|
||||
"markdown": display_path(output_md, skill_dir),
|
||||
"approval_ledger": display_path(ledger_path, skill_dir),
|
||||
},
|
||||
})
|
||||
write_json(output_json, report)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
return report
|
||||
|
||||
approval = find_approval(ledger, args.proposal_id) if not failures else {}
|
||||
if not failures and not approval:
|
||||
failures.append(f"Proposal id is not approved in the ledger: {args.proposal_id}")
|
||||
if approval:
|
||||
refresh_ledger_summary(ledger)
|
||||
write_json(ledger_path, ledger)
|
||||
failures.extend(validate_approval(approval, today or date.today()))
|
||||
|
||||
if not failures:
|
||||
approved_targets = set(str(item) for item in approval.get("target_files", []))
|
||||
proposal_targets = set(str(item) for item in proposal.get("target_files", []))
|
||||
patch_targets = set(target_files)
|
||||
if patch_sha != approval.get("patch_sha256"):
|
||||
failures.append("Patch sha256 does not match approval ledger.")
|
||||
if not patch_targets:
|
||||
failures.append("Patch does not declare any target files.")
|
||||
if not patch_targets <= approved_targets:
|
||||
failures.append("Patch touches files outside approval target_files.")
|
||||
if not patch_targets <= proposal_targets:
|
||||
failures.append("Patch touches files outside proposal target_files.")
|
||||
if not failures:
|
||||
baseline_failures, observed_target_file_sha256 = validate_target_file_sha256(
|
||||
skill_dir,
|
||||
target_files,
|
||||
approval.get("target_file_sha256", {}),
|
||||
)
|
||||
failures.extend(baseline_failures)
|
||||
|
||||
check = git_apply_check(skill_dir, patch_path) if not failures else {"ok": False, "returncode": None, "stdout": "", "stderr": ""}
|
||||
if not failures and not check["ok"]:
|
||||
failures.append(f"git apply --check failed: {check['stderr'].strip()}")
|
||||
|
||||
mode = "apply" if args.apply else "dry-run"
|
||||
applied = False
|
||||
apply_result = {"ok": False, "returncode": None, "stdout": "", "stderr": "not requested"}
|
||||
if not failures and args.apply:
|
||||
apply_result = git_apply(skill_dir, patch_path)
|
||||
applied = bool(apply_result["ok"])
|
||||
if not applied:
|
||||
failures.append(f"git apply failed: {apply_result['stderr'].strip()}")
|
||||
|
||||
regression_runs: list[dict[str, Any]] = []
|
||||
if not failures and args.run_verification:
|
||||
for command in approval.get("verification_commands", []):
|
||||
regression_runs.append(run_command(str(command), skill_dir))
|
||||
if any(not item["ok"] for item in regression_runs):
|
||||
failures.append("One or more regression commands failed or were not allowed.")
|
||||
|
||||
rollback_result = {"ok": None, "returncode": None, "stdout": "", "stderr": "not needed"}
|
||||
rolled_back = False
|
||||
if applied and failures and args.rollback_on_failure:
|
||||
rollback_result = git_apply_reverse(skill_dir, patch_path)
|
||||
rolled_back = bool(rollback_result["ok"])
|
||||
if not rolled_back:
|
||||
failures.append(f"Automatic rollback failed: {rollback_result['stderr'].strip()}")
|
||||
|
||||
attempt = {
|
||||
"proposal_id": args.proposal_id or "",
|
||||
"mode": mode,
|
||||
"status": "failed-rolled-back" if rolled_back else ("failed" if failures else ("applied" if applied else "dry-run-pass")),
|
||||
"patch": display_path(patch_path, skill_dir) if args.patch_file else "",
|
||||
"patch_sha256": patch_sha,
|
||||
"target_files": target_files,
|
||||
"approval": {
|
||||
"reviewer": approval.get("reviewer", ""),
|
||||
"approved_at": approval.get("approved_at", ""),
|
||||
"expires_at": approval.get("expires_at", ""),
|
||||
},
|
||||
"target_file_sha256": {
|
||||
"expected": {
|
||||
target: str(approval.get("target_file_sha256", {}).get(target, ""))
|
||||
for target in target_files
|
||||
},
|
||||
"observed": observed_target_file_sha256,
|
||||
},
|
||||
"git_apply_check": check,
|
||||
"git_apply": apply_result,
|
||||
"regression_runs": regression_runs,
|
||||
"rollback_result": rollback_result,
|
||||
"rollback": {
|
||||
"plan": approval.get("rollback_plan", ""),
|
||||
"command": f"git apply -R {display_path(patch_path, skill_dir)}" if args.patch_file else "",
|
||||
},
|
||||
}
|
||||
report = decorate_regression_report({
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
"summary": {
|
||||
"apply_supported": True,
|
||||
"attempt_count": 1,
|
||||
"approval_draft_count": 0,
|
||||
"applied_count": 1 if applied and not rolled_back else 0,
|
||||
"dry_run_count": 0 if args.apply else 1,
|
||||
"rollback_count": 1 if rolled_back else 0,
|
||||
"regression_run_count": len(regression_runs),
|
||||
"regression_pass_count": sum(1 for item in regression_runs if item["ok"]),
|
||||
"failure_count": len(failures),
|
||||
},
|
||||
"apply_contract": dict(APPLY_CONTRACT),
|
||||
"attempts": [attempt],
|
||||
"failures": failures,
|
||||
"artifacts": {
|
||||
"json": display_path(output_json, skill_dir),
|
||||
"markdown": display_path(output_md, skill_dir),
|
||||
"approval_ledger": display_path(ledger_path, skill_dir),
|
||||
},
|
||||
})
|
||||
write_json(output_json, report)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Apply an approved adaptation patch with review and regression evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--proposal-id")
|
||||
parser.add_argument("--patch-file")
|
||||
parser.add_argument("--proposals-json", default="reports/adaptation_proposals.json")
|
||||
parser.add_argument("--approval-ledger", default="reports/adaptation_approval_ledger.json")
|
||||
parser.add_argument("--output-json", default="reports/adaptation_regression_report.json")
|
||||
parser.add_argument("--output-md", default="reports/adaptation_regression_report.md")
|
||||
parser.add_argument("--generated-at")
|
||||
parser.add_argument("--today")
|
||||
parser.add_argument("--write-template", action="store_true")
|
||||
parser.add_argument(
|
||||
"--prepare-approval",
|
||||
action="store_true",
|
||||
help="Create or update a pending approval ledger entry with patch and target baseline hashes.",
|
||||
)
|
||||
parser.add_argument("--apply", action="store_true", help="Write the patch after every approval and allowlist check passes.")
|
||||
parser.add_argument("--run-verification", action="store_true")
|
||||
parser.add_argument(
|
||||
"--no-rollback-on-failure",
|
||||
dest="rollback_on_failure",
|
||||
action="store_false",
|
||||
help="Leave an applied patch in place if verification fails. Default is to reverse the patch.",
|
||||
)
|
||||
parser.set_defaults(rollback_on_failure=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(args)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Git source/generated dirty classification for benchmark release locks."""
|
||||
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_benchmark_reproducibility.py to keep release-lock git status classification out of the report renderer."
|
||||
|
||||
|
||||
GENERATED_DIRTY_PREFIXES = (
|
||||
"dist/",
|
||||
"registry/index.json",
|
||||
"registry/packages/",
|
||||
"reports/",
|
||||
"skill_atlas/",
|
||||
"skill-ir/examples/",
|
||||
)
|
||||
|
||||
|
||||
def status_path(line: str) -> str:
|
||||
path = line[3:].strip() if len(line) > 3 else line.strip()
|
||||
if " -> " in path:
|
||||
path = path.rsplit(" -> ", 1)[-1].strip()
|
||||
return path
|
||||
|
||||
|
||||
def is_generated_dirty_path(path: str) -> bool:
|
||||
return any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in GENERATED_DIRTY_PREFIXES)
|
||||
|
||||
|
||||
def git_status(skill_dir: Path) -> dict[str, Any]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["git", "status", "--short", "--untracked-files=all"],
|
||||
cwd=skill_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return {
|
||||
"available": False,
|
||||
"dirty": None,
|
||||
"changed_file_count": None,
|
||||
"source_dirty": None,
|
||||
"source_changed_file_count": None,
|
||||
"generated_dirty": None,
|
||||
"generated_changed_file_count": None,
|
||||
}
|
||||
lines = [line for line in proc.stdout.splitlines() if line.strip()]
|
||||
paths = [status_path(line) for line in lines]
|
||||
generated_lines = [line for line, path in zip(lines, paths) if is_generated_dirty_path(path)]
|
||||
source_lines = [line for line, path in zip(lines, paths) if not is_generated_dirty_path(path)]
|
||||
return {
|
||||
"available": True,
|
||||
"dirty": bool(lines),
|
||||
"changed_file_count": len(lines),
|
||||
"generated_dirty": bool(generated_lines),
|
||||
"generated_changed_file_count": len(generated_lines),
|
||||
"source_dirty": bool(source_lines),
|
||||
"source_changed_file_count": len(source_lines),
|
||||
"sample": lines[:12],
|
||||
"source_sample": source_lines[:12],
|
||||
"generated_sample": generated_lines[:12],
|
||||
"generated_dirty_prefixes": list(GENERATED_DIRTY_PREFIXES),
|
||||
"scope": "generation-time status before this report is written",
|
||||
}
|
||||
|
||||
|
||||
def release_lock_status(status: dict[str, Any], commit: str) -> dict[str, Any]:
|
||||
available = status.get("available") is True
|
||||
source_clean = status.get("source_dirty") is False
|
||||
known_commit = bool(commit and commit != "unknown")
|
||||
ready = available and source_clean and known_commit
|
||||
reasons = []
|
||||
if not available:
|
||||
reasons.append("git status unavailable")
|
||||
if not known_commit:
|
||||
reasons.append("git commit unavailable")
|
||||
if status.get("source_dirty") is True:
|
||||
reasons.append("source files were dirty at generation time")
|
||||
if status.get("source_dirty") is None:
|
||||
reasons.append("working tree cleanliness unknown")
|
||||
if status.get("generated_dirty") is True and status.get("source_dirty") is False:
|
||||
reasons.append("only generated evidence artifacts were dirty at generation time")
|
||||
if not reasons:
|
||||
reasons.append("clean source tree at generation-time HEAD")
|
||||
return {
|
||||
"ready": ready,
|
||||
"commit": commit,
|
||||
"status_scope": status.get("scope", "generation-time status"),
|
||||
"source_changed_file_count": status.get("source_changed_file_count"),
|
||||
"generated_changed_file_count": status.get("generated_changed_file_count"),
|
||||
"reason": "; ".join(reasons),
|
||||
}
|
||||
+129
-93
@@ -1,7 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import csv
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections import Counter, defaultdict
|
||||
@@ -10,6 +9,9 @@ from io import StringIO
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from build_skill_atlas_opportunities import no_route_opportunities
|
||||
from build_skill_atlas_layout import render_html
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
@@ -57,6 +59,7 @@ DEFAULT_SCOPE = {
|
||||
"actionable": True,
|
||||
"scope_reason": "default release-actionable skill",
|
||||
}
|
||||
TELEMETRY_REQUIRED_MATURITIES = {"production", "library", "governed"}
|
||||
|
||||
|
||||
def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]:
|
||||
@@ -217,6 +220,105 @@ def collect_skill(workspace_root: Path, skill_dir: Path, policy: dict[str, Any])
|
||||
}
|
||||
|
||||
|
||||
def load_telemetry_profile(workspace_root: Path, skill_dir: Path, skill: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
report_path = skill_dir / "reports" / "adoption_drift_report.json"
|
||||
rel_report = safe_rel(workspace_root, report_path)
|
||||
maturity = str(skill.get("maturity", "")).casefold()
|
||||
report_present = report_path.exists()
|
||||
telemetry = {
|
||||
"report_present": report_present,
|
||||
"report": rel_report,
|
||||
"risk_band": "missing",
|
||||
"event_count": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"candidate_count": 0,
|
||||
}
|
||||
signals: list[dict[str, Any]] = []
|
||||
if not report_present:
|
||||
if skill.get("actionable") and maturity in TELEMETRY_REQUIRED_MATURITIES:
|
||||
signals.append(
|
||||
{
|
||||
"name": skill["name"],
|
||||
"path": skill["path"],
|
||||
"source": rel_report,
|
||||
"risk_band": "no-data",
|
||||
"signal_types": ["no telemetry"],
|
||||
"recommendation": "Render adoption drift evidence or record a small metadata-only sample before release review.",
|
||||
"actionable": bool(skill.get("actionable")),
|
||||
"scope": str(skill.get("atlas_scope", "")),
|
||||
"summary": {"event_count": 0, "adoption_sample_count": 0},
|
||||
}
|
||||
)
|
||||
return telemetry, signals
|
||||
|
||||
payload = load_json(report_path)
|
||||
summary = payload.get("summary", {}) if isinstance(payload.get("summary"), dict) else {}
|
||||
candidates = payload.get("next_iteration_candidates", [])
|
||||
candidates = candidates if isinstance(candidates, list) else []
|
||||
risk_band = str(summary.get("risk_band") or "unknown")
|
||||
telemetry.update(
|
||||
{
|
||||
"risk_band": risk_band,
|
||||
"event_count": int(summary.get("event_count") or 0),
|
||||
"adoption_sample_count": int(summary.get("adoption_sample_count") or 0),
|
||||
"adoption_rate": summary.get("adoption_rate", 0),
|
||||
"candidate_count": len(candidates),
|
||||
}
|
||||
)
|
||||
|
||||
signal_types: list[str] = []
|
||||
if telemetry["event_count"] == 0 and maturity in TELEMETRY_REQUIRED_MATURITIES:
|
||||
signal_types.append("no telemetry")
|
||||
if int(summary.get("missed_trigger_count") or 0):
|
||||
signal_types.append("missed trigger")
|
||||
if int(summary.get("wrong_trigger_count") or 0):
|
||||
signal_types.append("wrong trigger")
|
||||
if int(summary.get("bad_output_count") or 0):
|
||||
signal_types.append("bad output")
|
||||
if int(summary.get("missing_resource_count") or 0):
|
||||
signal_types.append("missing resource")
|
||||
if int(summary.get("script_error_count") or 0):
|
||||
signal_types.append("script error")
|
||||
if int(summary.get("review_overdue_count") or 0):
|
||||
signal_types.append("review overdue")
|
||||
if risk_band in {"medium", "high"} and not signal_types:
|
||||
signal_types.append("telemetry drift")
|
||||
|
||||
if signal_types:
|
||||
recommendation = "Convert telemetry drift into eval, trust, or owner-review actions."
|
||||
for candidate in candidates:
|
||||
if not isinstance(candidate, dict):
|
||||
continue
|
||||
if str(candidate.get("signal", "")) in signal_types:
|
||||
recommendation = str(candidate.get("recommendation") or recommendation)
|
||||
break
|
||||
signals.append(
|
||||
{
|
||||
"name": skill["name"],
|
||||
"path": skill["path"],
|
||||
"source": rel_report,
|
||||
"risk_band": risk_band,
|
||||
"signal_types": signal_types,
|
||||
"recommendation": recommendation,
|
||||
"actionable": bool(skill.get("actionable")),
|
||||
"scope": str(skill.get("atlas_scope", "")),
|
||||
"summary": {
|
||||
"event_count": telemetry["event_count"],
|
||||
"adoption_sample_count": telemetry["adoption_sample_count"],
|
||||
"adoption_rate": telemetry["adoption_rate"],
|
||||
"missed_trigger_count": int(summary.get("missed_trigger_count") or 0),
|
||||
"wrong_trigger_count": int(summary.get("wrong_trigger_count") or 0),
|
||||
"bad_output_count": int(summary.get("bad_output_count") or 0),
|
||||
"missing_resource_count": int(summary.get("missing_resource_count") or 0),
|
||||
"script_error_count": int(summary.get("script_error_count") or 0),
|
||||
"review_overdue_count": int(summary.get("review_overdue_count") or 0),
|
||||
},
|
||||
}
|
||||
)
|
||||
return telemetry, signals
|
||||
|
||||
|
||||
def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
rows = []
|
||||
collisions = []
|
||||
@@ -351,22 +453,6 @@ def owner_review_gaps(skills: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return gaps
|
||||
|
||||
|
||||
def no_route_opportunities(workspace_root: Path) -> list[dict[str, Any]]:
|
||||
opportunities = []
|
||||
for path in sorted(workspace_root.rglob("failure-cases.md")):
|
||||
if should_skip(path, workspace_root):
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("-"):
|
||||
continue
|
||||
lowered = stripped.casefold()
|
||||
if "no_route" in lowered or "no route" in lowered or "missed" in lowered or "under-trigger" in lowered:
|
||||
opportunities.append({"source": safe_rel(workspace_root, path), "note": stripped.lstrip("- ").strip()})
|
||||
return opportunities[:50]
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status", "actionable", "scope_a", "scope_b"]
|
||||
@@ -378,93 +464,35 @@ def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
|
||||
path.write_text(buffer.getvalue(), encoding="utf-8")
|
||||
|
||||
|
||||
def render_html(payload: dict[str, Any]) -> str:
|
||||
summary = payload["summary"]
|
||||
rows = []
|
||||
for skill in payload["catalog"]["skills"][:80]:
|
||||
rows.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(skill['name'])}</td>"
|
||||
f"<td>{html.escape(skill['path'])}</td>"
|
||||
f"<td>{html.escape(skill.get('owner') or 'missing')}</td>"
|
||||
f"<td>{html.escape(skill.get('maturity') or 'unknown')}</td>"
|
||||
f"<td>{html.escape(skill.get('review_cadence') or 'missing')}</td>"
|
||||
f"<td>{html.escape(skill.get('atlas_scope') or 'release')}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
blockers = payload["actionable_route_collisions"][:20] + payload["actionable_owner_review_gaps"][:20] + payload["actionable_stale_skills"][:20]
|
||||
blocker_items = "".join(
|
||||
f"<li><strong>{html.escape(item.get('name', item.get('skill_a', 'issue')))}</strong> {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', [])))))}</li>"
|
||||
for item in blockers
|
||||
)
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Skill Atlas</title>
|
||||
<style>
|
||||
body {{ margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #172033; background: #fff; }}
|
||||
main {{ max-width: 1120px; margin: 0 auto; padding: 40px 24px; }}
|
||||
h1 {{ font-size: 34px; margin-bottom: 8px; }}
|
||||
.grid {{ display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: 12px; margin: 28px 0; }}
|
||||
.card {{ border: 1px solid #d9e0ea; border-radius: 8px; padding: 16px; background: #f8fafc; }}
|
||||
.card span {{ display: block; color: #697386; font-size: 13px; }}
|
||||
.card strong {{ display: block; font-size: 28px; color: #1B365D; margin-top: 6px; }}
|
||||
table {{ width: 100%; border-collapse: collapse; margin-top: 20px; }}
|
||||
th, td {{ text-align: left; border-bottom: 1px solid #e5e9f0; padding: 10px; vertical-align: top; }}
|
||||
th {{ color: #1B365D; font-size: 13px; }}
|
||||
li {{ margin: 8px 0; }}
|
||||
@media (max-width: 760px) {{ .grid {{ grid-template-columns: 1fr 1fr; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Skill Atlas</h1>
|
||||
<p>Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.</p>
|
||||
<section class="grid">
|
||||
<div class="card"><span>Skills</span><strong>{summary['skill_count']}</strong></div>
|
||||
<div class="card"><span>Actionable</span><strong>{summary['actionable_skill_count']}</strong></div>
|
||||
<div class="card"><span>Route Collisions</span><strong>{summary['actionable_route_collision_count']}</strong></div>
|
||||
<div class="card"><span>Owner Gaps</span><strong>{summary['actionable_owner_gap_count']}</strong></div>
|
||||
<div class="card"><span>Stale Skills</span><strong>{summary['actionable_stale_count']}</strong></div>
|
||||
<div class="card"><span>No-Route Opportunities</span><strong>{summary['no_route_opportunity_count']}</strong></div>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Actionable Issues</h2>
|
||||
<ul>{blocker_items or '<li>No blocking portfolio issues detected.</li>'}</ul>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Full Portfolio Counts</h2>
|
||||
<p>All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, and {summary['stale_count']} total stale signals.</p>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Catalog</h2>
|
||||
<table>
|
||||
<thead><tr><th>Name</th><th>Path</th><th>Owner</th><th>Maturity</th><th>Review</th><th>Scope</th></tr></thead>
|
||||
<tbody>{''.join(rows)}</tbody>
|
||||
</table>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, report_json: Path, threshold: float, today: date) -> dict[str, Any]:
|
||||
workspace_root = workspace_root.resolve()
|
||||
scope_policy = load_scope_policy(workspace_root)
|
||||
skill_dirs = find_skill_dirs(workspace_root)
|
||||
skills = [collect_skill(workspace_root, skill_dir, scope_policy) for skill_dir in skill_dirs]
|
||||
skills = []
|
||||
drift_signals: list[dict[str, Any]] = []
|
||||
telemetry_report_count = 0
|
||||
for skill_dir in skill_dirs:
|
||||
skill = collect_skill(workspace_root, skill_dir, scope_policy)
|
||||
telemetry, signals = load_telemetry_profile(workspace_root, skill_dir, skill)
|
||||
skill["telemetry"] = telemetry
|
||||
telemetry_report_count += 1 if telemetry["report_present"] else 0
|
||||
drift_signals.extend(signals)
|
||||
skills.append(skill)
|
||||
overlap_rows, collisions = route_overlap(skills, threshold)
|
||||
graph = dependency_graph(skills)
|
||||
stale = stale_skills(skills, today)
|
||||
owner_gaps = owner_review_gaps(skills)
|
||||
opportunities = no_route_opportunities(workspace_root)
|
||||
opportunities = no_route_opportunities(
|
||||
workspace_root,
|
||||
drift_signals,
|
||||
should_skip=should_skip,
|
||||
safe_rel=safe_rel,
|
||||
)
|
||||
actionable_skills = [skill for skill in skills if skill.get("actionable")]
|
||||
actionable_collisions = [item for item in collisions if item.get("actionable")]
|
||||
actionable_stale = [item for item in stale if item.get("actionable")]
|
||||
actionable_owner_gaps = [item for item in owner_gaps if item.get("actionable")]
|
||||
actionable_drift_signals = [item for item in drift_signals if item.get("actionable")]
|
||||
summary = {
|
||||
"skill_count": len(skills),
|
||||
"actionable_skill_count": len(actionable_skills),
|
||||
@@ -476,9 +504,13 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor
|
||||
"actionable_stale_count": len(actionable_stale),
|
||||
"shared_resource_count": len(graph["shared_resources"]),
|
||||
"no_route_opportunity_count": len(opportunities),
|
||||
"telemetry_report_count": telemetry_report_count,
|
||||
"drift_signal_count": len(drift_signals),
|
||||
"actionable_drift_signal_count": len(actionable_drift_signals),
|
||||
"non_actionable_issue_count": (len(collisions) - len(actionable_collisions))
|
||||
+ (len(owner_gaps) - len(actionable_owner_gaps))
|
||||
+ (len(stale) - len(actionable_stale)),
|
||||
+ (len(stale) - len(actionable_stale))
|
||||
+ (len(drift_signals) - len(actionable_drift_signals)),
|
||||
}
|
||||
catalog = {
|
||||
"workspace_root": display_path(workspace_root),
|
||||
@@ -499,6 +531,8 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor
|
||||
"actionable_stale_skills": actionable_stale,
|
||||
"owner_review_gaps": owner_gaps,
|
||||
"actionable_owner_review_gaps": actionable_owner_gaps,
|
||||
"drift_signals": drift_signals,
|
||||
"actionable_drift_signals": actionable_drift_signals,
|
||||
"no_route_opportunities": opportunities,
|
||||
"artifacts": {
|
||||
"catalog": display_path(output_dir / "catalog.json"),
|
||||
@@ -506,6 +540,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor
|
||||
"dependency_graph": display_path(output_dir / "dependency_graph.json"),
|
||||
"stale_skills": display_path(output_dir / "stale_skills.json"),
|
||||
"owner_review_gaps": display_path(output_dir / "owner_review_gaps.json"),
|
||||
"drift_signals": display_path(output_dir / "drift_signals.json"),
|
||||
"no_route_opportunities": display_path(output_dir / "no_route_opportunities.json"),
|
||||
"report_json": display_path(report_json),
|
||||
"report_html": display_path(report_html),
|
||||
@@ -517,6 +552,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor
|
||||
(output_dir / "dependency_graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(output_dir / "stale_skills.json").write_text(json.dumps(stale, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(output_dir / "owner_review_gaps.json").write_text(json.dumps(owner_gaps, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(output_dir / "drift_signals.json").write_text(json.dumps(drift_signals, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
(output_dir / "no_route_opportunities.json").write_text(json.dumps(opportunities, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
report_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_html.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""HTML layout helpers for the Skill Atlas report."""
|
||||
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by build_skill_atlas.py to render the static Skill Atlas HTML report."
|
||||
|
||||
|
||||
def render_html(payload: dict[str, Any]) -> str:
|
||||
summary = payload["summary"]
|
||||
rows = []
|
||||
for skill in payload["catalog"]["skills"][:80]:
|
||||
rows.append(
|
||||
"<tr>"
|
||||
f"<td>{html.escape(skill['name'])}</td>"
|
||||
f"<td>{html.escape(skill['path'])}</td>"
|
||||
f"<td>{html.escape(skill.get('owner') or 'missing')}</td>"
|
||||
f"<td>{html.escape(skill.get('maturity') or 'unknown')}</td>"
|
||||
f"<td>{html.escape(skill.get('review_cadence') or 'missing')}</td>"
|
||||
f"<td>{html.escape(skill.get('atlas_scope') or 'release')}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
blockers = (
|
||||
payload["actionable_route_collisions"][:20]
|
||||
+ payload["actionable_owner_review_gaps"][:20]
|
||||
+ payload["actionable_stale_skills"][:20]
|
||||
+ payload["actionable_drift_signals"][:20]
|
||||
)
|
||||
blocker_items = "".join(
|
||||
f"<li><strong>{html.escape(item.get('name', item.get('skill_a', 'issue')))}</strong> {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', item.get('signal_types', []))))))}</li>"
|
||||
for item in blockers
|
||||
)
|
||||
opportunity_items = "".join(
|
||||
(
|
||||
"<li>"
|
||||
f"<strong>{html.escape(item.get('skill') or item.get('source_type', 'opportunity'))}</strong> "
|
||||
f"{html.escape(item.get('note') or item.get('recommendation') or item.get('signal', 'no-route opportunity'))}"
|
||||
f"<br><small>{html.escape(item.get('source', ''))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
for item in payload["no_route_opportunities"][:20]
|
||||
)
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Skill Atlas</title>
|
||||
<style>
|
||||
body {{ margin: 0; font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; color: #172033; background: #fff; }}
|
||||
main {{ max-width: 1120px; margin: 0 auto; padding: 40px 24px; }}
|
||||
h1 {{ font-size: 34px; margin-bottom: 8px; }}
|
||||
.grid {{ display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: 12px; margin: 28px 0; }}
|
||||
.card {{ border: 1px solid #d9e0ea; border-radius: 8px; padding: 16px; background: #f8fafc; }}
|
||||
.card span {{ display: block; color: #697386; font-size: 13px; }}
|
||||
.card strong {{ display: block; font-size: 28px; color: #1B365D; margin-top: 6px; }}
|
||||
table {{ width: 100%; border-collapse: collapse; margin-top: 20px; }}
|
||||
th, td {{ text-align: left; border-bottom: 1px solid #e5e9f0; padding: 10px; vertical-align: top; }}
|
||||
th {{ color: #1B365D; font-size: 13px; }}
|
||||
li {{ margin: 8px 0; }}
|
||||
@media (max-width: 760px) {{ .grid {{ grid-template-columns: 1fr 1fr; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>Skill Atlas</h1>
|
||||
<p>Portfolio-level review for route overlap, stale ownership, shared resources, telemetry drift, and no-route opportunities.</p>
|
||||
<section class="grid">
|
||||
<div class="card"><span>Skills</span><strong>{summary['skill_count']}</strong></div>
|
||||
<div class="card"><span>Actionable</span><strong>{summary['actionable_skill_count']}</strong></div>
|
||||
<div class="card"><span>Route Collisions</span><strong>{summary['actionable_route_collision_count']}</strong></div>
|
||||
<div class="card"><span>Owner Gaps</span><strong>{summary['actionable_owner_gap_count']}</strong></div>
|
||||
<div class="card"><span>Stale Skills</span><strong>{summary['actionable_stale_count']}</strong></div>
|
||||
<div class="card"><span>Drift Signals</span><strong>{summary['actionable_drift_signal_count']}</strong></div>
|
||||
<div class="card"><span>No-Route Opportunities</span><strong>{summary['no_route_opportunity_count']}</strong></div>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Actionable Issues</h2>
|
||||
<ul>{blocker_items or '<li>No blocking portfolio issues detected.</li>'}</ul>
|
||||
</section>
|
||||
<section>
|
||||
<h2>No-Route Opportunities</h2>
|
||||
<p>Missed-trigger telemetry and explicit failure cases become candidate routing work without storing raw prompts or outputs.</p>
|
||||
<ul>{opportunity_items or '<li>No no-route opportunities detected.</li>'}</ul>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Full Portfolio Counts</h2>
|
||||
<p>All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, {summary['stale_count']} total stale signals, and {summary['drift_signal_count']} telemetry drift signals.</p>
|
||||
</section>
|
||||
<section>
|
||||
<h2>Catalog</h2>
|
||||
<table>
|
||||
<thead><tr><th>Name</th><th>Path</th><th>Owner</th><th>Maturity</th><th>Review</th><th>Scope</th></tr></thead>
|
||||
<tbody>{''.join(rows)}</tbody>
|
||||
</table>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
@@ -0,0 +1,82 @@
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by build_skill_atlas.py to keep no-route opportunity detection out of the atlas CLI."
|
||||
|
||||
|
||||
SkipPredicate = Callable[[Path, Path], bool]
|
||||
RelFormatter = Callable[[Path, Path], str]
|
||||
|
||||
|
||||
def failure_case_no_route_opportunities(
|
||||
workspace_root: Path,
|
||||
*,
|
||||
should_skip: SkipPredicate,
|
||||
safe_rel: RelFormatter,
|
||||
) -> list[dict[str, Any]]:
|
||||
opportunities = []
|
||||
for path in sorted(workspace_root.rglob("failure-cases.md")):
|
||||
if should_skip(path, workspace_root):
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped.startswith("-"):
|
||||
continue
|
||||
lowered = stripped.casefold()
|
||||
if "no_route" in lowered or "no route" in lowered or "missed" in lowered or "under-trigger" in lowered:
|
||||
opportunities.append(
|
||||
{
|
||||
"source_type": "failure-case",
|
||||
"source": safe_rel(workspace_root, path),
|
||||
"note": stripped.lstrip("- ").strip(),
|
||||
"actionable": True,
|
||||
"privacy_contract": "source note only; raw prompts are not required",
|
||||
}
|
||||
)
|
||||
return opportunities[:50]
|
||||
|
||||
|
||||
def telemetry_no_route_opportunities(drift_signals: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
opportunities = []
|
||||
for signal in drift_signals:
|
||||
signal_types = {str(item) for item in signal.get("signal_types", [])}
|
||||
if not {"missed trigger", "under trigger"} & signal_types:
|
||||
continue
|
||||
summary = signal.get("summary", {}) if isinstance(signal.get("summary"), dict) else {}
|
||||
opportunities.append(
|
||||
{
|
||||
"source_type": "telemetry",
|
||||
"source": str(signal.get("source", "")),
|
||||
"skill": str(signal.get("name", "")),
|
||||
"path": str(signal.get("path", "")),
|
||||
"signal": "missed trigger",
|
||||
"missed_trigger_count": int(summary.get("missed_trigger_count") or 0),
|
||||
"recommendation": str(
|
||||
signal.get("recommendation")
|
||||
or "Add missed prompts to trigger eval and evaluate whether a new skill route is needed."
|
||||
),
|
||||
"actionable": bool(signal.get("actionable")),
|
||||
"scope": str(signal.get("scope", "")),
|
||||
"privacy_contract": "metadata-only telemetry; no raw prompt, output, transcript, or note is stored",
|
||||
}
|
||||
)
|
||||
return opportunities
|
||||
|
||||
|
||||
def no_route_opportunities(
|
||||
workspace_root: Path,
|
||||
drift_signals: list[dict[str, Any]],
|
||||
*,
|
||||
should_skip: SkipPredicate,
|
||||
safe_rel: RelFormatter,
|
||||
) -> list[dict[str, Any]]:
|
||||
opportunities = failure_case_no_route_opportunities(
|
||||
workspace_root,
|
||||
should_skip=should_skip,
|
||||
safe_rel=safe_rel,
|
||||
)
|
||||
opportunities.extend(telemetry_no_route_opportunities(drift_signals))
|
||||
return opportunities[:80]
|
||||
@@ -20,6 +20,7 @@ SAFE_ENV_KEYS = (
|
||||
"TERM",
|
||||
"TMP",
|
||||
"TMPDIR",
|
||||
"TZ",
|
||||
)
|
||||
DEFAULT_TARGETS = [
|
||||
"eval",
|
||||
@@ -29,14 +30,20 @@ DEFAULT_TARGETS = [
|
||||
"description-optimization",
|
||||
"description-optimization-check",
|
||||
"promotion-check",
|
||||
"python-compat-check",
|
||||
"architecture-maintainability-check",
|
||||
"yao-cli-check",
|
||||
"yao-cli-world-class-check",
|
||||
"skill-overview-check",
|
||||
"skill-interpretation-check",
|
||||
"skill-report-metrics-check",
|
||||
"skill-report-charts-check",
|
||||
"html-rendering-check",
|
||||
"skill-ir-check",
|
||||
"compiler-check",
|
||||
"output-eval-check",
|
||||
"output-execution-check",
|
||||
"output-review-kit-check",
|
||||
"output-review-adjudication-check",
|
||||
"runtime-conformance-check",
|
||||
"runtime-permission-check",
|
||||
@@ -48,8 +55,28 @@ DEFAULT_TARGETS = [
|
||||
"upgrade-check",
|
||||
"review-viewer-check",
|
||||
"review-studio-check",
|
||||
"skill-os2-audit-check",
|
||||
"skill-os2-coverage-check",
|
||||
"world-class-evidence-check",
|
||||
"world-class-ledger-check",
|
||||
"world-class-intake-check",
|
||||
"world-class-submission-kit-check",
|
||||
"world-class-preflight-check",
|
||||
"world-class-submission-review-check",
|
||||
"world-class-runbook-check",
|
||||
"world-class-claim-guard-check",
|
||||
"benchmark-reproducibility-check",
|
||||
"evidence-consistency-check",
|
||||
"feedback-check",
|
||||
"adaptation-safety-check",
|
||||
"skillops-opportunity-check",
|
||||
"daily-skillops-check",
|
||||
"weekly-curator-check",
|
||||
"adoption-drift-check",
|
||||
"telemetry-import-check",
|
||||
"telemetry-emit-check",
|
||||
"telemetry-hooks-check",
|
||||
"telemetry-native-host-check",
|
||||
"review-waivers-check",
|
||||
"review-annotations-check",
|
||||
"baseline-compare-check",
|
||||
|
||||
+5
-284
@@ -10,96 +10,15 @@ try:
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
from skill_ir_paths import find_skill_ir as find_skill_ir_document
|
||||
from compile_skill_targets import TARGET_TRANSFORMS, target_native_contract, target_permission_contract
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
COMPILER_SCHEMA_VERSION = "1.0"
|
||||
COMPILER_NAME = "yao-skill-ir-compiler"
|
||||
|
||||
|
||||
TARGET_TRANSFORMS: dict[str, dict[str, Any]] = {
|
||||
"openai": {
|
||||
"adapter_mode": "metadata-adapter",
|
||||
"generated_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"metadata_mapping": {
|
||||
"display_name": "targets/openai/agents/openai.yaml::interface.display_name",
|
||||
"default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt",
|
||||
"activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode",
|
||||
"execution": "targets/openai/agents/openai.yaml::compatibility.execution_context",
|
||||
"trust": "targets/openai/agents/openai.yaml::compatibility.trust_level",
|
||||
"permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract",
|
||||
"degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": ["client-native script permission prompts are represented as permission contract metadata"],
|
||||
},
|
||||
"claude": {
|
||||
"adapter_mode": "neutral-source-plus-adapter",
|
||||
"generated_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"metadata_mapping": {
|
||||
"display_name": "targets/claude/adapter.json::display_name",
|
||||
"default_prompt": "targets/claude/adapter.json::default_prompt",
|
||||
"activation": "targets/claude/adapter.json::activation_mode",
|
||||
"execution": "targets/claude/adapter.json::execution_context",
|
||||
"trust": "targets/claude/adapter.json::trust_level",
|
||||
"permissions": "targets/claude/adapter.json::target_permission_contract",
|
||||
"degradation": "targets/claude/adapter.json::degradation_strategy",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": ["vendor-native metadata fields are carried as adapter JSON and README notes"],
|
||||
},
|
||||
"generic": {
|
||||
"adapter_mode": "agent-skills-compatible",
|
||||
"generated_files": ["targets/generic/adapter.json"],
|
||||
"metadata_mapping": {
|
||||
"display_name": "targets/generic/adapter.json::display_name",
|
||||
"default_prompt": "targets/generic/adapter.json::default_prompt",
|
||||
"activation": "targets/generic/adapter.json::activation_mode",
|
||||
"execution": "targets/generic/adapter.json::execution_context",
|
||||
"trust": "targets/generic/adapter.json::trust_level",
|
||||
"permissions": "targets/generic/adapter.json::target_permission_contract",
|
||||
"degradation": "targets/generic/adapter.json::degradation_strategy",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": [],
|
||||
},
|
||||
"agent-skills-compatible": {
|
||||
"adapter_mode": "neutral-agent-skills-source",
|
||||
"generated_files": ["SKILL.md", "agents/interface.yaml"],
|
||||
"metadata_mapping": {
|
||||
"name": "SKILL.md::frontmatter.name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"interface": "agents/interface.yaml",
|
||||
"manifest": "manifest.json",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": [],
|
||||
},
|
||||
"agent-skills": {
|
||||
"adapter_mode": "neutral-agent-skills-source",
|
||||
"generated_files": ["SKILL.md", "agents/interface.yaml"],
|
||||
"metadata_mapping": {
|
||||
"name": "SKILL.md::frontmatter.name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"interface": "agents/interface.yaml",
|
||||
"manifest": "manifest.json",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": [],
|
||||
},
|
||||
"vscode": {
|
||||
"adapter_mode": "agent-skills-compatible-project-or-user-scope",
|
||||
"generated_files": ["SKILL.md", "agents/interface.yaml"],
|
||||
"metadata_mapping": {
|
||||
"name": "folder-name-and-SKILL.md::frontmatter.name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"interface": "agents/interface.yaml",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": ["VS Code installation scope is documented but not installed by this compiler"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path = ROOT) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
@@ -146,20 +65,7 @@ def read_frontmatter(path: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "skill-ir.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{name}.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
|
||||
]
|
||||
seen: set[Path] = set()
|
||||
for path in candidates:
|
||||
if path in seen:
|
||||
continue
|
||||
seen.add(path)
|
||||
payload = load_json(path)
|
||||
if payload:
|
||||
return payload, display_path(path, skill_dir)
|
||||
return {}, "frontmatter-fallback"
|
||||
return find_skill_ir_document(skill_dir, name, fallback_source="frontmatter-fallback")
|
||||
|
||||
|
||||
def count_list(payload: dict[str, Any], key: str) -> int:
|
||||
@@ -246,191 +152,6 @@ def permission_contract(skill_dir: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
TARGET_PERMISSION_MODELS = {
|
||||
"openai": {
|
||||
"model": "metadata-only",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json",
|
||||
"operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package.",
|
||||
},
|
||||
"claude": {
|
||||
"model": "neutral-source-plus-adapter",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/claude/adapter.json::target_permission_contract and README notes",
|
||||
"operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review.",
|
||||
},
|
||||
"generic": {
|
||||
"model": "agent-skills-compatible-metadata",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/generic/adapter.json::target_permission_contract",
|
||||
"operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
TARGET_NATIVE_MODELS = {
|
||||
"openai": {
|
||||
"native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source",
|
||||
"activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.",
|
||||
"resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
|
||||
"script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
|
||||
"permission_enforcement": "metadata-only",
|
||||
"install_scope": "plugin or skill package consumer",
|
||||
"review_artifacts": ["targets/openai/agents/openai.yaml", "targets/openai/adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.",
|
||||
"unsupported_native_features": [
|
||||
"client-native permission prompts",
|
||||
"provider-executed scripts",
|
||||
],
|
||||
},
|
||||
"claude": {
|
||||
"native_surface": "Claude-compatible neutral source folder with adapter notes",
|
||||
"activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.",
|
||||
"resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
|
||||
"script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
|
||||
"permission_enforcement": "metadata-fallback",
|
||||
"install_scope": "user or project skill directory",
|
||||
"review_artifacts": ["targets/claude/README.md", "targets/claude/adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.",
|
||||
"unsupported_native_features": [
|
||||
"vendor-native permission enforcement",
|
||||
"provider-specific execution transforms",
|
||||
],
|
||||
},
|
||||
"generic": {
|
||||
"native_surface": "Agent Skills compatible neutral package",
|
||||
"activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.",
|
||||
"resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
|
||||
"script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
|
||||
"permission_enforcement": "consumer-enforced-or-metadata-only",
|
||||
"install_scope": "generic Agent Skills compatible root",
|
||||
"review_artifacts": ["targets/generic/adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "Neutral source is the runtime fallback.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
"agent-skills-compatible": {
|
||||
"native_surface": "Agent Skills standard source tree",
|
||||
"activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.",
|
||||
"resource_strategy": "Keep optional directories as relative resources next to SKILL.md.",
|
||||
"script_strategy": "Scripts remain local optional resources and should advertise --help when executable.",
|
||||
"permission_enforcement": "consumer-enforced-or-metadata-only",
|
||||
"install_scope": "Agent Skills source root",
|
||||
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
|
||||
"fallback_behavior": "The source tree itself is the target artifact.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
"agent-skills": {
|
||||
"native_surface": "Agent Skills standard source tree",
|
||||
"activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.",
|
||||
"resource_strategy": "Keep optional directories as relative resources next to SKILL.md.",
|
||||
"script_strategy": "Scripts remain local optional resources and should advertise --help when executable.",
|
||||
"permission_enforcement": "consumer-enforced-or-metadata-only",
|
||||
"install_scope": "Agent Skills source root",
|
||||
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
|
||||
"fallback_behavior": "The source tree itself is the target artifact.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
"vscode": {
|
||||
"native_surface": "VS Code/Copilot Agent Skills project or user scope",
|
||||
"activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.",
|
||||
"resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
|
||||
"script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
|
||||
"permission_enforcement": "client-or-workspace-trust",
|
||||
"install_scope": "VS Code user or project skills directory",
|
||||
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
|
||||
"fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.",
|
||||
"unsupported_native_features": [
|
||||
"automatic VS Code installation",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def target_permission_contract(target: str, permissions: dict[str, Any]) -> dict[str, Any]:
|
||||
model = TARGET_PERMISSION_MODELS.get(
|
||||
target,
|
||||
{
|
||||
"model": "metadata-only",
|
||||
"native_enforcement": False,
|
||||
"representation": "adapter metadata",
|
||||
"operator_note": "Permission semantics are preserved as metadata for reviewer visibility.",
|
||||
},
|
||||
)
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"target": target,
|
||||
"permission_model": model["model"],
|
||||
"native_enforcement": model["native_enforcement"],
|
||||
"representation": model["representation"],
|
||||
"review_required": bool(permissions.get("review_required")),
|
||||
"declared_capabilities": permissions.get("declared_capabilities", []),
|
||||
"capability_counts": {
|
||||
name: item.get("script_count", 0)
|
||||
for name, item in permissions.get("capabilities", {}).items()
|
||||
},
|
||||
"evidence": permissions.get("source", ""),
|
||||
"operator_note": model["operator_note"],
|
||||
}
|
||||
|
||||
|
||||
def target_native_contract(
|
||||
target: str,
|
||||
profile: dict[str, Any],
|
||||
contract: dict[str, Any],
|
||||
target_permissions: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
model = TARGET_NATIVE_MODELS.get(
|
||||
target,
|
||||
{
|
||||
"native_surface": "adapter metadata",
|
||||
"activation_policy": "Carry activation semantics as metadata for the target consumer.",
|
||||
"resource_strategy": "Preserve neutral package resources.",
|
||||
"script_strategy": "Expose script metadata for reviewer visibility.",
|
||||
"permission_enforcement": "metadata-only",
|
||||
"install_scope": "target consumer",
|
||||
"review_artifacts": ["adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "Use neutral source package.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
)
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"target": target,
|
||||
"native_surface": model["native_surface"],
|
||||
"activation": {
|
||||
"policy": model["activation_policy"],
|
||||
"trigger_description": contract.get("trigger", {}).get("description", ""),
|
||||
"manual_activation_supported": True,
|
||||
"automatic_activation_note": "Depends on the target client route/catalog implementation.",
|
||||
},
|
||||
"resources": {
|
||||
"strategy": model["resource_strategy"],
|
||||
"counts": contract.get("resources", {}).get("counts", {}),
|
||||
"generated_files": profile.get("generated_files", []),
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": model["script_strategy"],
|
||||
"script_count": contract.get("resources", {}).get("counts", {}).get("scripts", 0),
|
||||
"help_smoke_failed_count": contract.get("permissions", {}).get("help_smoke", {}).get("failed_count", 0),
|
||||
},
|
||||
"permissions": {
|
||||
"enforcement": model["permission_enforcement"],
|
||||
"native_enforcement": bool(target_permissions.get("native_enforcement")),
|
||||
"declared_capabilities": target_permissions.get("declared_capabilities", []),
|
||||
"review_required": bool(target_permissions.get("review_required")),
|
||||
},
|
||||
"review": {
|
||||
"artifacts": model["review_artifacts"],
|
||||
"fallback_behavior": model["fallback_behavior"],
|
||||
"unsupported_native_features": [
|
||||
*model.get("unsupported_native_features", []),
|
||||
*profile.get("unsupported_features", []),
|
||||
],
|
||||
},
|
||||
"install_scope": model["install_scope"],
|
||||
}
|
||||
|
||||
|
||||
def load_sources(skill_dir: Path) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
|
||||
@@ -545,7 +266,7 @@ def compile_target_contract(skill_dir: Path, target: str) -> dict[str, Any]:
|
||||
"preserved_semantics": [],
|
||||
"unsupported_features": [],
|
||||
}
|
||||
if target not in declared and not (target == "agent-skills" and "agent-skills-compatible" in declared):
|
||||
if target not in declared and not (target in {"agent-skills", "vscode"} and "agent-skills-compatible" in declared):
|
||||
warnings.append(f"Target is not declared in Skill IR or interface metadata: {target}")
|
||||
if not sources["ir"]:
|
||||
warnings.append("Skill IR is missing; compiler used frontmatter fallback")
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by compile_skill for target platform model and contract builders."
|
||||
|
||||
|
||||
TARGET_TRANSFORMS: dict[str, dict[str, Any]] = {
|
||||
"openai": {
|
||||
"adapter_mode": "metadata-adapter",
|
||||
"generated_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"metadata_mapping": {
|
||||
"display_name": "targets/openai/agents/openai.yaml::interface.display_name",
|
||||
"default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt",
|
||||
"activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode",
|
||||
"execution": "targets/openai/agents/openai.yaml::compatibility.execution_context",
|
||||
"trust": "targets/openai/agents/openai.yaml::compatibility.trust_level",
|
||||
"permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract",
|
||||
"degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": ["client-native script permission prompts are represented as permission contract metadata"],
|
||||
},
|
||||
"claude": {
|
||||
"adapter_mode": "neutral-source-plus-adapter",
|
||||
"generated_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"metadata_mapping": {
|
||||
"display_name": "targets/claude/adapter.json::display_name",
|
||||
"default_prompt": "targets/claude/adapter.json::default_prompt",
|
||||
"activation": "targets/claude/adapter.json::activation_mode",
|
||||
"execution": "targets/claude/adapter.json::execution_context",
|
||||
"trust": "targets/claude/adapter.json::trust_level",
|
||||
"permissions": "targets/claude/adapter.json::target_permission_contract",
|
||||
"degradation": "targets/claude/adapter.json::degradation_strategy",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": ["vendor-native metadata fields are carried as adapter JSON and README notes"],
|
||||
},
|
||||
"generic": {
|
||||
"adapter_mode": "agent-skills-compatible",
|
||||
"generated_files": ["targets/generic/adapter.json"],
|
||||
"metadata_mapping": {
|
||||
"display_name": "targets/generic/adapter.json::display_name",
|
||||
"default_prompt": "targets/generic/adapter.json::default_prompt",
|
||||
"activation": "targets/generic/adapter.json::activation_mode",
|
||||
"execution": "targets/generic/adapter.json::execution_context",
|
||||
"trust": "targets/generic/adapter.json::trust_level",
|
||||
"permissions": "targets/generic/adapter.json::target_permission_contract",
|
||||
"degradation": "targets/generic/adapter.json::degradation_strategy",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": [],
|
||||
},
|
||||
"agent-skills-compatible": {
|
||||
"adapter_mode": "neutral-agent-skills-source",
|
||||
"generated_files": ["SKILL.md", "agents/interface.yaml"],
|
||||
"metadata_mapping": {
|
||||
"name": "SKILL.md::frontmatter.name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"interface": "agents/interface.yaml",
|
||||
"manifest": "manifest.json",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": [],
|
||||
},
|
||||
"agent-skills": {
|
||||
"adapter_mode": "neutral-agent-skills-source",
|
||||
"generated_files": ["SKILL.md", "agents/interface.yaml"],
|
||||
"metadata_mapping": {
|
||||
"name": "SKILL.md::frontmatter.name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"interface": "agents/interface.yaml",
|
||||
"manifest": "manifest.json",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": [],
|
||||
},
|
||||
"vscode": {
|
||||
"adapter_mode": "vscode-agent-skills-adapter",
|
||||
"generated_files": ["targets/vscode/adapter.json", "targets/vscode/README.md"],
|
||||
"metadata_mapping": {
|
||||
"name": "folder-name-and-SKILL.md::frontmatter.name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"interface": "agents/interface.yaml",
|
||||
"permissions": "targets/vscode/adapter.json::target_permission_contract",
|
||||
"install_scope": "targets/vscode/README.md",
|
||||
},
|
||||
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
|
||||
"unsupported_features": ["VS Code installation scope is documented but not installed by this compiler"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
TARGET_PERMISSION_MODELS = {
|
||||
"openai": {
|
||||
"model": "metadata-only",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json",
|
||||
"operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package.",
|
||||
},
|
||||
"claude": {
|
||||
"model": "neutral-source-plus-adapter",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/claude/adapter.json::target_permission_contract and README notes",
|
||||
"operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review.",
|
||||
},
|
||||
"generic": {
|
||||
"model": "agent-skills-compatible-metadata",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/generic/adapter.json::target_permission_contract",
|
||||
"operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review.",
|
||||
},
|
||||
"vscode": {
|
||||
"model": "vscode-workspace-trust-plus-metadata",
|
||||
"native_enforcement": False,
|
||||
"representation": "targets/vscode/adapter.json::target_permission_contract and targets/vscode/README.md install notes",
|
||||
"operator_note": "VS Code target relies on project or user skill installation plus VS Code workspace trust; Yao preserves permission metadata for reviewer and installer checks.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
TARGET_NATIVE_MODELS = {
|
||||
"openai": {
|
||||
"native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source",
|
||||
"activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.",
|
||||
"resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
|
||||
"script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
|
||||
"permission_enforcement": "metadata-only",
|
||||
"install_scope": "plugin or skill package consumer",
|
||||
"review_artifacts": ["targets/openai/agents/openai.yaml", "targets/openai/adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.",
|
||||
"unsupported_native_features": [
|
||||
"client-native permission prompts",
|
||||
"provider-executed scripts",
|
||||
],
|
||||
},
|
||||
"claude": {
|
||||
"native_surface": "Claude-compatible neutral source folder with adapter notes",
|
||||
"activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.",
|
||||
"resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
|
||||
"script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
|
||||
"permission_enforcement": "metadata-fallback",
|
||||
"install_scope": "user or project skill directory",
|
||||
"review_artifacts": ["targets/claude/README.md", "targets/claude/adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.",
|
||||
"unsupported_native_features": [
|
||||
"vendor-native permission enforcement",
|
||||
"provider-specific execution transforms",
|
||||
],
|
||||
},
|
||||
"generic": {
|
||||
"native_surface": "Agent Skills compatible neutral package",
|
||||
"activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.",
|
||||
"resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
|
||||
"script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
|
||||
"permission_enforcement": "consumer-enforced-or-metadata-only",
|
||||
"install_scope": "generic Agent Skills compatible root",
|
||||
"review_artifacts": ["targets/generic/adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "Neutral source is the runtime fallback.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
"agent-skills-compatible": {
|
||||
"native_surface": "Agent Skills standard source tree",
|
||||
"activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.",
|
||||
"resource_strategy": "Keep optional directories as relative resources next to SKILL.md.",
|
||||
"script_strategy": "Scripts remain local optional resources and should advertise --help when executable.",
|
||||
"permission_enforcement": "consumer-enforced-or-metadata-only",
|
||||
"install_scope": "Agent Skills source root",
|
||||
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
|
||||
"fallback_behavior": "The source tree itself is the target artifact.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
"agent-skills": {
|
||||
"native_surface": "Agent Skills standard source tree",
|
||||
"activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.",
|
||||
"resource_strategy": "Keep optional directories as relative resources next to SKILL.md.",
|
||||
"script_strategy": "Scripts remain local optional resources and should advertise --help when executable.",
|
||||
"permission_enforcement": "consumer-enforced-or-metadata-only",
|
||||
"install_scope": "Agent Skills source root",
|
||||
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
|
||||
"fallback_behavior": "The source tree itself is the target artifact.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
"vscode": {
|
||||
"native_surface": "VS Code/Copilot Agent Skills project or user scope",
|
||||
"activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.",
|
||||
"resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
|
||||
"script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
|
||||
"permission_enforcement": "client-or-workspace-trust",
|
||||
"install_scope": "VS Code user or project skills directory",
|
||||
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
|
||||
"fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.",
|
||||
"unsupported_native_features": [
|
||||
"automatic VS Code installation",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def target_permission_contract(target: str, permissions: dict[str, Any]) -> dict[str, Any]:
|
||||
model = TARGET_PERMISSION_MODELS.get(
|
||||
target,
|
||||
{
|
||||
"model": "metadata-only",
|
||||
"native_enforcement": False,
|
||||
"representation": "adapter metadata",
|
||||
"operator_note": "Permission semantics are preserved as metadata for reviewer visibility.",
|
||||
},
|
||||
)
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"target": target,
|
||||
"permission_model": model["model"],
|
||||
"native_enforcement": model["native_enforcement"],
|
||||
"representation": model["representation"],
|
||||
"review_required": bool(permissions.get("review_required")),
|
||||
"declared_capabilities": permissions.get("declared_capabilities", []),
|
||||
"capability_counts": {
|
||||
name: item.get("script_count", 0)
|
||||
for name, item in permissions.get("capabilities", {}).items()
|
||||
},
|
||||
"evidence": permissions.get("source", ""),
|
||||
"operator_note": model["operator_note"],
|
||||
}
|
||||
|
||||
|
||||
def target_native_contract(
|
||||
target: str,
|
||||
profile: dict[str, Any],
|
||||
contract: dict[str, Any],
|
||||
target_permissions: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
model = TARGET_NATIVE_MODELS.get(
|
||||
target,
|
||||
{
|
||||
"native_surface": "adapter metadata",
|
||||
"activation_policy": "Carry activation semantics as metadata for the target consumer.",
|
||||
"resource_strategy": "Preserve neutral package resources.",
|
||||
"script_strategy": "Expose script metadata for reviewer visibility.",
|
||||
"permission_enforcement": "metadata-only",
|
||||
"install_scope": "target consumer",
|
||||
"review_artifacts": ["adapter.json", "reports/review-studio.html"],
|
||||
"fallback_behavior": "Use neutral source package.",
|
||||
"unsupported_native_features": [],
|
||||
},
|
||||
)
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"target": target,
|
||||
"native_surface": model["native_surface"],
|
||||
"activation": {
|
||||
"policy": model["activation_policy"],
|
||||
"trigger_description": contract.get("trigger", {}).get("description", ""),
|
||||
"manual_activation_supported": True,
|
||||
"automatic_activation_note": "Depends on the target client route/catalog implementation.",
|
||||
},
|
||||
"resources": {
|
||||
"strategy": model["resource_strategy"],
|
||||
"counts": contract.get("resources", {}).get("counts", {}),
|
||||
"generated_files": profile.get("generated_files", []),
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": model["script_strategy"],
|
||||
"script_count": contract.get("resources", {}).get("counts", {}).get("scripts", 0),
|
||||
"help_smoke_failed_count": contract.get("permissions", {}).get("help_smoke", {}).get("failed_count", 0),
|
||||
},
|
||||
"permissions": {
|
||||
"enforcement": model["permission_enforcement"],
|
||||
"native_enforcement": bool(target_permissions.get("native_enforcement")),
|
||||
"declared_capabilities": target_permissions.get("declared_capabilities", []),
|
||||
"review_required": bool(target_permissions.get("review_required")),
|
||||
},
|
||||
"review": {
|
||||
"artifacts": model["review_artifacts"],
|
||||
"fallback_behavior": model["fallback_behavior"],
|
||||
"unsupported_native_features": [
|
||||
*model.get("unsupported_native_features", []),
|
||||
*profile.get("unsupported_features", []),
|
||||
],
|
||||
},
|
||||
"install_scope": model["install_scope"],
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TEXT_EXTS = {".md", ".txt", ".yaml", ".yml", ".json", ".py", ".sh", ".js", ".ts"}
|
||||
TEXT_EXTS = {".css", ".md", ".txt", ".yaml", ".yml", ".json", ".py", ".sh", ".js", ".ts"}
|
||||
IGNORED_RELATIVE_DIRS = {
|
||||
Path("reports") / "release_snapshots",
|
||||
Path("tests") / "tmp",
|
||||
@@ -26,6 +26,27 @@ PACKAGE_PATHS = (
|
||||
"input",
|
||||
"outputs",
|
||||
)
|
||||
IGNORED_FILE_PATTERNS = {
|
||||
"reports/benchmark_reproducibility*.json",
|
||||
"reports/benchmark_reproducibility*.md",
|
||||
"reports/context_budget*.json",
|
||||
"reports/context_budget*.md",
|
||||
"reports/evidence_consistency*.json",
|
||||
"reports/evidence_consistency*.md",
|
||||
"reports/review-studio*.html",
|
||||
"reports/review-studio*.json",
|
||||
"reports/review-viewer*.html",
|
||||
"reports/review-viewer*.json",
|
||||
"reports/skill-interpretation*.html",
|
||||
"reports/skill-interpretation*.json",
|
||||
"reports/skill-overview*.html",
|
||||
"reports/skill-overview*.json",
|
||||
"reports/world_class_evidence_preflight*.json",
|
||||
"reports/world_class_evidence_preflight*.md",
|
||||
"reports/world_class_evidence_preflight*.html",
|
||||
"reports/*pattern-analysis*.md",
|
||||
"reports/*research-plan*.md",
|
||||
}
|
||||
|
||||
|
||||
def estimate_tokens(text: str) -> int:
|
||||
@@ -61,6 +82,8 @@ def should_ignore(path: Path, skill_dir: Path) -> bool:
|
||||
rel = path.relative_to(skill_dir)
|
||||
if any(rel == ignored or ignored in rel.parents for ignored in IGNORED_RELATIVE_DIRS):
|
||||
return True
|
||||
if any(rel.match(pattern) for pattern in IGNORED_FILE_PATTERNS):
|
||||
return True
|
||||
return len(rel.parts) >= 2 and rel.parts[0] == "tests" and rel.parts[1].startswith("tmp_")
|
||||
|
||||
|
||||
|
||||
+52
-140
@@ -3,10 +3,12 @@ import argparse
|
||||
import json
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from pathlib import Path, PurePosixPath
|
||||
import yaml
|
||||
|
||||
from compile_skill import compile_target_contract
|
||||
from cross_packager_contracts import PLATFORM_CONTRACTS
|
||||
from skill_ir_paths import find_skill_ir as find_skill_ir_document
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path) -> str:
|
||||
@@ -48,20 +50,15 @@ def read_interface(skill_dir: Path) -> dict:
|
||||
|
||||
|
||||
def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict, str]:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "skill-ir.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{name}.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
|
||||
]
|
||||
seen = set()
|
||||
for path in candidates:
|
||||
if path in seen:
|
||||
continue
|
||||
seen.add(path)
|
||||
payload = read_json(path)
|
||||
if payload:
|
||||
return payload, display_path(path, skill_dir)
|
||||
return {}, "frontmatter-fallback"
|
||||
return find_skill_ir_document(skill_dir, name, fallback_source="frontmatter-fallback")
|
||||
|
||||
|
||||
def package_name_from_manifest(manifest: dict, skill_dir: Path) -> str:
|
||||
name = str(manifest.get("name") or "").strip()
|
||||
if name:
|
||||
return name
|
||||
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
|
||||
return str(frontmatter.get("name") or skill_dir.name)
|
||||
|
||||
|
||||
def require_fields(payload: dict, fields: list[str], label: str) -> None:
|
||||
@@ -147,13 +144,17 @@ def build_semantic_contract(
|
||||
"targets": target_values,
|
||||
"source_files_count": count_list(ir, "source_files") if ir else 0,
|
||||
}
|
||||
alias_declared = (
|
||||
platform in {"agent-skills", "vscode"} and "agent-skills-compatible" in target_values
|
||||
)
|
||||
semantic_parity = {
|
||||
"source": "skill-ir" if ir else "frontmatter-fallback",
|
||||
"ir_source": ir_source,
|
||||
"name_matches_ir": bool(ir) and frontmatter_name == name,
|
||||
"description_matches_ir": bool(ir) and frontmatter_description == description,
|
||||
"platform_declared_in_ir": platform in target_values
|
||||
or (platform == "generic" and "agent-skills-compatible" in target_values),
|
||||
or (platform == "generic" and "agent-skills-compatible" in target_values)
|
||||
or alias_declared,
|
||||
"platform_declared_in_interface": platform in adapter_targets,
|
||||
"display_name_present": bool(interface.get("display_name")),
|
||||
"default_prompt_present": bool(interface.get("default_prompt")),
|
||||
@@ -209,7 +210,7 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
"description": semantic["description"],
|
||||
"version": manifest.get("version") or frontmatter.get("version", "1.0.0"),
|
||||
"platform": platform,
|
||||
"skill_root": skill_dir.name,
|
||||
"skill_root": semantic["name"],
|
||||
"job_to_be_done": semantic["job_to_be_done"],
|
||||
"ir_source": semantic["ir_source"],
|
||||
"ir_schema_version": semantic["ir_schema_version"],
|
||||
@@ -250,115 +251,7 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
PLATFORM_CONTRACTS = {
|
||||
"openai": {
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
"permission_contract",
|
||||
"target_permission_contract",
|
||||
"target_native_contract",
|
||||
],
|
||||
"required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"field_mapping": {
|
||||
"display_name": "interface.display_name",
|
||||
"short_description": "interface.short_description",
|
||||
"default_prompt": "interface.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
},
|
||||
},
|
||||
"claude": {
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
"permission_contract",
|
||||
"target_permission_contract",
|
||||
"target_native_contract",
|
||||
],
|
||||
"required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"field_mapping": {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
},
|
||||
},
|
||||
"generic": {
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
"permission_contract",
|
||||
"target_permission_contract",
|
||||
"target_native_contract",
|
||||
],
|
||||
"required_files": ["targets/generic/adapter.json"],
|
||||
"field_mapping": {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
EXCLUDED_ARCHIVE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"}
|
||||
EXCLUDED_ARCHIVE_PARTS = {".git", ".previews", "__pycache__", ".venv", "venv", "node_modules", "dist"}
|
||||
|
||||
|
||||
def should_skip_archive_path(rel_path: Path) -> bool:
|
||||
@@ -454,7 +347,7 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
|
||||
notes = target_dir / "README.md"
|
||||
native = payload["target_native_contract"]
|
||||
notes.write_text(
|
||||
f"# Claude-Compatible Package\n\nUse `{skill_dir.name}` with its neutral source files. This target does not require vendor metadata by default.\n\n"
|
||||
f"# Claude-Compatible Package\n\nUse `{payload['name']}` with its neutral source files. This target does not require vendor metadata by default.\n\n"
|
||||
f"Native surface: {native['native_surface']}.\n\n"
|
||||
f"Activation: {native['activation']['policy']}\n\n"
|
||||
f"Resources: {native['resources']['strategy']}\n\n"
|
||||
@@ -463,16 +356,34 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
|
||||
encoding="utf-8",
|
||||
)
|
||||
payload["install_hint"] = f"Use the packaged skill directly; this target relies on SKILL.md and optional neutral metadata."
|
||||
elif platform == "vscode":
|
||||
notes = target_dir / "README.md"
|
||||
native = payload["target_native_contract"]
|
||||
notes.write_text(
|
||||
f"# VS Code / Copilot Agent Skills Package\n\n"
|
||||
f"Install `{payload['name']}` as a VS Code user or project scoped Agent Skill. Keep the folder name aligned with `SKILL.md` frontmatter name.\n\n"
|
||||
f"Native surface: {native['native_surface']}.\n\n"
|
||||
f"Activation: {native['activation']['policy']}\n\n"
|
||||
f"Resources: {native['resources']['strategy']}\n\n"
|
||||
f"Scripts: {native['scripts']['strategy']}\n\n"
|
||||
f"Permission model: {payload['target_permission_contract']['permission_model']}. "
|
||||
"Review `target_permission_contract`, workspace trust, and `reports/security_trust_report.md` before running scripts.\n\n"
|
||||
"This adapter does not perform automatic VS Code installation; it preserves the reviewed source package plus install notes.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
payload["install_hint"] = (
|
||||
"Install the package as a VS Code user or project scoped Agent Skill; use targets/vscode/README.md for scope and trust notes."
|
||||
)
|
||||
else:
|
||||
payload["install_hint"] = f"Use {skill_dir.name} as an Agent Skills compatible package."
|
||||
payload["install_hint"] = f"Use {payload['name']} as an Agent Skills compatible package."
|
||||
path = target_dir / "adapter.json"
|
||||
payload["contract"] = PLATFORM_CONTRACTS.get(platform, PLATFORM_CONTRACTS["generic"])
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
|
||||
def make_zip(skill_dir: Path, out_dir: Path) -> Path:
|
||||
zip_path = out_dir / f"{skill_dir.name}.zip"
|
||||
def make_zip(skill_dir: Path, out_dir: Path, package_name: str) -> Path:
|
||||
zip_path = out_dir / f"{package_name}.zip"
|
||||
skill_root = skill_dir.resolve()
|
||||
out_root = out_dir.resolve()
|
||||
with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED) as zf:
|
||||
@@ -487,17 +398,18 @@ def make_zip(skill_dir: Path, out_dir: Path) -> Path:
|
||||
rel_path = path.relative_to(skill_dir)
|
||||
if should_skip_archive_path(rel_path):
|
||||
continue
|
||||
zf.write(path, arcname=str(path.relative_to(skill_dir.parent)))
|
||||
zf.write(path, arcname=str(PurePosixPath(package_name, *rel_path.parts)))
|
||||
return zip_path
|
||||
|
||||
|
||||
def copy_manifest(skill_dir: Path, out_dir: Path) -> Path:
|
||||
def copy_manifest(skill_dir: Path, out_dir: Path) -> tuple[Path, str]:
|
||||
manifest_path = out_dir / "manifest.json"
|
||||
manifest = build_manifest(skill_dir, "generic")
|
||||
manifest_path.write_text(
|
||||
json.dumps(build_manifest(skill_dir, "generic"), ensure_ascii=False, indent=2),
|
||||
json.dumps(manifest, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest_path
|
||||
return manifest_path, package_name_from_manifest(manifest, skill_dir)
|
||||
|
||||
|
||||
def load_expectations(path: Path | None) -> dict:
|
||||
@@ -511,9 +423,9 @@ def validate_exports(out_dir: Path, expectations: dict) -> dict:
|
||||
required_targets = expectations.get("required_targets", [])
|
||||
required_fields = expectations.get("required_fields", [])
|
||||
required_by_target = {
|
||||
"openai": expectations.get("openai_required_files", []),
|
||||
"claude": expectations.get("claude_required_files", []),
|
||||
"generic": expectations.get("generic_required_files", []),
|
||||
key[: -len("_required_files")]: value
|
||||
for key, value in expectations.items()
|
||||
if key.endswith("_required_files")
|
||||
}
|
||||
|
||||
for target in required_targets:
|
||||
@@ -534,7 +446,7 @@ def validate_exports(out_dir: Path, expectations: dict) -> dict:
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Generate lightweight cross-platform packaging artifacts.")
|
||||
parser.add_argument("skill_dir", help="Path to the skill directory")
|
||||
parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic")
|
||||
parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic, vscode")
|
||||
parser.add_argument("--output-dir", default="dist", help="Output directory")
|
||||
parser.add_argument("--expectations", help="JSON file describing packaging expectations")
|
||||
parser.add_argument("--zip", action="store_true", help="Create a zip package")
|
||||
@@ -548,12 +460,12 @@ def main() -> None:
|
||||
if out_dir.exists():
|
||||
shutil.rmtree(out_dir)
|
||||
out_dir.mkdir(parents=True)
|
||||
manifest = copy_manifest(skill_dir, out_dir)
|
||||
manifest, package_name = copy_manifest(skill_dir, out_dir)
|
||||
generated.append(str(manifest))
|
||||
for platform in (args.platform or ["generic"]):
|
||||
generated.append(str(write_adapter(skill_dir, out_dir, platform)))
|
||||
if args.zip:
|
||||
generated.append(str(make_zip(skill_dir, out_dir)))
|
||||
generated.append(str(make_zip(skill_dir, out_dir, package_name)))
|
||||
except (FileNotFoundError, ValueError, yaml.YAMLError) as exc:
|
||||
failures.append(str(exc))
|
||||
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Platform contract definitions for cross-platform skill packages."""
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by cross_packager.py to keep platform contract data separate from packaging flow."
|
||||
|
||||
|
||||
COMMON_REQUIRED_FIELDS = [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
"permission_contract",
|
||||
"target_permission_contract",
|
||||
"target_native_contract",
|
||||
]
|
||||
|
||||
STANDARD_FIELD_MAPPING = {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
}
|
||||
|
||||
|
||||
def interface_field_mapping() -> dict[str, str]:
|
||||
return {
|
||||
"display_name": "interface.display_name",
|
||||
"short_description": "interface.short_description",
|
||||
"default_prompt": "interface.default_prompt",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"shell": "compatibility.execution.shell",
|
||||
}
|
||||
|
||||
|
||||
PLATFORM_CONTRACTS = {
|
||||
"openai": {
|
||||
"required_fields": list(COMMON_REQUIRED_FIELDS),
|
||||
"required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"field_mapping": interface_field_mapping(),
|
||||
},
|
||||
"claude": {
|
||||
"required_fields": list(COMMON_REQUIRED_FIELDS),
|
||||
"required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"field_mapping": dict(STANDARD_FIELD_MAPPING),
|
||||
},
|
||||
"generic": {
|
||||
"required_fields": list(COMMON_REQUIRED_FIELDS),
|
||||
"required_files": ["targets/generic/adapter.json"],
|
||||
"field_mapping": dict(STANDARD_FIELD_MAPPING),
|
||||
},
|
||||
"vscode": {
|
||||
"required_fields": [
|
||||
"name",
|
||||
"description",
|
||||
"version",
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"compiler",
|
||||
"compiled_contract",
|
||||
"permission_contract",
|
||||
"target_permission_contract",
|
||||
"target_native_contract",
|
||||
"target_transform",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
"execution_context",
|
||||
"shell",
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile",
|
||||
],
|
||||
"required_files": ["targets/vscode/adapter.json", "targets/vscode/README.md"],
|
||||
"field_mapping": {
|
||||
"name": "SKILL.md::frontmatter.name and folder name",
|
||||
"description": "SKILL.md::frontmatter.description",
|
||||
"display_name": "agents/interface.yaml::interface.display_name",
|
||||
"execution_context": "compatibility.execution.context",
|
||||
"permissions": "adapter.target_permission_contract",
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by optimize_description.py to render description optimization reports."
|
||||
|
||||
|
||||
def render_markdown(report: dict, title: str) -> str:
|
||||
lines = [
|
||||
f"# {title}",
|
||||
"",
|
||||
f"Winner: `{report['winner']['label']}`",
|
||||
"",
|
||||
f"- current tokens: `{report['current_candidate']['estimated_tokens']}`",
|
||||
f"- winner tokens: `{report['winner']['estimated_tokens']}`",
|
||||
]
|
||||
if report["baseline"]:
|
||||
lines.append(f"- baseline tokens: `{report['baseline']['estimated_tokens']}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Winner",
|
||||
"",
|
||||
report["winner"]["description"],
|
||||
"",
|
||||
"## Candidate Ranking",
|
||||
"",
|
||||
"| Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for candidate in report["candidates"]:
|
||||
holdout = candidate.get("holdout", {})
|
||||
lines.append(
|
||||
f"| `{candidate['label']}` | {candidate['estimated_tokens']} | {candidate['dev']['false_positives']} | {candidate['dev']['false_negatives']} | {candidate['dev']['near_neighbor_pass_rate']} | {holdout.get('false_positives', '-')} | {holdout.get('false_negatives', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Acceptance Gates",
|
||||
"",
|
||||
"| Gate | Winner FP | Winner FN | Current FP | Current FN | Baseline FP | Baseline FN |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for gate_name, gate in (
|
||||
("Holdout", report["acceptance_gates"]["holdout_non_regression"]),
|
||||
("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]),
|
||||
("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]),
|
||||
("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]),
|
||||
):
|
||||
winner_gate = gate.get("winner") or {}
|
||||
current_gate = gate.get("current") or {}
|
||||
baseline_gate = gate.get("baseline") or {}
|
||||
if not winner_gate and not current_gate and not baseline_gate:
|
||||
continue
|
||||
lines.append(
|
||||
f"| {gate_name} | {winner_gate.get('false_positives', '-')} | {winner_gate.get('false_negatives', '-')} | {current_gate.get('false_positives', '-')} | {current_gate.get('false_negatives', '-')} | {baseline_gate.get('false_positives', '-')} | {baseline_gate.get('false_negatives', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Calibration",
|
||||
"",
|
||||
"| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |",
|
||||
"| --- | ---: | --- | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for gate_name, gate in (
|
||||
("Holdout", report["acceptance_gates"]["holdout_non_regression"]),
|
||||
("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]),
|
||||
("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]),
|
||||
("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]),
|
||||
):
|
||||
winner_calibration = gate.get("winner_calibration") or {}
|
||||
current_calibration = gate.get("current_calibration") or {}
|
||||
baseline_calibration = gate.get("baseline_calibration") or {}
|
||||
if not winner_calibration and not current_calibration and not baseline_calibration:
|
||||
continue
|
||||
lines.append(
|
||||
f"| {gate_name} | {winner_calibration.get('score_gap', '-')} | {winner_calibration.get('risk_band', '-')} | {winner_calibration.get('boundary_case_rate', '-')} | {current_calibration.get('score_gap', '-')} | {baseline_calibration.get('score_gap', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Judge Blind Summary",
|
||||
"",
|
||||
"| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement |",
|
||||
"| --- | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
judge_gate = report["acceptance_gates"]["judge_blind_holdout_non_regression"]
|
||||
judge_winner = (judge_gate.get("winner") or {}).get("judge_summary") or {}
|
||||
judge_current = (judge_gate.get("current") or {}).get("judge_summary") or {}
|
||||
judge_baseline = (judge_gate.get("baseline") or {}).get("judge_summary") or {}
|
||||
if judge_winner or judge_current or judge_baseline:
|
||||
lines.append(
|
||||
f"| Judge Blind Holdout | {judge_winner.get('agreement_rate', '-')} | {judge_winner.get('mean_confidence', '-')} | {judge_current.get('agreement_rate', '-')} | {judge_baseline.get('agreement_rate', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Family Health",
|
||||
"",
|
||||
"| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for gate_name, gate in (
|
||||
("Holdout", report["acceptance_gates"]["holdout_non_regression"]),
|
||||
("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]),
|
||||
("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]),
|
||||
("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]),
|
||||
):
|
||||
winner_health = gate.get("winner_family_health") or {}
|
||||
current_health = gate.get("current_family_health") or {}
|
||||
baseline_health = gate.get("baseline_family_health") or {}
|
||||
if not winner_health and not current_health and not baseline_health:
|
||||
continue
|
||||
weakest = winner_health.get("weakest_family") or {}
|
||||
weakest_label = (
|
||||
f"{weakest.get('family')} ({weakest.get('errors')} errors)"
|
||||
if weakest.get("family")
|
||||
else "-"
|
||||
)
|
||||
lines.append(
|
||||
f"| {gate_name} | {winner_health.get('clean_family_count', '-')}/{winner_health.get('family_count', '-')} | {weakest_label} | {current_health.get('clean_family_count', '-')}/{current_health.get('family_count', '-')} | {baseline_health.get('clean_family_count', '-')}/{baseline_health.get('family_count', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Selection Logic",
|
||||
"",
|
||||
"Ordered by:",
|
||||
]
|
||||
)
|
||||
for item in report["selection_logic"]["priority"]:
|
||||
lines.append(f"- {item}")
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_adoption_drift_report import (
|
||||
ALLOWED_ACTIVATION_TYPES,
|
||||
ALLOWED_EVENTS,
|
||||
ALLOWED_FAILURE_TYPES,
|
||||
ALLOWED_OUTCOMES,
|
||||
ALLOWED_SOURCES,
|
||||
display_path,
|
||||
normalize_event,
|
||||
skill_defaults,
|
||||
)
|
||||
|
||||
|
||||
def default_spool_path(skill_dir: Path) -> Path:
|
||||
return skill_dir / ".yao" / "telemetry_spool" / "external_events.jsonl"
|
||||
|
||||
|
||||
def append_event(path: Path, event: dict[str, str]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def import_command(skill_dir: Path, output_jsonl: Path) -> str:
|
||||
return shlex.join(
|
||||
[
|
||||
"python3",
|
||||
"scripts/yao.py",
|
||||
"telemetry-import",
|
||||
display_path(skill_dir),
|
||||
"--input-jsonl",
|
||||
display_path(output_jsonl),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def emit_event(
|
||||
skill_dir: Path,
|
||||
output_jsonl: Path | None = None,
|
||||
event_name: str = "script_run",
|
||||
activation_type: str = "manual",
|
||||
outcome: str = "unknown",
|
||||
failure_type: str = "none",
|
||||
source: str = "external",
|
||||
command: str = "external-client",
|
||||
timestamp: str | None = None,
|
||||
skill_name: str | None = None,
|
||||
version: str | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
output_jsonl = (output_jsonl or default_spool_path(skill_dir)).resolve()
|
||||
raw_event: dict[str, Any] = {
|
||||
"event": event_name,
|
||||
"activation_type": activation_type,
|
||||
"outcome": outcome,
|
||||
"failure_type": failure_type,
|
||||
"source": source,
|
||||
"command": command,
|
||||
}
|
||||
if timestamp:
|
||||
raw_event["timestamp"] = timestamp
|
||||
if skill_name:
|
||||
raw_event["skill"] = skill_name
|
||||
if version:
|
||||
raw_event["version"] = version
|
||||
event, failures = normalize_event(raw_event, skill_defaults(skill_dir), "emit")
|
||||
if event and not failures and not dry_run:
|
||||
append_event(output_jsonl, event)
|
||||
return {
|
||||
"ok": not failures,
|
||||
"schema_version": "1.0",
|
||||
"skill_dir": display_path(skill_dir),
|
||||
"output_jsonl": display_path(output_jsonl),
|
||||
"dry_run": dry_run,
|
||||
"emitted": bool(event and not failures and not dry_run),
|
||||
"event": event or {},
|
||||
"failures": failures,
|
||||
"artifacts": {
|
||||
"spool_jsonl": display_path(output_jsonl),
|
||||
"import_command": import_command(skill_dir, output_jsonl),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Emit one metadata-only telemetry event for later import.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-jsonl")
|
||||
parser.add_argument("--event", choices=sorted(ALLOWED_EVENTS), default="script_run")
|
||||
parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="manual")
|
||||
parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown")
|
||||
parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none")
|
||||
parser.add_argument("--source", choices=sorted(ALLOWED_SOURCES), default="external")
|
||||
parser.add_argument("--command", default="external-client")
|
||||
parser.add_argument("--timestamp")
|
||||
parser.add_argument("--skill-name")
|
||||
parser.add_argument("--version")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = emit_event(
|
||||
Path(args.skill_dir),
|
||||
output_jsonl=Path(args.output_jsonl) if args.output_jsonl else None,
|
||||
event_name=args.event,
|
||||
activation_type=args.activation_type,
|
||||
outcome=args.outcome,
|
||||
failure_type=args.failure_type,
|
||||
source=args.source,
|
||||
command=args.command,
|
||||
timestamp=args.timestamp,
|
||||
skill_name=args.skill_name,
|
||||
version=args.version,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare artifact-role handoffs across world-class evidence reports."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts."
|
||||
|
||||
|
||||
def _compare_check(
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
expected: Any,
|
||||
actual: Any,
|
||||
paths: list[str],
|
||||
detail: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"key": key,
|
||||
"label": label,
|
||||
"status": "pass" if expected == actual else "fail",
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": paths,
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
|
||||
def role_contract_signature(contract: dict[str, Any]) -> dict[str, Any]:
|
||||
roles = {
|
||||
str(item.get("role", "")): item
|
||||
for item in contract.get("roles", [])
|
||||
if isinstance(item, dict) and str(item.get("role", "")).strip()
|
||||
}
|
||||
return {
|
||||
"role_source": contract.get("role_source"),
|
||||
"counts_as_evidence": contract.get("counts_as_evidence"),
|
||||
"artifact_prefill_counts_as_evidence": contract.get("artifact_prefill_counts_as_evidence"),
|
||||
"submission_ref_total_count": contract.get("submission_ref_total_count"),
|
||||
"submission_ref_ready_count": contract.get("submission_ref_ready_count"),
|
||||
"supporting_evidence_total_count": contract.get("supporting_evidence_total_count"),
|
||||
"supporting_evidence_ready_count": contract.get("supporting_evidence_ready_count"),
|
||||
"submission_ref_copy_to_artifact_refs": roles.get("submission-ref", {}).get("copy_to_artifact_refs"),
|
||||
"supporting_evidence_copy_to_artifact_refs": roles.get("supporting-evidence", {}).get(
|
||||
"copy_to_artifact_refs"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def preflight_role_signatures(world_class_preflight: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
signatures: dict[str, dict[str, Any]] = {}
|
||||
items = world_class_preflight.get("items", []) if isinstance(world_class_preflight, dict) else []
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = str(item.get("evidence_key", "")).strip()
|
||||
submission_kit = item.get("submission_kit", {}) if isinstance(item.get("submission_kit", {}), dict) else {}
|
||||
contract = (
|
||||
submission_kit.get("artifact_role_contract", {})
|
||||
if isinstance(submission_kit.get("artifact_role_contract", {}), dict)
|
||||
else {}
|
||||
)
|
||||
if key and contract:
|
||||
signatures[key] = role_contract_signature(contract)
|
||||
return signatures
|
||||
|
||||
|
||||
def review_studio_role_signatures(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
actions = review_studio.get("review_actions", []) if isinstance(review_studio, dict) else []
|
||||
for action in actions:
|
||||
if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence":
|
||||
continue
|
||||
signatures: dict[str, dict[str, Any]] = {}
|
||||
for step in action.get("evidence_steps", []):
|
||||
if not isinstance(step, dict):
|
||||
continue
|
||||
key = str(step.get("key", "")).strip()
|
||||
contract = (
|
||||
step.get("artifact_role_contract", {})
|
||||
if isinstance(step.get("artifact_role_contract", {}), dict)
|
||||
else {}
|
||||
)
|
||||
if key and contract:
|
||||
signatures[key] = role_contract_signature(contract)
|
||||
return signatures
|
||||
return {}
|
||||
|
||||
|
||||
def build_preflight_artifact_role_handoff_checks(
|
||||
*,
|
||||
skill_dir: Path,
|
||||
world_class_preflight: dict[str, Any],
|
||||
review_studio: dict[str, Any],
|
||||
report_paths: dict[str, str],
|
||||
) -> list[dict[str, Any]]:
|
||||
preflight_submissions = (
|
||||
world_class_preflight.get("submissions", {})
|
||||
if isinstance(world_class_preflight.get("submissions", {}), dict)
|
||||
else {}
|
||||
)
|
||||
preflight_commands = (
|
||||
preflight_submissions.get("commands", {})
|
||||
if isinstance(preflight_submissions.get("commands", {}), dict)
|
||||
else {}
|
||||
)
|
||||
preflight_role_contract = (
|
||||
preflight_submissions.get("artifact_role_contract", {})
|
||||
if isinstance(preflight_submissions.get("artifact_role_contract", {}), dict)
|
||||
else {}
|
||||
)
|
||||
preflight_roles = {
|
||||
str(item.get("role", "")): item for item in preflight_role_contract.get("roles", []) if isinstance(item, dict)
|
||||
}
|
||||
default_submissions_dir = "evidence/world_class/submissions"
|
||||
expected_preflight_handoff = {
|
||||
"directory": default_submissions_dir,
|
||||
"drafts_count_as_evidence": False,
|
||||
"preflight_counts_submission_as_completion": False,
|
||||
"html_report": "reports/world_class_evidence_preflight.html",
|
||||
"html_exists": True,
|
||||
"prepare_submission": f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir}",
|
||||
"prepare_prefilled_submission": (
|
||||
f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir} "
|
||||
"--prefill-artifacts"
|
||||
),
|
||||
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {default_submissions_dir}",
|
||||
"submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {default_submissions_dir}",
|
||||
"refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {default_submissions_dir}",
|
||||
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
"artifact_prefill_counts_as_evidence": False,
|
||||
"artifact_role_source": "world-class-submission-kit",
|
||||
"artifact_role_counts_as_evidence": False,
|
||||
"artifact_role_prefill_counts_as_evidence": False,
|
||||
"submission_ref_role_present": True,
|
||||
"supporting_evidence_role_present": True,
|
||||
"submission_ref_copy_to_artifact_refs": True,
|
||||
"supporting_evidence_copy_to_artifact_refs": False,
|
||||
"submission_ref_total_present": True,
|
||||
"supporting_evidence_total_present": True,
|
||||
}
|
||||
actual_preflight_handoff = {
|
||||
"directory": preflight_submissions.get("directory"),
|
||||
"drafts_count_as_evidence": preflight_submissions.get("drafts_count_as_evidence"),
|
||||
"preflight_counts_submission_as_completion": preflight_submissions.get(
|
||||
"preflight_counts_submission_as_completion"
|
||||
),
|
||||
"html_report": world_class_preflight.get("artifacts", {}).get("html")
|
||||
if isinstance(world_class_preflight.get("artifacts", {}), dict)
|
||||
else None,
|
||||
"html_exists": (skill_dir / "reports" / "world_class_evidence_preflight.html").exists(),
|
||||
"prepare_submission": preflight_commands.get("prepare_submission"),
|
||||
"prepare_prefilled_submission": preflight_commands.get("prepare_prefilled_submission"),
|
||||
"validate_intake": preflight_commands.get("validate_intake"),
|
||||
"submission_review": preflight_commands.get("submission_review"),
|
||||
"refresh_ledger": preflight_commands.get("refresh_ledger"),
|
||||
"guard_claim": preflight_commands.get("guard_claim"),
|
||||
"artifact_prefill_counts_as_evidence": preflight_submissions.get("artifact_prefill_counts_as_evidence"),
|
||||
"artifact_role_source": preflight_role_contract.get("role_source"),
|
||||
"artifact_role_counts_as_evidence": preflight_role_contract.get("counts_as_evidence"),
|
||||
"artifact_role_prefill_counts_as_evidence": preflight_role_contract.get(
|
||||
"artifact_prefill_counts_as_evidence"
|
||||
),
|
||||
"submission_ref_role_present": "submission-ref" in preflight_roles,
|
||||
"supporting_evidence_role_present": "supporting-evidence" in preflight_roles,
|
||||
"submission_ref_copy_to_artifact_refs": preflight_roles.get("submission-ref", {}).get(
|
||||
"copy_to_artifact_refs"
|
||||
),
|
||||
"supporting_evidence_copy_to_artifact_refs": preflight_roles.get("supporting-evidence", {}).get(
|
||||
"copy_to_artifact_refs"
|
||||
),
|
||||
"submission_ref_total_present": int(preflight_role_contract.get("submission_ref_total_count", 0)) > 0,
|
||||
"supporting_evidence_total_present": int(preflight_role_contract.get("supporting_evidence_total_count", 0))
|
||||
> 0,
|
||||
}
|
||||
return [
|
||||
_compare_check(
|
||||
key="preflight-submission-kit-handoff",
|
||||
label="Preflight exposes a safe submission-kit handoff",
|
||||
expected=expected_preflight_handoff,
|
||||
actual=actual_preflight_handoff,
|
||||
paths=[report_paths["world_class_preflight"], "reports/world_class_evidence_preflight.html"],
|
||||
detail=(
|
||||
"Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, "
|
||||
"and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence."
|
||||
),
|
||||
),
|
||||
_compare_check(
|
||||
key="review-studio-preflight-artifact-role-handoff",
|
||||
label="Review Studio mirrors preflight artifact roles",
|
||||
expected=preflight_role_signatures(world_class_preflight),
|
||||
actual=review_studio_role_signatures(review_studio),
|
||||
paths=[report_paths["world_class_preflight"], report_paths["review_studio"]],
|
||||
detail=(
|
||||
"The Review Studio world-class action card must carry the same submission-ref versus "
|
||||
"supporting-evidence contract as the preflight handoff."
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for cross-report evidence consistency checks."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py for shared report loading, comparison, and Markdown rendering helpers."
|
||||
|
||||
REQUIRED_REPORTS = {
|
||||
"benchmark": "reports/benchmark_reproducibility.json",
|
||||
"overview": "reports/skill-overview.json",
|
||||
"interpretation": "reports/skill-interpretation.json",
|
||||
"adoption": "reports/adoption_drift_report.json",
|
||||
"world_class_ledger": "reports/world_class_evidence_ledger.json",
|
||||
"world_class_plan": "reports/world_class_evidence_plan.json",
|
||||
"world_class_intake": "reports/world_class_evidence_intake.json",
|
||||
"world_class_preflight": "reports/world_class_evidence_preflight.json",
|
||||
"world_class_submission_review": "reports/world_class_submission_review.json",
|
||||
"world_class_operator_runbook": "reports/world_class_operator_runbook.json",
|
||||
"skill_os2_coverage": "reports/skill_os2_coverage.json",
|
||||
"review_studio": "reports/review-studio.json",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulation": "reports/install_simulation.json",
|
||||
"trust": "reports/security_trust_report.json",
|
||||
"context_budget": "reports/context_budget.json",
|
||||
"world_class_claim_guard": "reports/world_class_claim_guard.json",
|
||||
}
|
||||
REQUIRED_TEXT_REPORTS = {
|
||||
"skill_os2_review": "reports/skill-os-2-review.md",
|
||||
}
|
||||
BENCHMARK_SUMMARY_KEYS = [
|
||||
"release_lock_ready",
|
||||
"required_artifact_count",
|
||||
"missing_artifact_count",
|
||||
"source_contract_sha256",
|
||||
"archive_sha256",
|
||||
"world_class_ledger_pending_count",
|
||||
"world_class_source_check_count",
|
||||
"world_class_source_pass_count",
|
||||
"world_class_source_blocked_count",
|
||||
"beta_test_ready",
|
||||
"beta_test_blocker_count",
|
||||
"beta_test_deferred_evidence_count",
|
||||
"public_claim_ready",
|
||||
"public_claim_blocker_count",
|
||||
]
|
||||
ADOPTION_SUMMARY_KEYS = [
|
||||
"event_count",
|
||||
"adoption_sample_count",
|
||||
"activation_count",
|
||||
"accepted_count",
|
||||
"adoption_rate",
|
||||
"risk_band",
|
||||
"event_types",
|
||||
"source_types",
|
||||
]
|
||||
LEDGER_SUMMARY_KEYS = [
|
||||
"ledger_entry_count",
|
||||
"accepted_count",
|
||||
"pending_count",
|
||||
"human_pending_count",
|
||||
"external_pending_count",
|
||||
"source_check_count",
|
||||
"source_pass_count",
|
||||
"source_blocked_count",
|
||||
"ready_to_claim_world_class",
|
||||
"decision",
|
||||
]
|
||||
LOCKSTEP_SECTIONS = [
|
||||
"scorecard",
|
||||
"capability_profile",
|
||||
"principle_model",
|
||||
"contract_boundary",
|
||||
"quality_review",
|
||||
"risk_governance",
|
||||
"world_class_readiness",
|
||||
"package_assets",
|
||||
"iteration_roadmap",
|
||||
]
|
||||
|
||||
|
||||
def load_json(path: Path) -> tuple[dict[str, Any], str | None]:
|
||||
if not path.exists():
|
||||
return {}, "missing"
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError as exc:
|
||||
return {}, f"invalid-json: {exc}"
|
||||
if not isinstance(payload, dict):
|
||||
return {}, "json-root-not-object"
|
||||
return payload, None
|
||||
|
||||
|
||||
def load_text(path: Path) -> tuple[str, str | None]:
|
||||
if not path.exists():
|
||||
return "", "missing"
|
||||
return path.read_text(encoding="utf-8"), None
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def nested(payload: dict[str, Any], path: list[str], default: Any = None) -> Any:
|
||||
current: Any = payload
|
||||
for key in path:
|
||||
if not isinstance(current, dict) or key not in current:
|
||||
return default
|
||||
current = current[key]
|
||||
return current
|
||||
|
||||
|
||||
def scanned_surface_paths(payload: dict[str, Any]) -> set[str]:
|
||||
surfaces = payload.get("scanned_surfaces")
|
||||
if not isinstance(surfaces, list):
|
||||
return set()
|
||||
paths: set[str] = set()
|
||||
for item in surfaces:
|
||||
if isinstance(item, dict) and isinstance(item.get("path"), str):
|
||||
paths.add(item["path"])
|
||||
elif isinstance(item, str):
|
||||
paths.add(item)
|
||||
return paths
|
||||
|
||||
|
||||
def as_int(value: Any) -> int | None:
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def add_check(
|
||||
checks: list[dict[str, Any]],
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
status: str,
|
||||
expected: Any,
|
||||
actual: Any,
|
||||
paths: list[str],
|
||||
detail: str,
|
||||
) -> None:
|
||||
checks.append(
|
||||
{
|
||||
"key": key,
|
||||
"label": label,
|
||||
"status": status,
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": paths,
|
||||
"detail": detail,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def compare_values(
|
||||
checks: list[dict[str, Any]],
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
expected: Any,
|
||||
actual: Any,
|
||||
paths: list[str],
|
||||
detail: str,
|
||||
) -> None:
|
||||
add_check(
|
||||
checks,
|
||||
key=key,
|
||||
label=label,
|
||||
status="pass" if expected == actual else "fail",
|
||||
expected=expected,
|
||||
actual=actual,
|
||||
paths=paths,
|
||||
detail=detail,
|
||||
)
|
||||
|
||||
|
||||
def compare_summary_keys(
|
||||
checks: list[dict[str, Any]],
|
||||
*,
|
||||
key_prefix: str,
|
||||
label: str,
|
||||
source_summary: dict[str, Any],
|
||||
embedded_summary: dict[str, Any],
|
||||
keys: list[str],
|
||||
paths: list[str],
|
||||
) -> None:
|
||||
expected = {key: source_summary.get(key) for key in keys}
|
||||
actual = {key: embedded_summary.get(key) for key in keys}
|
||||
compare_values(
|
||||
checks,
|
||||
key=key_prefix,
|
||||
label=label,
|
||||
expected=expected,
|
||||
actual=actual,
|
||||
paths=paths,
|
||||
detail="Selected summary fields must match exactly across generated reports.",
|
||||
)
|
||||
|
||||
|
||||
def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]:
|
||||
gates = review_studio.get("gates")
|
||||
if not isinstance(gates, list):
|
||||
return {}
|
||||
for item in gates:
|
||||
if isinstance(item, dict) and item.get("key") == key:
|
||||
return item
|
||||
return {}
|
||||
|
||||
|
||||
def beta_public_claim_split_values(
|
||||
benchmark: dict[str, Any],
|
||||
ledger: dict[str, Any],
|
||||
review_studio: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
benchmark_summary = nested(benchmark, ["summary"], {})
|
||||
ledger_summary = nested(ledger, ["summary"], {})
|
||||
expected_beta_ready = (
|
||||
bool(benchmark_summary.get("reproducibility_ready"))
|
||||
and bool(benchmark_summary.get("release_lock_ready"))
|
||||
and bool(benchmark_summary.get("provider_evidence_complete"))
|
||||
)
|
||||
output_review_summary = nested(review_studio, ["data", "output_review_adjudication", "summary"], {})
|
||||
review_pair_count = as_int(output_review_summary.get("pair_count")) if isinstance(output_review_summary, dict) else None
|
||||
review_pending_count = as_int(output_review_summary.get("pending_count")) if isinstance(output_review_summary, dict) else None
|
||||
expected_human_review_complete = (
|
||||
review_pair_count is not None and review_pair_count > 0 and review_pending_count == 0
|
||||
)
|
||||
ledger_entries = ledger.get("entries", []) if isinstance(ledger, dict) else []
|
||||
pending_ledger_keys = sorted(
|
||||
str(entry.get("key", ""))
|
||||
for entry in ledger_entries
|
||||
if isinstance(entry, dict) and entry.get("status") == "pending" and str(entry.get("key", ""))
|
||||
)
|
||||
beta_boundary = {
|
||||
"beta_test_ready": expected_beta_ready,
|
||||
"public_claim_ready": ledger_summary.get("ready_to_claim_world_class"),
|
||||
"human_review_complete": expected_human_review_complete,
|
||||
"beta_release_ready": expected_beta_ready,
|
||||
"beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
|
||||
"deferred_evidence_keys": pending_ledger_keys,
|
||||
"deferred_human_review": "human-adjudication" in pending_ledger_keys,
|
||||
}
|
||||
beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {}
|
||||
deferred_keys = sorted(
|
||||
str(item.get("key", ""))
|
||||
for item in beta_release.get("allowed_deferred_evidence", [])
|
||||
if isinstance(item, dict) and str(item.get("key", ""))
|
||||
)
|
||||
actual_beta_boundary = {
|
||||
"beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None,
|
||||
"public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None,
|
||||
"human_review_complete": benchmark_summary.get("human_review_complete")
|
||||
if isinstance(benchmark_summary, dict)
|
||||
else None,
|
||||
"beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None,
|
||||
"beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None,
|
||||
"deferred_evidence_keys": deferred_keys,
|
||||
"deferred_human_review": "human-adjudication" in deferred_keys,
|
||||
}
|
||||
return beta_boundary, actual_beta_boundary
|
||||
|
||||
|
||||
def report_contract(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
contract = payload.get("report_contract")
|
||||
return contract if isinstance(contract, dict) else {}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Evidence Consistency",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- checks: `{summary['check_count']}`",
|
||||
f"- pass: `{summary['pass_count']}`",
|
||||
f"- warn: `{summary['warn_count']}`",
|
||||
f"- fail: `{summary['fail_count']}`",
|
||||
"",
|
||||
"This gate compares generated evidence reports against each other. It does not create provider, human, native-client, or permission-enforcement evidence; it only catches drift between reports that already exist.",
|
||||
"",
|
||||
"## Checks",
|
||||
"",
|
||||
"| Check | Status | Detail | Paths |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
for check in report["checks"]:
|
||||
paths = ", ".join(f"`{path}`" for path in check["paths"])
|
||||
lines.append(
|
||||
"| "
|
||||
+ " | ".join(
|
||||
[
|
||||
check["label"].replace("|", "\\|"),
|
||||
f"`{check['status']}`",
|
||||
check["detail"].replace("|", "\\|"),
|
||||
paths.replace("|", "\\|"),
|
||||
]
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
failures = [check for check in report["checks"] if check["status"] == "fail"]
|
||||
if failures:
|
||||
lines.extend(["", "## Failures", ""])
|
||||
for check in failures:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {check['label']}",
|
||||
"",
|
||||
f"- key: `{check['key']}`",
|
||||
f"- expected: `{json.dumps(check['expected'], ensure_ascii=False, sort_keys=True)}`",
|
||||
f"- actual: `{json.dumps(check['actual'], ensure_ascii=False, sort_keys=True)}`",
|
||||
"",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare world-class phase queues across generated reports."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from evidence_consistency_world_class import world_class_review_action_steps
|
||||
from world_class_phase_queue import build_phase_queue, summarize_phase_queue
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to prevent preflight and Review Studio phase-queue drift."
|
||||
|
||||
|
||||
def phase_queue_signature(queue: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
signature: list[dict[str, Any]] = []
|
||||
for item in queue:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
signature.append(
|
||||
{
|
||||
"phase": item.get("phase"),
|
||||
"priority": item.get("priority"),
|
||||
"status": item.get("status"),
|
||||
"blocked_count": item.get("blocked_count"),
|
||||
"row_count": item.get("row_count"),
|
||||
"owners": sorted(str(owner) for owner in item.get("owners", [])),
|
||||
"evidence_keys": sorted(str(key) for key in item.get("evidence_keys", [])),
|
||||
"next_action_id": item.get("next_action_id"),
|
||||
"verification_command": item.get("verification_command"),
|
||||
"counts_as_completion": item.get("counts_as_completion"),
|
||||
}
|
||||
)
|
||||
return signature
|
||||
|
||||
|
||||
def summary_signature(summary: dict[str, Any]) -> dict[str, Any]:
|
||||
keys = [
|
||||
"phase_queue_count",
|
||||
"phase_queue_blocked_count",
|
||||
"phase_queue_row_count",
|
||||
"phase_queue_next_phase",
|
||||
"phase_queue_next_action_id",
|
||||
"phase_queue_next_command",
|
||||
"phase_queue_counts_as_completion",
|
||||
]
|
||||
return {key: summary.get(key) for key in keys}
|
||||
|
||||
|
||||
def keyed_preflight_items(world_class_preflight: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
items = world_class_preflight.get("items", []) if isinstance(world_class_preflight, dict) else []
|
||||
keyed: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = str(item.get("evidence_key", "")).strip()
|
||||
if key:
|
||||
keyed[key] = item
|
||||
return keyed
|
||||
|
||||
|
||||
def keyed_phase_queue_signatures_from_repair_rows(items: dict[str, dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
|
||||
signatures: dict[str, list[dict[str, Any]]] = {}
|
||||
for key, item in items.items():
|
||||
rows = item.get("repair_checklist", []) if isinstance(item.get("repair_checklist", []), list) else []
|
||||
signatures[key] = phase_queue_signature(build_phase_queue(rows))
|
||||
return signatures
|
||||
|
||||
|
||||
def keyed_phase_queue_signatures_from_items(items: dict[str, dict[str, Any]], queue_key: str) -> dict[str, list[dict[str, Any]]]:
|
||||
signatures: dict[str, list[dict[str, Any]]] = {}
|
||||
for key, item in items.items():
|
||||
queue = item.get(queue_key, []) if isinstance(item.get(queue_key, []), list) else []
|
||||
signatures[key] = phase_queue_signature(queue)
|
||||
return signatures
|
||||
|
||||
|
||||
def any_phase_queue_counts_as_completion(
|
||||
*queues: list[dict[str, Any]],
|
||||
item_maps: dict[str, dict[str, Any]] | None = None,
|
||||
) -> bool:
|
||||
item_maps = item_maps or {}
|
||||
for queue in queues:
|
||||
for item in queue:
|
||||
if isinstance(item, dict) and item.get("counts_as_completion") is True:
|
||||
return True
|
||||
for item in item_maps.values():
|
||||
queue = item.get("phase_queue", []) if isinstance(item.get("phase_queue", []), list) else []
|
||||
for row in queue:
|
||||
if isinstance(row, dict) and row.get("counts_as_completion") is True:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_phase_queue_consistency_check(
|
||||
*,
|
||||
world_class_preflight: dict[str, Any],
|
||||
world_class_operator_runbook: dict[str, Any],
|
||||
review_studio: dict[str, Any],
|
||||
report_paths: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
repair_rows = (
|
||||
world_class_preflight.get("repair_checklist", [])
|
||||
if isinstance(world_class_preflight.get("repair_checklist", []), list)
|
||||
else []
|
||||
)
|
||||
expected_queue = build_phase_queue(repair_rows)
|
||||
expected_summary = summarize_phase_queue(expected_queue)
|
||||
preflight_items = keyed_preflight_items(world_class_preflight)
|
||||
operator_runbook_items = keyed_preflight_items(world_class_operator_runbook)
|
||||
review_steps = world_class_review_action_steps(review_studio)
|
||||
expected = {
|
||||
"summary": summary_signature(expected_summary),
|
||||
"top_level_phase_queue": phase_queue_signature(expected_queue),
|
||||
"item_phase_queues": keyed_phase_queue_signatures_from_repair_rows(preflight_items),
|
||||
"phase_queue_counts_as_completion": False,
|
||||
}
|
||||
actual = {
|
||||
"summary": summary_signature(
|
||||
world_class_preflight.get("summary", {})
|
||||
if isinstance(world_class_preflight.get("summary", {}), dict)
|
||||
else {}
|
||||
),
|
||||
"top_level_phase_queue": phase_queue_signature(
|
||||
world_class_preflight.get("phase_queue", [])
|
||||
if isinstance(world_class_preflight.get("phase_queue", []), list)
|
||||
else []
|
||||
),
|
||||
"operator_runbook_summary": summary_signature(
|
||||
world_class_operator_runbook.get("summary", {})
|
||||
if isinstance(world_class_operator_runbook.get("summary", {}), dict)
|
||||
else {}
|
||||
),
|
||||
"operator_runbook_top_level_phase_queue": phase_queue_signature(
|
||||
world_class_operator_runbook.get("phase_queue", [])
|
||||
if isinstance(world_class_operator_runbook.get("phase_queue", []), list)
|
||||
else []
|
||||
),
|
||||
"item_phase_queues": keyed_phase_queue_signatures_from_items(preflight_items, "phase_queue"),
|
||||
"operator_runbook_phase_queues": keyed_phase_queue_signatures_from_items(
|
||||
operator_runbook_items,
|
||||
"phase_queue",
|
||||
),
|
||||
"review_studio_phase_queues": keyed_phase_queue_signatures_from_items(review_steps, "phase_queue"),
|
||||
"phase_queue_counts_as_completion": any_phase_queue_counts_as_completion(
|
||||
world_class_preflight.get("phase_queue", [])
|
||||
if isinstance(world_class_preflight.get("phase_queue", []), list)
|
||||
else [],
|
||||
world_class_operator_runbook.get("phase_queue", [])
|
||||
if isinstance(world_class_operator_runbook.get("phase_queue", []), list)
|
||||
else [],
|
||||
item_maps={**preflight_items, **operator_runbook_items, **review_steps},
|
||||
),
|
||||
}
|
||||
expected["operator_runbook_summary"] = expected["summary"]
|
||||
expected["operator_runbook_top_level_phase_queue"] = expected["top_level_phase_queue"]
|
||||
expected["operator_runbook_phase_queues"] = expected["item_phase_queues"]
|
||||
expected["review_studio_phase_queues"] = expected["item_phase_queues"]
|
||||
return {
|
||||
"key": "world-class-phase-queue-consistency",
|
||||
"label": "World-class phase queues mirror repair rows",
|
||||
"status": "pass" if expected == actual else "fail",
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": [
|
||||
report_paths["world_class_preflight"],
|
||||
report_paths["world_class_operator_runbook"],
|
||||
report_paths["review_studio"],
|
||||
],
|
||||
"detail": (
|
||||
"Phase queues must be derived from repair rows in preflight and mirrored into the operator runbook "
|
||||
"and Review Studio without counting queue guidance as completion evidence."
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to verify release evidence refresh instructions."
|
||||
|
||||
SOURCE_REFRESH_HEADER = "After source changes that affect scripts"
|
||||
CLEAN_LOCK_HEADER = "For final release evidence"
|
||||
CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`"
|
||||
|
||||
SOURCE_REFRESH_REPORT_COMMANDS = [
|
||||
'python3 scripts/run_output_execution.py --runner-command \'["python3","scripts/local_output_eval_runner.py"]\'',
|
||||
'python3 scripts/compile_skill.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip",
|
||||
'python3 scripts/simulate_install.py . --package-dir dist --install-root dist/install-simulation --output-json reports/install_simulation.json --output-md reports/install_simulation.md --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/trust_check.py . --output-json reports/security_trust_report.json --output-md reports/security_trust_report.md",
|
||||
'python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_adoption_drift_report.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_architecture_maintainability.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/python_compat_check.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/probe_runtime_permissions.py . --package-dir dist",
|
||||
'python3 scripts/render_review_waivers.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/render_review_annotations.py .",
|
||||
'python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json --today "$GENERATED_AT"',
|
||||
'python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_daily_skillops_report.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_weekly_curator_report.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_skill_os2_audit.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/render_skill_overview.py .",
|
||||
"python3 scripts/render_skill_interpretation.py .",
|
||||
"python3 scripts/render_review_viewer.py .",
|
||||
'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json",
|
||||
'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"',
|
||||
]
|
||||
CLEAN_LOCK_REPORT_COMMANDS = [
|
||||
'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_daily_skillops_report.py . --generated-at "$GENERATED_AT"',
|
||||
'python3 scripts/render_weekly_curator_report.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/render_skill_overview.py .",
|
||||
"python3 scripts/render_skill_interpretation.py .",
|
||||
"python3 scripts/render_review_viewer.py .",
|
||||
'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"',
|
||||
"python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json",
|
||||
'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"',
|
||||
]
|
||||
|
||||
|
||||
def section_between(text: str, start: str, end: str) -> str:
|
||||
if start not in text:
|
||||
return ""
|
||||
section = text.split(start, 1)[1]
|
||||
if end in section:
|
||||
section = section.split(end, 1)[0]
|
||||
return section
|
||||
|
||||
|
||||
def command_presence(section: str, commands: list[str]) -> dict[str, bool]:
|
||||
return {command: command in section for command in commands}
|
||||
|
||||
|
||||
def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]:
|
||||
agents_path = skill_dir / "AGENTS.md"
|
||||
agents_text = agents_path.read_text(encoding="utf-8") if agents_path.exists() else ""
|
||||
source_refresh = section_between(agents_text, SOURCE_REFRESH_HEADER, CLEAN_LOCK_HEADER)
|
||||
clean_lock = section_between(agents_text, CLEAN_LOCK_HEADER, CLEAN_LOCK_END)
|
||||
expected = {
|
||||
"AGENTS.md": True,
|
||||
"source_refresh_section": True,
|
||||
"clean_lock_section": True,
|
||||
"source_refresh_commands": {command: True for command in SOURCE_REFRESH_REPORT_COMMANDS},
|
||||
"clean_lock_commands": {command: True for command in CLEAN_LOCK_REPORT_COMMANDS},
|
||||
}
|
||||
actual = {
|
||||
"AGENTS.md": agents_path.exists(),
|
||||
"source_refresh_section": bool(source_refresh),
|
||||
"clean_lock_section": bool(clean_lock),
|
||||
"source_refresh_commands": command_presence(source_refresh, SOURCE_REFRESH_REPORT_COMMANDS),
|
||||
"clean_lock_commands": command_presence(clean_lock, CLEAN_LOCK_REPORT_COMMANDS),
|
||||
}
|
||||
return {
|
||||
"key": "release-evidence-flow-covers-first-class-reports",
|
||||
"label": "Release evidence flow covers first-class reports",
|
||||
"status": "pass" if expected == actual else "fail",
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": [
|
||||
"AGENTS.md",
|
||||
"reports/output_execution_runs.json",
|
||||
"reports/install_simulation.json",
|
||||
"reports/security_trust_report.json",
|
||||
"reports/registry_audit.json",
|
||||
"reports/package_verification.json",
|
||||
"reports/upgrade_check.json",
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/architecture_maintainability.json",
|
||||
"reports/python_compatibility.json",
|
||||
"reports/runtime_permission_probes.json",
|
||||
"reports/review_waivers.json",
|
||||
"reports/review_annotations.json",
|
||||
"reports/skill_atlas.json",
|
||||
"reports/skill_os2_audit.json",
|
||||
"reports/skill_os2_coverage.json",
|
||||
"reports/context_budget.json",
|
||||
"reports/context_budget_summary.json",
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-overview.json",
|
||||
"reports/skill-interpretation.json",
|
||||
"reports/review-viewer.json",
|
||||
"reports/world_class_evidence_preflight.json",
|
||||
"reports/skillops/daily",
|
||||
"reports/skillops/weekly",
|
||||
"reports/review-studio.json",
|
||||
"reports/evidence_consistency.json",
|
||||
],
|
||||
"detail": (
|
||||
"Release refresh and clean-lock instructions must regenerate every first-class report "
|
||||
"before evidence consistency can be trusted."
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import ast
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = (
|
||||
"Imported by render_evidence_consistency.py to keep Skill OS 2.0 review summary drift checks "
|
||||
"out of the main consistency renderer."
|
||||
)
|
||||
|
||||
|
||||
def ci_default_target_count(path: Path) -> int | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
for node in tree.body:
|
||||
if not isinstance(node, ast.Assign):
|
||||
continue
|
||||
if not any(isinstance(target, ast.Name) and target.id == "DEFAULT_TARGETS" for target in node.targets):
|
||||
continue
|
||||
if isinstance(node.value, (ast.List, ast.Tuple)):
|
||||
return len(node.value.elts)
|
||||
return None
|
||||
|
||||
|
||||
def build_skill_os2_review_current_evidence_check(
|
||||
*,
|
||||
skill_dir: Path,
|
||||
skill_os2_review: str,
|
||||
studio_summary: dict[str, Any],
|
||||
trust_summary: dict[str, Any],
|
||||
package_summary: dict[str, Any],
|
||||
install_summary: dict[str, Any],
|
||||
benchmark_summary: dict[str, Any],
|
||||
context_stats: dict[str, Any],
|
||||
required_text_reports: dict[str, str],
|
||||
required_reports: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
ci_target_count = ci_default_target_count(skill_dir / "scripts" / "ci_test.py")
|
||||
expected_review_snippets = [
|
||||
f"score `{studio_summary.get('world_class_score')}`",
|
||||
f"`{studio_summary.get('gate_count')}` gates",
|
||||
f"`{studio_summary.get('warning_count')}` warnings",
|
||||
f"`{trust_summary.get('internal_module_count')}` declared internal modules",
|
||||
(
|
||||
f"`{trust_summary.get('help_smoke_checked_count')} / {trust_summary.get('help_smoke_checked_count')}` "
|
||||
f"CLI help smoke checks passing across `{trust_summary.get('script_count')}` scripts"
|
||||
),
|
||||
f"`{package_summary.get('archive_entry_count')}` zip entries",
|
||||
f"archive with `{package_summary.get('archive_entry_count')}` entries",
|
||||
f"`{install_summary.get('installer_permission_enforced_count')}` installer permission checks enforced",
|
||||
f"`{install_summary.get('installer_permission_failure_count')}` permission failures",
|
||||
f"`{benchmark_summary.get('required_artifact_count')}` required artifacts",
|
||||
f"`{benchmark_summary.get('command_count')}` reproduction commands",
|
||||
(
|
||||
f"initial load `{context_stats.get('estimated_initial_load_tokens')}/"
|
||||
f"{context_stats.get('context_budget_limit')}`"
|
||||
),
|
||||
f"target count is `{ci_target_count}`",
|
||||
]
|
||||
missing_review_snippets = [snippet for snippet in expected_review_snippets if snippet not in skill_os2_review]
|
||||
return {
|
||||
"key": "skill-os-2-review-current-evidence",
|
||||
"label": "Skill OS 2.0 review summary mirrors current evidence",
|
||||
"status": "pass" if not missing_review_snippets else "fail",
|
||||
"expected": expected_review_snippets,
|
||||
"actual": "all present" if not missing_review_snippets else {"missing": missing_review_snippets},
|
||||
"paths": [
|
||||
required_text_reports["skill_os2_review"],
|
||||
required_reports["review_studio"],
|
||||
required_reports["package_verification"],
|
||||
required_reports["install_simulation"],
|
||||
required_reports["trust"],
|
||||
required_reports["context_budget"],
|
||||
required_reports["benchmark"],
|
||||
"scripts/ci_test.py",
|
||||
],
|
||||
"detail": (
|
||||
"Manual 2.0 review summaries must not drift from generated gate, package, trust, "
|
||||
"context, benchmark, or CI evidence."
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks."
|
||||
|
||||
|
||||
def keyed_items(payload: dict[str, Any], collection_key: str) -> dict[str, dict[str, Any]]:
|
||||
collection = payload.get(collection_key)
|
||||
if not isinstance(collection, list):
|
||||
return {}
|
||||
keyed: dict[str, dict[str, Any]] = {}
|
||||
for item in collection:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = item.get("key") or item.get("evidence_key")
|
||||
if isinstance(key, str):
|
||||
keyed[key] = item
|
||||
return keyed
|
||||
|
||||
|
||||
def command_key_set(item: dict[str, Any]) -> set[str]:
|
||||
commands = item.get("commands")
|
||||
if isinstance(commands, dict):
|
||||
return {key for key, value in commands.items() if isinstance(key, str) and value}
|
||||
if isinstance(commands, list):
|
||||
keys: set[str] = set()
|
||||
for command in commands:
|
||||
if isinstance(command, dict) and isinstance(command.get("key"), str) and command.get("command"):
|
||||
keys.add(command["key"])
|
||||
return keys
|
||||
return set()
|
||||
|
||||
|
||||
def world_class_review_action_steps(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
actions = review_studio.get("review_actions")
|
||||
if not isinstance(actions, list):
|
||||
return {}
|
||||
for action in actions:
|
||||
if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence":
|
||||
continue
|
||||
steps = action.get("evidence_steps")
|
||||
if not isinstance(steps, list):
|
||||
return {}
|
||||
keyed: dict[str, dict[str, Any]] = {}
|
||||
for step in steps:
|
||||
if isinstance(step, dict) and isinstance(step.get("key"), str):
|
||||
keyed[step["key"]] = step
|
||||
return keyed
|
||||
return {}
|
||||
|
||||
|
||||
def command_groups_present(command_keys: set[str]) -> dict[str, bool]:
|
||||
return {
|
||||
"prepare_submission": "prepare_submission" in command_keys,
|
||||
"validate_intake": "validate_intake" in command_keys,
|
||||
"submission_review": bool({"submission_review", "review_queue"} & command_keys),
|
||||
"refresh_ledger": "refresh_ledger" in command_keys,
|
||||
"guard_claim": "guard_claim" in command_keys,
|
||||
}
|
||||
|
||||
|
||||
def has_next_action(item: dict[str, Any]) -> bool:
|
||||
for key in ["next_action", "audit_next_action"]:
|
||||
value = item.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def command_expectations(keys: list[str]) -> dict[str, dict[str, dict[str, bool]]]:
|
||||
expected_group = {
|
||||
"prepare_submission": True,
|
||||
"validate_intake": True,
|
||||
"submission_review": True,
|
||||
"refresh_ledger": True,
|
||||
"guard_claim": True,
|
||||
}
|
||||
return {
|
||||
key: {
|
||||
"intake": dict(expected_group),
|
||||
"operator_runbook": dict(expected_group),
|
||||
"review_studio": dict(expected_group),
|
||||
}
|
||||
for key in keys
|
||||
}
|
||||
|
||||
|
||||
def build_world_class_workflow_check(
|
||||
*,
|
||||
ledger: dict[str, Any],
|
||||
world_class_plan: dict[str, Any],
|
||||
world_class_intake: dict[str, Any],
|
||||
world_class_submission_review: dict[str, Any],
|
||||
world_class_operator_runbook: dict[str, Any],
|
||||
review_studio: dict[str, Any],
|
||||
report_paths: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
ledger_summary = ledger.get("summary") if isinstance(ledger.get("summary"), dict) else {}
|
||||
plan_summary = world_class_plan.get("summary") if isinstance(world_class_plan.get("summary"), dict) else {}
|
||||
intake_summary = world_class_intake.get("summary") if isinstance(world_class_intake.get("summary"), dict) else {}
|
||||
submission_review_summary = (
|
||||
world_class_submission_review.get("summary")
|
||||
if isinstance(world_class_submission_review.get("summary"), dict)
|
||||
else {}
|
||||
)
|
||||
operator_runbook_summary = (
|
||||
world_class_operator_runbook.get("summary")
|
||||
if isinstance(world_class_operator_runbook.get("summary"), dict)
|
||||
else {}
|
||||
)
|
||||
|
||||
ledger_items = keyed_items(ledger, "entries")
|
||||
plan_tasks = keyed_items(world_class_plan, "tasks")
|
||||
intake_checklist = keyed_items(world_class_intake, "operator_checklist")
|
||||
submission_review_items = keyed_items(world_class_submission_review, "items")
|
||||
operator_runbook_items = keyed_items(world_class_operator_runbook, "items")
|
||||
coordination_plan = (
|
||||
world_class_operator_runbook.get("coordination_plan")
|
||||
if isinstance(world_class_operator_runbook.get("coordination_plan"), list)
|
||||
else []
|
||||
)
|
||||
release_gate = (
|
||||
world_class_operator_runbook.get("release_gate")
|
||||
if isinstance(world_class_operator_runbook.get("release_gate"), dict)
|
||||
else {}
|
||||
)
|
||||
review_action_steps = world_class_review_action_steps(review_studio)
|
||||
pending_keys = sorted(key for key, item in ledger_items.items() if item.get("status") != "accepted")
|
||||
operator_coordination_keys = sorted(
|
||||
key
|
||||
for key in {str(step.get("evidence_key", "")) for step in coordination_plan if isinstance(step, dict)}
|
||||
if key
|
||||
)
|
||||
|
||||
actual_command_groups = {
|
||||
key: {
|
||||
"intake": command_groups_present(command_key_set(intake_checklist.get(key, {}))),
|
||||
"operator_runbook": command_groups_present(command_key_set(operator_runbook_items.get(key, {}))),
|
||||
"review_studio": command_groups_present(command_key_set(review_action_steps.get(key, {}))),
|
||||
}
|
||||
for key in pending_keys
|
||||
}
|
||||
expected = {
|
||||
"keys": pending_keys,
|
||||
"pending_count": ledger_summary.get("pending_count"),
|
||||
"human_pending_count": ledger_summary.get("human_pending_count"),
|
||||
"external_pending_count": ledger_summary.get("external_pending_count"),
|
||||
"source_check_count": ledger_summary.get("source_check_count"),
|
||||
"source_pass_count": ledger_summary.get("source_pass_count"),
|
||||
"source_blocked_count": ledger_summary.get("source_blocked_count"),
|
||||
"plan_keys": pending_keys,
|
||||
"intake_keys": pending_keys,
|
||||
"submission_review_keys": pending_keys,
|
||||
"operator_runbook_keys": pending_keys,
|
||||
"operator_coordination_keys": pending_keys,
|
||||
"operator_coordination_counts_as_completion": False,
|
||||
"operator_release_gate_ready": ledger_summary.get("ready_to_claim_world_class") is True,
|
||||
"operator_release_gate_counts_as_completion": False,
|
||||
"review_studio_keys": pending_keys,
|
||||
"intake_ready_to_claim_world_class": False,
|
||||
"submission_review_ready_to_claim_world_class": False,
|
||||
"submission_review_counts_as_completion": False,
|
||||
"operator_runbook_ready_to_claim_world_class": False,
|
||||
"operator_runbook_counts_as_completion": False,
|
||||
"next_actions_present": {key: True for key in pending_keys},
|
||||
"commands": command_expectations(pending_keys),
|
||||
}
|
||||
actual = {
|
||||
"keys": pending_keys,
|
||||
"pending_count": plan_summary.get("task_count"),
|
||||
"human_pending_count": plan_summary.get("human_task_count"),
|
||||
"external_pending_count": plan_summary.get("external_task_count"),
|
||||
"source_check_count": submission_review_summary.get("source_check_count"),
|
||||
"source_pass_count": submission_review_summary.get("source_pass_count"),
|
||||
"source_blocked_count": submission_review_summary.get("source_blocked_count"),
|
||||
"plan_keys": sorted(plan_tasks),
|
||||
"intake_keys": sorted(intake_checklist),
|
||||
"submission_review_keys": sorted(submission_review_items),
|
||||
"operator_runbook_keys": sorted(operator_runbook_items),
|
||||
"operator_coordination_keys": operator_coordination_keys,
|
||||
"operator_coordination_counts_as_completion": operator_runbook_summary.get(
|
||||
"coordination_counts_as_completion"
|
||||
),
|
||||
"operator_release_gate_ready": release_gate.get("ready"),
|
||||
"operator_release_gate_counts_as_completion": release_gate.get("counts_as_completion"),
|
||||
"review_studio_keys": sorted(review_action_steps),
|
||||
"intake_ready_to_claim_world_class": intake_summary.get("ready_to_claim_world_class"),
|
||||
"submission_review_ready_to_claim_world_class": submission_review_summary.get("ready_to_claim_world_class"),
|
||||
"submission_review_counts_as_completion": submission_review_summary.get(
|
||||
"review_counts_submission_as_completion"
|
||||
),
|
||||
"operator_runbook_ready_to_claim_world_class": operator_runbook_summary.get("ready_to_claim_world_class"),
|
||||
"operator_runbook_counts_as_completion": operator_runbook_summary.get("runbook_counts_as_completion"),
|
||||
"next_actions_present": {
|
||||
key: all(
|
||||
has_next_action(collection.get(key, {}))
|
||||
for collection in [plan_tasks, intake_checklist, submission_review_items, review_action_steps]
|
||||
)
|
||||
for key in pending_keys
|
||||
},
|
||||
"commands": actual_command_groups,
|
||||
}
|
||||
return {
|
||||
"key": "world-class-evidence-workflow-coverage",
|
||||
"label": "World-class evidence workflows cover every pending ledger entry",
|
||||
"status": "pass" if expected == actual else "fail",
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": [
|
||||
report_paths["world_class_ledger"],
|
||||
report_paths["world_class_plan"],
|
||||
report_paths["world_class_intake"],
|
||||
report_paths["world_class_submission_review"],
|
||||
report_paths["world_class_operator_runbook"],
|
||||
report_paths["review_studio"],
|
||||
],
|
||||
"detail": (
|
||||
"Every pending world-class evidence key must have matching plan, intake, submission review, "
|
||||
"operator runbook, and Review Studio actions without counting planned work as completion."
|
||||
),
|
||||
}
|
||||
@@ -51,6 +51,8 @@ KEY_REPORTS = [
|
||||
"reports/security_trust_report.md",
|
||||
"reports/runtime_permission_probes.json",
|
||||
"reports/runtime_permission_probes.md",
|
||||
"reports/telemetry_hook_recipes.json",
|
||||
"reports/telemetry_hook_recipes.md",
|
||||
"reports/skill_atlas.json",
|
||||
"reports/skill_atlas.html",
|
||||
"reports/skill-os-2-review.md",
|
||||
@@ -157,7 +159,7 @@ def report_list(skill_dir: Path) -> list[str]:
|
||||
return [rel for rel in KEY_REPORTS if (skill_dir / rel).exists()]
|
||||
|
||||
|
||||
def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, limit: int = 80) -> list[str]:
|
||||
def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, limit: int | None = None) -> list[str]:
|
||||
target = skill_dir / folder
|
||||
if not target.exists():
|
||||
return []
|
||||
@@ -168,7 +170,7 @@ def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, li
|
||||
if suffixes is not None and path.suffix not in suffixes:
|
||||
continue
|
||||
paths.append(str(path.relative_to(skill_dir)))
|
||||
if len(paths) >= limit:
|
||||
if limit is not None and len(paths) >= limit:
|
||||
break
|
||||
return paths
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared HTML rendering helpers for static report generators."""
|
||||
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Used by report renderers to escape HTML while preserving meaningful falsey values."
|
||||
|
||||
|
||||
def html_text(value: Any) -> str:
|
||||
"""Escape text for HTML without dropping 0 or False values."""
|
||||
return html.escape("" if value is None else str(value), quote=True)
|
||||
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from adjudicate_output_review import (
|
||||
DEFAULT_ANSWER_KEY,
|
||||
DEFAULT_BLIND_PACK,
|
||||
DEFAULT_DECISIONS,
|
||||
DEFAULT_OUTPUT_JSON,
|
||||
DEFAULT_OUTPUT_MD,
|
||||
adjudicate_output_review,
|
||||
build_decision_template,
|
||||
default_reviewer_attestation,
|
||||
normalize_variant,
|
||||
pair_index,
|
||||
review_integrity,
|
||||
)
|
||||
from output_review_privacy import forbidden_decision_field_paths
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Imports human blind A/B reviewer decisions into the canonical output-review decision file."
|
||||
|
||||
|
||||
def display_path(path: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(ROOT.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def load_json(path: Path) -> tuple[Any, list[str]]:
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8")), []
|
||||
except json.JSONDecodeError as exc:
|
||||
return None, [f"Invalid JSON file {display_path(path)}: {exc}"]
|
||||
|
||||
|
||||
def detect_format(path: Path, requested: str) -> str:
|
||||
if requested != "auto":
|
||||
return requested
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".jsonl":
|
||||
return "jsonl"
|
||||
if suffix == ".csv":
|
||||
return "csv"
|
||||
return "json"
|
||||
|
||||
|
||||
def read_decision_source(path: Path, source_format: str) -> tuple[dict[str, Any], list[dict[str, Any]], list[str]]:
|
||||
if not path.exists():
|
||||
return {}, [], [f"Missing decision source: {display_path(path)}"]
|
||||
if source_format == "json":
|
||||
payload, failures = load_json(path)
|
||||
if failures:
|
||||
return {}, [], failures
|
||||
if isinstance(payload, list):
|
||||
return {}, payload, []
|
||||
if not isinstance(payload, dict):
|
||||
return {}, [], ["JSON decision source must be an object or list"]
|
||||
decisions = payload.get("decisions", [])
|
||||
if not isinstance(decisions, list):
|
||||
return payload, [], ["decisions must be a list"]
|
||||
return payload, decisions, []
|
||||
if source_format == "jsonl":
|
||||
decisions: list[dict[str, Any]] = []
|
||||
failures: list[str] = []
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"JSONL line {line_number} is invalid: {exc}")
|
||||
continue
|
||||
if not isinstance(item, dict):
|
||||
failures.append(f"JSONL line {line_number} must be an object")
|
||||
continue
|
||||
decisions.append(item)
|
||||
return {}, decisions, failures
|
||||
if source_format == "csv":
|
||||
with path.open("r", encoding="utf-8", newline="") as handle:
|
||||
return {}, [dict(row) for row in csv.DictReader(handle)], []
|
||||
return {}, [], [f"Unsupported decision source format: {source_format}"]
|
||||
|
||||
|
||||
def known_case_ids(blind_pack_path: Path) -> tuple[set[str], list[str]]:
|
||||
payload, failures = load_json(blind_pack_path)
|
||||
if failures:
|
||||
return set(), failures
|
||||
if not isinstance(payload, dict):
|
||||
return set(), [f"Blind pack root must be an object: {display_path(blind_pack_path)}"]
|
||||
pairs = pair_index(payload)
|
||||
if not pairs:
|
||||
return set(), [f"Blind pack has no pairs: {display_path(blind_pack_path)}"]
|
||||
return set(pairs), []
|
||||
|
||||
|
||||
def parse_confidence(value: Any, case_id: str) -> tuple[float | None, str | None]:
|
||||
if value is None or str(value).strip() == "":
|
||||
return None, None
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None, f"{case_id}: confidence must be numeric"
|
||||
if parsed < 0 or parsed > 1:
|
||||
return None, f"{case_id}: confidence must be between 0 and 1"
|
||||
return round(parsed, 3), None
|
||||
|
||||
|
||||
def normalize_decisions(
|
||||
source_items: list[dict[str, Any]],
|
||||
case_ids: set[str],
|
||||
) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
failures: list[str] = []
|
||||
normalized: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for index, item in enumerate(source_items, start=1):
|
||||
if not isinstance(item, dict):
|
||||
failures.append(f"decision #{index} must be an object")
|
||||
continue
|
||||
blocked_fields = forbidden_decision_field_paths(item, f"decision #{index}")
|
||||
if blocked_fields:
|
||||
failures.append(f"decision #{index} contains forbidden raw or answer-key fields: {', '.join(blocked_fields)}")
|
||||
case_id = str(item.get("case_id", "")).strip()
|
||||
if not case_id:
|
||||
failures.append(f"decision #{index} is missing case_id")
|
||||
continue
|
||||
if case_id not in case_ids:
|
||||
failures.append(f"decision #{index} references unknown case_id: {case_id}")
|
||||
if case_id in seen:
|
||||
failures.append(f"duplicate decision for case_id: {case_id}")
|
||||
seen.add(case_id)
|
||||
winner = normalize_variant(item.get("winner_variant", item.get("winner", item.get("reviewer_winner_variant", ""))))
|
||||
if winner and winner not in {"A", "B"}:
|
||||
failures.append(f"{case_id}: winner_variant must be A or B")
|
||||
confidence, confidence_failure = parse_confidence(item.get("confidence"), case_id)
|
||||
if confidence_failure:
|
||||
failures.append(confidence_failure)
|
||||
reason = str(item.get("reason", "")).strip()
|
||||
if winner in {"A", "B"} and not reason:
|
||||
failures.append(f"{case_id}: reason is required for imported human decisions")
|
||||
normalized.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"winner_variant": winner,
|
||||
"confidence": confidence,
|
||||
"reason": reason,
|
||||
}
|
||||
)
|
||||
return normalized, failures
|
||||
|
||||
|
||||
def canonical_payload(
|
||||
blind_pack_path: Path,
|
||||
source_path: Path,
|
||||
source_format: str,
|
||||
reviewer: str,
|
||||
reviewed_at: str,
|
||||
decisions: list[dict[str, Any]],
|
||||
reviewer_attestation: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
blind_pack = json.loads(blind_pack_path.read_text(encoding="utf-8"))
|
||||
template = build_decision_template(blind_pack)
|
||||
completed = sum(1 for item in decisions if item.get("winner_variant") in {"A", "B"})
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"reviewer": reviewer,
|
||||
"reviewed_at": reviewed_at,
|
||||
"review_integrity": review_integrity(blind_pack),
|
||||
"reviewer_attestation": reviewer_attestation,
|
||||
"decision_contract": template["decision_contract"],
|
||||
"import_contract": {
|
||||
"schema_version": "1.0",
|
||||
"source_path": display_path(source_path),
|
||||
"source_format": source_format,
|
||||
"raw_content_allowed": False,
|
||||
"answer_key_fields_allowed": False,
|
||||
"answer_key_opened_by_importer": False,
|
||||
"all_or_nothing_write": True,
|
||||
"decision_count": len(decisions),
|
||||
"completed_decision_count": completed,
|
||||
"pending_decision_count": len(decisions) - completed,
|
||||
"blind_review_attestation_supported": True,
|
||||
"blind_review_attested": reviewer_attestation.get("blind_review_completed_before_answer_key") is True
|
||||
and reviewer_attestation.get("answer_key_not_opened_before_decisions") is True,
|
||||
},
|
||||
"decisions": decisions,
|
||||
}
|
||||
|
||||
|
||||
def normalize_reviewer_attestation(source_meta: dict[str, Any], blind_review_attested: bool) -> dict[str, Any]:
|
||||
attestation = default_reviewer_attestation()
|
||||
source_attestation = source_meta.get("reviewer_attestation", {})
|
||||
if isinstance(source_attestation, dict):
|
||||
for key in attestation:
|
||||
if source_attestation.get(key) is True:
|
||||
attestation[key] = True
|
||||
elif source_attestation.get(key) is False:
|
||||
attestation[key] = False
|
||||
if blind_review_attested:
|
||||
attestation["blind_review_completed_before_answer_key"] = True
|
||||
attestation["answer_key_not_opened_before_decisions"] = True
|
||||
return attestation
|
||||
|
||||
|
||||
def import_output_review_decisions(
|
||||
source_path: Path,
|
||||
blind_pack_path: Path,
|
||||
output_json: Path,
|
||||
source_format: str = "auto",
|
||||
reviewer: str = "",
|
||||
reviewed_at: str = "",
|
||||
run_adjudication: bool = False,
|
||||
answer_key_path: Path = DEFAULT_ANSWER_KEY,
|
||||
adjudication_json: Path = DEFAULT_OUTPUT_JSON,
|
||||
adjudication_md: Path = DEFAULT_OUTPUT_MD,
|
||||
blind_review_attested: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
detected_format = detect_format(source_path, source_format)
|
||||
source_meta, source_items, failures = read_decision_source(source_path, detected_format)
|
||||
reviewer = reviewer or str(source_meta.get("reviewer", "")).strip()
|
||||
reviewed_at = reviewed_at or str(source_meta.get("reviewed_at", "")).strip()
|
||||
if not reviewer:
|
||||
failures.append("reviewer is required for imported human decisions")
|
||||
if not reviewed_at:
|
||||
failures.append("reviewed_at is required for imported human decisions")
|
||||
reviewer_attestation = normalize_reviewer_attestation(source_meta, blind_review_attested)
|
||||
case_ids, case_failures = known_case_ids(blind_pack_path)
|
||||
failures.extend(case_failures)
|
||||
decisions, decision_failures = normalize_decisions(source_items, case_ids)
|
||||
failures.extend(decision_failures)
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
"summary": {
|
||||
"decision_count": len(decisions),
|
||||
"completed_decision_count": sum(1 for item in decisions if item.get("winner_variant") in {"A", "B"}),
|
||||
"pending_decision_count": sum(1 for item in decisions if not item.get("winner_variant")),
|
||||
"failure_count": len(failures),
|
||||
"canonical_written": False,
|
||||
"adjudication_run": False,
|
||||
},
|
||||
"artifacts": {
|
||||
"source": display_path(source_path),
|
||||
"blind_pack": display_path(blind_pack_path),
|
||||
"decisions": display_path(output_json),
|
||||
"adjudication_json": display_path(adjudication_json),
|
||||
"adjudication_markdown": display_path(adjudication_md),
|
||||
},
|
||||
"failures": failures,
|
||||
}
|
||||
if failures:
|
||||
return payload
|
||||
canonical = canonical_payload(
|
||||
blind_pack_path,
|
||||
source_path,
|
||||
detected_format,
|
||||
reviewer,
|
||||
reviewed_at,
|
||||
decisions,
|
||||
reviewer_attestation,
|
||||
)
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(canonical, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
payload["summary"]["canonical_written"] = True
|
||||
payload["canonical"] = canonical
|
||||
if run_adjudication:
|
||||
adjudication = adjudicate_output_review(
|
||||
blind_pack_path=blind_pack_path,
|
||||
answer_key_path=answer_key_path,
|
||||
decisions_path=output_json,
|
||||
output_json=adjudication_json,
|
||||
output_md=adjudication_md,
|
||||
)
|
||||
payload["summary"]["adjudication_run"] = True
|
||||
payload["summary"]["adjudication_ok"] = adjudication["ok"]
|
||||
payload["summary"]["adjudication_pending_count"] = adjudication["summary"]["pending_count"]
|
||||
payload["summary"]["adjudication_invalid_decision_count"] = adjudication["summary"]["invalid_decision_count"]
|
||||
payload["adjudication_summary"] = adjudication["summary"]
|
||||
if not adjudication["ok"]:
|
||||
payload["ok"] = False
|
||||
payload["failures"] = list(adjudication.get("failures", []))
|
||||
payload["summary"]["failure_count"] = len(payload["failures"])
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Import blind A/B reviewer decisions into the canonical output-review decision file.")
|
||||
parser.add_argument("--input", required=True, help="Reviewer decision source in JSON, JSONL, or CSV format.")
|
||||
parser.add_argument("--format", choices=["auto", "json", "jsonl", "csv"], default="auto")
|
||||
parser.add_argument("--blind-pack", default=str(DEFAULT_BLIND_PACK))
|
||||
parser.add_argument("--output-json", default=str(DEFAULT_DECISIONS))
|
||||
parser.add_argument("--reviewer")
|
||||
parser.add_argument("--reviewed-at", default=date.today().isoformat())
|
||||
parser.add_argument(
|
||||
"--blind-review-attested",
|
||||
action="store_true",
|
||||
help="Attest that reviewer choices were completed before the answer key was opened.",
|
||||
)
|
||||
parser.add_argument("--run-adjudication", action="store_true")
|
||||
parser.add_argument("--answer-key", default=str(DEFAULT_ANSWER_KEY))
|
||||
parser.add_argument("--adjudication-json", default=str(DEFAULT_OUTPUT_JSON))
|
||||
parser.add_argument("--adjudication-md", default=str(DEFAULT_OUTPUT_MD))
|
||||
args = parser.parse_args()
|
||||
payload = import_output_review_decisions(
|
||||
source_path=Path(args.input).resolve(),
|
||||
blind_pack_path=Path(args.blind_pack).resolve(),
|
||||
output_json=Path(args.output_json).resolve(),
|
||||
source_format=args.format,
|
||||
reviewer=args.reviewer or "",
|
||||
reviewed_at=args.reviewed_at or "",
|
||||
run_adjudication=args.run_adjudication,
|
||||
answer_key_path=Path(args.answer_key).resolve(),
|
||||
adjudication_json=Path(args.adjudication_json).resolve(),
|
||||
adjudication_md=Path(args.adjudication_md).resolve(),
|
||||
blind_review_attested=args.blind_review_attested,
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if payload["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_adoption_drift_report import display_path, normalize_event, render_report, skill_defaults
|
||||
|
||||
|
||||
def load_candidate_events(
|
||||
input_jsonl: Path,
|
||||
defaults: dict[str, str],
|
||||
default_source: str,
|
||||
default_command: str,
|
||||
) -> tuple[list[dict[str, str]], list[str]]:
|
||||
events: list[dict[str, str]] = []
|
||||
failures: list[str] = []
|
||||
for index, line in enumerate(input_jsonl.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"line {index}: invalid JSONL event: {exc.msg}")
|
||||
continue
|
||||
if not isinstance(raw, dict):
|
||||
failures.append(f"line {index}: telemetry event must be a JSON object")
|
||||
continue
|
||||
raw.setdefault("source", default_source)
|
||||
raw.setdefault("command", default_command)
|
||||
event, event_failures = normalize_event(raw, defaults, f"line {index}")
|
||||
failures.extend(event_failures)
|
||||
if event:
|
||||
events.append(event)
|
||||
return events, failures
|
||||
|
||||
|
||||
def append_events(events_jsonl: Path, events: list[dict[str, str]]) -> None:
|
||||
events_jsonl.parent.mkdir(parents=True, exist_ok=True)
|
||||
with events_jsonl.open("a", encoding="utf-8") as handle:
|
||||
for event in events:
|
||||
handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def import_events(
|
||||
skill_dir: Path,
|
||||
input_jsonl: Path,
|
||||
events_jsonl: Path | None = None,
|
||||
output_json: Path | None = None,
|
||||
output_md: Path | None = None,
|
||||
generated_at: str | None = None,
|
||||
default_source: str = "external",
|
||||
default_command: str = "external-client",
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
input_jsonl = input_jsonl.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
events_jsonl = (events_jsonl or reports_dir / "telemetry_events.jsonl").resolve()
|
||||
output_json = (output_json or reports_dir / "adoption_drift_report.json").resolve()
|
||||
output_md = (output_md or reports_dir / "adoption_drift_report.md").resolve()
|
||||
failures: list[str] = []
|
||||
if not input_jsonl.exists():
|
||||
failures.append(f"Input telemetry JSONL does not exist: {display_path(input_jsonl)}")
|
||||
candidate_events: list[dict[str, str]] = []
|
||||
else:
|
||||
candidate_events, load_failures = load_candidate_events(
|
||||
input_jsonl,
|
||||
skill_defaults(skill_dir),
|
||||
default_source,
|
||||
default_command,
|
||||
)
|
||||
failures.extend(load_failures)
|
||||
|
||||
adoption_report: dict[str, Any] | None = None
|
||||
if not failures and not dry_run:
|
||||
append_events(events_jsonl, candidate_events)
|
||||
adoption_report = render_report(
|
||||
skill_dir,
|
||||
events_jsonl=events_jsonl,
|
||||
output_json=output_json,
|
||||
output_md=output_md,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": not failures,
|
||||
"schema_version": "1.0",
|
||||
"skill_dir": display_path(skill_dir),
|
||||
"input_jsonl": display_path(input_jsonl),
|
||||
"events_jsonl": display_path(events_jsonl),
|
||||
"dry_run": dry_run,
|
||||
"imported_count": 0 if failures or dry_run else len(candidate_events),
|
||||
"candidate_count": len(candidate_events),
|
||||
"failures": failures,
|
||||
"imported_preview": candidate_events[:5],
|
||||
"artifacts": {
|
||||
"events_jsonl": display_path(events_jsonl),
|
||||
"adoption_drift_json": display_path(output_json),
|
||||
"adoption_drift_md": display_path(output_md),
|
||||
},
|
||||
"adoption_drift": {
|
||||
"summary": adoption_report.get("summary", {}) if adoption_report else {},
|
||||
"artifacts": adoption_report.get("artifacts", {}) if adoption_report else {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Import external metadata-only telemetry into a local skill event stream.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--input-jsonl", required=True)
|
||||
parser.add_argument("--events-jsonl")
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--generated-at")
|
||||
parser.add_argument("--source", choices=["external", "manual", "unknown", "yao_cli"], default="external")
|
||||
parser.add_argument("--command", default="external-client")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = import_events(
|
||||
Path(args.skill_dir),
|
||||
Path(args.input_jsonl),
|
||||
events_jsonl=Path(args.events_jsonl) if args.events_jsonl else None,
|
||||
output_json=Path(args.output_json) if args.output_json else None,
|
||||
output_md=Path(args.output_md) if args.output_md else None,
|
||||
generated_at=args.generated_at,
|
||||
default_source=args.source,
|
||||
default_command=args.command,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+28
-16
@@ -21,6 +21,7 @@ from render_reference_scan import parse_reference, render_reference_scan
|
||||
from render_reference_synthesis import render_reference_synthesis
|
||||
from render_review_studio import render_review_studio
|
||||
from render_review_viewer import render_review_viewer
|
||||
from render_skill_interpretation import render_skill_interpretation
|
||||
from render_skill_overview import render_skill_overview
|
||||
from render_system_model import render_system_model
|
||||
|
||||
@@ -73,19 +74,20 @@ README_TEMPLATE = """# {title}
|
||||
4. Review `reports/intent-confidence.md` to see whether the real job, inputs, outputs, and exclusions are clear enough yet.
|
||||
5. Open `reports/reference-synthesis.md` to see the GitHub benchmarks plus curated official, research, and principle tracks in one place.
|
||||
6. Follow the workflow steps in `SKILL.md`.
|
||||
7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right.
|
||||
8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view.
|
||||
9. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up.
|
||||
10. Open `reports/review-viewer.html` for a compact visual review of the package.
|
||||
11. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
|
||||
12. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
|
||||
13. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix.
|
||||
14. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging.
|
||||
15. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts.
|
||||
16. Review `reports/iteration-directions.md` for the three most valuable next moves.
|
||||
17. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes.
|
||||
18. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals.
|
||||
19. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates.
|
||||
7. Open `reports/skill-interpretation.html` first for the first-class bilingual interpretation report: role, principle, scenarios, trigger, inputs, outputs, highlights, risks, assets, and upgrade directions. It defaults to Simplified Chinese and includes an English switch in the top right.
|
||||
8. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap.
|
||||
9. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view.
|
||||
10. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up.
|
||||
11. Open `reports/review-viewer.html` for a compact visual review of the package.
|
||||
12. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
|
||||
13. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
|
||||
14. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix.
|
||||
15. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging.
|
||||
16. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts.
|
||||
17. Review `reports/iteration-directions.md` for the three most valuable next moves.
|
||||
18. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes.
|
||||
19. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals.
|
||||
20. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates.
|
||||
|
||||
## Honest Boundaries
|
||||
|
||||
@@ -109,6 +111,7 @@ README_TEMPLATE = """# {title}
|
||||
- `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points
|
||||
- `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance
|
||||
- `reports/compiled_targets.md`: target compiler report showing generated contracts, adapter modes, preserved semantics, warnings, and unsupported features
|
||||
- `reports/skill-interpretation.html`: first-class bilingual interpretation report for role, principle, scenarios, trigger, inputs, outputs, quality evidence, risks, assets, highlights, and upgrade directions
|
||||
- `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap
|
||||
- `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness
|
||||
- `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps
|
||||
@@ -129,6 +132,7 @@ compatibility:
|
||||
- "openai"
|
||||
- "claude"
|
||||
- "generic"
|
||||
- "vscode"
|
||||
activation:
|
||||
mode: "manual"
|
||||
paths: []
|
||||
@@ -143,6 +147,7 @@ compatibility:
|
||||
openai: "metadata-adapter"
|
||||
claude: "neutral-source-plus-adapter"
|
||||
generic: "neutral-source"
|
||||
vscode: "agent-skills-source-with-vscode-notes"
|
||||
"""
|
||||
|
||||
|
||||
@@ -213,6 +218,7 @@ def build_manifest(name: str, mode: str, archetype: str) -> dict:
|
||||
"claude",
|
||||
"generic",
|
||||
"agent-skills-compatible",
|
||||
"vscode",
|
||||
],
|
||||
"factory_components": [
|
||||
"references",
|
||||
@@ -225,23 +231,26 @@ def build_manifest(name: str, mode: str, archetype: str) -> dict:
|
||||
def build_report_view(artifacts: dict) -> dict:
|
||||
html_report = artifacts.get("skill_overview_html", "")
|
||||
json_report = artifacts.get("skill_overview_json", "")
|
||||
interpretation_report = artifacts.get("skill_interpretation_html", "")
|
||||
system_model = artifacts.get("system_model_md", "")
|
||||
review_studio = artifacts.get("review_studio_html", "")
|
||||
return {
|
||||
"title": "Skill 总结报告",
|
||||
"html_report": html_report,
|
||||
"json_report": json_report,
|
||||
"interpretation_report": interpretation_report,
|
||||
"system_model": system_model,
|
||||
"review_studio": review_studio,
|
||||
"message": (
|
||||
f"Skill 已创建完成。建议先打开总结报告:{html_report}。"
|
||||
"它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线;"
|
||||
f"Skill 已创建完成。建议先打开解读报告:{interpretation_report};再查看总结报告:{html_report}。"
|
||||
"解读报告会用中文简体默认展示这个 Skill 的作用、原理、使用场景、触发方式、输入输出、亮点和后续升级方向;"
|
||||
"总结报告会展示概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线;"
|
||||
f"然后打开 Review Studio 2.0:{review_studio},检查意图、触发、输出评测、运行一致性、信任和发布闸门。"
|
||||
"后续 reviewer 的文件级或行级意见可以记录到 reports/review_annotations.md。"
|
||||
"如需审查平台适配细节,请打开 reports/compiled_targets.md。"
|
||||
"报告默认使用中文简体,右上角可以切换英文版。"
|
||||
),
|
||||
"next_action": "Open reports/skill-overview.html before editing more files.",
|
||||
"next_action": "Open reports/skill-interpretation.html before editing more files.",
|
||||
}
|
||||
|
||||
|
||||
@@ -356,6 +365,7 @@ def initialize_skill(
|
||||
review_waivers = render_review_waivers(root)
|
||||
review_annotations = render_review_annotations(root)
|
||||
overview = render_skill_overview(root)
|
||||
interpretation = render_skill_interpretation(root)
|
||||
review_viewer = render_review_viewer(root)
|
||||
review_studio = render_review_studio(root)
|
||||
artifacts = {
|
||||
@@ -366,6 +376,8 @@ def initialize_skill(
|
||||
"intent_context_json": intent_confidence["artifacts"]["context_json"],
|
||||
"skill_overview_html": overview["artifacts"]["html"],
|
||||
"skill_overview_json": overview["artifacts"]["json"],
|
||||
"skill_interpretation_html": interpretation["artifacts"]["html"],
|
||||
"skill_interpretation_json": interpretation["artifacts"]["json"],
|
||||
"intent_dialogue_md": intent_dialogue["artifacts"]["markdown"],
|
||||
"intent_dialogue_json": intent_dialogue["artifacts"]["json"],
|
||||
"reference_scan_md": reference_scan["artifacts"]["markdown"],
|
||||
|
||||
@@ -4,6 +4,7 @@ import json
|
||||
from pathlib import Path
|
||||
|
||||
from context_sizer import estimate_tokens
|
||||
from description_optimizer_reporting import render_markdown
|
||||
from judge_blind_eval import evaluate_judge
|
||||
from trigger_eval import (
|
||||
compare_reports,
|
||||
@@ -539,145 +540,6 @@ def optimize(
|
||||
return report
|
||||
|
||||
|
||||
def render_markdown(report: dict, title: str) -> str:
|
||||
lines = [
|
||||
f"# {title}",
|
||||
"",
|
||||
f"Winner: `{report['winner']['label']}`",
|
||||
"",
|
||||
f"- current tokens: `{report['current_candidate']['estimated_tokens']}`",
|
||||
f"- winner tokens: `{report['winner']['estimated_tokens']}`",
|
||||
]
|
||||
if report["baseline"]:
|
||||
lines.append(f"- baseline tokens: `{report['baseline']['estimated_tokens']}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Winner",
|
||||
"",
|
||||
report["winner"]["description"],
|
||||
"",
|
||||
"## Candidate Ranking",
|
||||
"",
|
||||
"| Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for candidate in report["candidates"]:
|
||||
holdout = candidate.get("holdout", {})
|
||||
lines.append(
|
||||
f"| `{candidate['label']}` | {candidate['estimated_tokens']} | {candidate['dev']['false_positives']} | {candidate['dev']['false_negatives']} | {candidate['dev']['near_neighbor_pass_rate']} | {holdout.get('false_positives', '-')} | {holdout.get('false_negatives', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Acceptance Gates",
|
||||
"",
|
||||
"| Gate | Winner FP | Winner FN | Current FP | Current FN | Baseline FP | Baseline FN |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for gate_name, gate in (
|
||||
("Holdout", report["acceptance_gates"]["holdout_non_regression"]),
|
||||
("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]),
|
||||
("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]),
|
||||
("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]),
|
||||
):
|
||||
winner_gate = gate.get("winner") or {}
|
||||
current_gate = gate.get("current") or {}
|
||||
baseline_gate = gate.get("baseline") or {}
|
||||
if not winner_gate and not current_gate and not baseline_gate:
|
||||
continue
|
||||
lines.append(
|
||||
f"| {gate_name} | {winner_gate.get('false_positives', '-')} | {winner_gate.get('false_negatives', '-')} | {current_gate.get('false_positives', '-')} | {current_gate.get('false_negatives', '-')} | {baseline_gate.get('false_positives', '-')} | {baseline_gate.get('false_negatives', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Calibration",
|
||||
"",
|
||||
"| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |",
|
||||
"| --- | ---: | --- | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
for gate_name, gate in (
|
||||
("Holdout", report["acceptance_gates"]["holdout_non_regression"]),
|
||||
("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]),
|
||||
("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]),
|
||||
("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]),
|
||||
):
|
||||
winner_calibration = gate.get("winner_calibration") or {}
|
||||
current_calibration = gate.get("current_calibration") or {}
|
||||
baseline_calibration = gate.get("baseline_calibration") or {}
|
||||
if not winner_calibration and not current_calibration and not baseline_calibration:
|
||||
continue
|
||||
lines.append(
|
||||
f"| {gate_name} | {winner_calibration.get('score_gap', '-')} | {winner_calibration.get('risk_band', '-')} | {winner_calibration.get('boundary_case_rate', '-')} | {current_calibration.get('score_gap', '-')} | {baseline_calibration.get('score_gap', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Judge Blind Summary",
|
||||
"",
|
||||
"| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement |",
|
||||
"| --- | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
)
|
||||
judge_gate = report["acceptance_gates"]["judge_blind_holdout_non_regression"]
|
||||
judge_winner = (judge_gate.get("winner") or {}).get("judge_summary") or {}
|
||||
judge_current = (judge_gate.get("current") or {}).get("judge_summary") or {}
|
||||
judge_baseline = (judge_gate.get("baseline") or {}).get("judge_summary") or {}
|
||||
if judge_winner or judge_current or judge_baseline:
|
||||
lines.append(
|
||||
f"| Judge Blind Holdout | {judge_winner.get('agreement_rate', '-')} | {judge_winner.get('mean_confidence', '-')} | {judge_current.get('agreement_rate', '-')} | {judge_baseline.get('agreement_rate', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Family Health",
|
||||
"",
|
||||
"| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for gate_name, gate in (
|
||||
("Holdout", report["acceptance_gates"]["holdout_non_regression"]),
|
||||
("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]),
|
||||
("Judge Blind Holdout", report["acceptance_gates"]["judge_blind_holdout_non_regression"]),
|
||||
("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]),
|
||||
):
|
||||
winner_health = gate.get("winner_family_health") or {}
|
||||
current_health = gate.get("current_family_health") or {}
|
||||
baseline_health = gate.get("baseline_family_health") or {}
|
||||
if not winner_health and not current_health and not baseline_health:
|
||||
continue
|
||||
weakest = winner_health.get("weakest_family") or {}
|
||||
weakest_label = (
|
||||
f"{weakest.get('family')} ({weakest.get('errors')} errors)"
|
||||
if weakest.get("family")
|
||||
else "-"
|
||||
)
|
||||
lines.append(
|
||||
f"| {gate_name} | {winner_health.get('clean_family_count', '-')}/{winner_health.get('family_count', '-')} | {weakest_label} | {current_health.get('clean_family_count', '-')}/{current_health.get('family_count', '-')} | {baseline_health.get('clean_family_count', '-')}/{baseline_health.get('family_count', '-')} |"
|
||||
)
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Selection Logic",
|
||||
"",
|
||||
"Ordered by:",
|
||||
]
|
||||
)
|
||||
for item in report["selection_logic"]["priority"]:
|
||||
lines.append(f"- {item}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate and score description candidates on dev, holdout, blind, and adversarial suites."
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by output-review import and world-class human evidence validators to reject raw content, credential, secret, token, and answer-key fields recursively."
|
||||
|
||||
RAW_CONTENT_FIELDS = {
|
||||
"api_key",
|
||||
"assistant_message",
|
||||
"assistant_messages",
|
||||
"baseline_output",
|
||||
"credential",
|
||||
"credentials",
|
||||
"input",
|
||||
"inputs",
|
||||
"message",
|
||||
"messages",
|
||||
"model_output",
|
||||
"output",
|
||||
"outputs",
|
||||
"prompt",
|
||||
"prompts",
|
||||
"raw_content",
|
||||
"raw_output",
|
||||
"raw_prompt",
|
||||
"raw_provider_prompt",
|
||||
"raw_user_content",
|
||||
"secret",
|
||||
"secrets",
|
||||
"token",
|
||||
"transcript",
|
||||
"transcripts",
|
||||
"user_message",
|
||||
"user_messages",
|
||||
"with_skill_output",
|
||||
}
|
||||
|
||||
ANSWER_KEY_FIELDS = {
|
||||
"answer_key",
|
||||
"baseline_label",
|
||||
"expected",
|
||||
"expected_winner",
|
||||
"expected_winner_role",
|
||||
"expected_winner_variant",
|
||||
"label",
|
||||
"variant_label",
|
||||
"with_skill_label",
|
||||
}
|
||||
|
||||
BLOCKED_DECISION_FIELDS = RAW_CONTENT_FIELDS | ANSWER_KEY_FIELDS
|
||||
|
||||
|
||||
def forbidden_decision_field_paths(value: Any, prefix: str) -> list[str]:
|
||||
found: list[str] = []
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
key_text = str(key).strip()
|
||||
child_path = f"{prefix}.{key_text}"
|
||||
if key_text.casefold() in BLOCKED_DECISION_FIELDS:
|
||||
found.append(child_path)
|
||||
found.extend(forbidden_decision_field_paths(child, child_path))
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
found.extend(forbidden_decision_field_paths(child, f"{prefix}[{index}]"))
|
||||
return found
|
||||
|
||||
|
||||
def forbidden_raw_content_field_paths(value: Any, prefix: str) -> list[str]:
|
||||
found: list[str] = []
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
key_text = str(key).strip()
|
||||
child_path = f"{prefix}.{key_text}"
|
||||
if key_text.casefold() in RAW_CONTENT_FIELDS:
|
||||
found.append(child_path)
|
||||
found.extend(forbidden_raw_content_field_paths(child, child_path))
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
found.extend(forbidden_raw_content_field_paths(child, f"{prefix}[{index}]"))
|
||||
return found
|
||||
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a reviewer-facing blind A/B output review kit."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from adjudicate_output_review import (
|
||||
build_decision_template,
|
||||
confidence_value,
|
||||
decision_index,
|
||||
display_path,
|
||||
load_json,
|
||||
normalize_variant,
|
||||
)
|
||||
from html_rendering import html_text
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_BLIND_PACK_JSON = ROOT / "reports" / "output_blind_review_pack.json"
|
||||
DEFAULT_BLIND_PACK_MD = ROOT / "reports" / "output_blind_review_pack.md"
|
||||
DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json"
|
||||
DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_kit.json"
|
||||
DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_kit.md"
|
||||
DEFAULT_OUTPUT_HTML = ROOT / "reports" / "output_review_kit.html"
|
||||
|
||||
|
||||
def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]:
|
||||
if not path.exists():
|
||||
return {"schema_version": "1.0", "decisions": []}, []
|
||||
return load_json(path)
|
||||
|
||||
|
||||
def pairs_from_pack(blind_pack: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
pairs = blind_pack.get("pairs", [])
|
||||
if not isinstance(pairs, list):
|
||||
return []
|
||||
return [item for item in pairs if isinstance(item, dict) and item.get("case_id")]
|
||||
|
||||
|
||||
def decision_state(decision: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not decision:
|
||||
return {
|
||||
"status": "awaiting-decision",
|
||||
"winner_variant_recorded": False,
|
||||
"confidence_recorded": False,
|
||||
"reason_recorded": False,
|
||||
"blocking_reason": "No reviewer choice has been recorded yet.",
|
||||
}
|
||||
winner = normalize_variant(decision.get("winner_variant", decision.get("reviewer_winner_variant", decision.get("winner", ""))))
|
||||
confidence, confidence_failure = confidence_value(decision.get("confidence"))
|
||||
reason = str(decision.get("reason", "")).strip()
|
||||
if not winner and confidence is None and not reason:
|
||||
return {
|
||||
"status": "awaiting-decision",
|
||||
"winner_variant_recorded": False,
|
||||
"confidence_recorded": False,
|
||||
"reason_recorded": False,
|
||||
"blocking_reason": "Decision template exists but this row is still blank.",
|
||||
}
|
||||
failures = []
|
||||
if winner not in {"A", "B"}:
|
||||
failures.append("winner_variant must be A or B")
|
||||
if confidence_failure:
|
||||
failures.append(confidence_failure)
|
||||
if not reason:
|
||||
failures.append("reason should explain the reviewer rationale")
|
||||
if failures:
|
||||
return {
|
||||
"status": "needs-fix",
|
||||
"winner_variant_recorded": winner in {"A", "B"},
|
||||
"confidence_recorded": confidence is not None,
|
||||
"reason_recorded": bool(reason),
|
||||
"blocking_reason": "; ".join(failures),
|
||||
}
|
||||
return {
|
||||
"status": "ready-for-adjudication",
|
||||
"winner_variant_recorded": True,
|
||||
"confidence_recorded": confidence is not None,
|
||||
"reason_recorded": True,
|
||||
"blocking_reason": "Reviewer choice is complete enough for adjudication.",
|
||||
}
|
||||
|
||||
|
||||
def build_cases(blind_pack: dict[str, Any], decisions: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
cases = []
|
||||
for pair in pairs_from_pack(blind_pack):
|
||||
case_id = str(pair.get("case_id", ""))
|
||||
rubric = []
|
||||
for item in pair.get("rubric", []) if isinstance(pair.get("rubric", []), list) else []:
|
||||
if isinstance(item, dict):
|
||||
rubric.append(
|
||||
{
|
||||
"id": str(item.get("id", "")),
|
||||
"description": str(item.get("description", "")),
|
||||
"weight": float(item.get("weight", 1) or 0),
|
||||
}
|
||||
)
|
||||
cases.append(
|
||||
{
|
||||
"case_id": case_id,
|
||||
"prompt": str(pair.get("prompt", "")),
|
||||
"input_files": pair.get("input_files", []) if isinstance(pair.get("input_files", []), list) else [],
|
||||
"review_instruction": str(pair.get("review_instruction", "")),
|
||||
"rubric": rubric,
|
||||
"variant_a": {
|
||||
"blind_id": str(pair.get("variant_a", {}).get("blind_id", "")) if isinstance(pair.get("variant_a"), dict) else "",
|
||||
"output": str(pair.get("variant_a", {}).get("output", "")) if isinstance(pair.get("variant_a"), dict) else "",
|
||||
},
|
||||
"variant_b": {
|
||||
"blind_id": str(pair.get("variant_b", {}).get("blind_id", "")) if isinstance(pair.get("variant_b"), dict) else "",
|
||||
"output": str(pair.get("variant_b", {}).get("output", "")) if isinstance(pair.get("variant_b"), dict) else "",
|
||||
},
|
||||
"decision_state": decision_state(decisions.get(case_id)),
|
||||
}
|
||||
)
|
||||
return cases
|
||||
|
||||
|
||||
def build_summary(cases: list[dict[str, Any]], failures: list[str], reviewer: str, reviewed_at: str) -> dict[str, Any]:
|
||||
status_counts: dict[str, int] = {}
|
||||
for case in cases:
|
||||
status = str(case.get("decision_state", {}).get("status", "awaiting-decision"))
|
||||
status_counts[status] = status_counts.get(status, 0) + 1
|
||||
ready_count = status_counts.get("ready-for-adjudication", 0)
|
||||
pending_count = status_counts.get("awaiting-decision", 0)
|
||||
invalid_count = status_counts.get("needs-fix", 0)
|
||||
return {
|
||||
"case_count": len(cases),
|
||||
"ready_for_adjudication_count": ready_count,
|
||||
"pending_decision_count": pending_count,
|
||||
"invalid_decision_count": invalid_count,
|
||||
"reviewer_metadata_present": bool(str(reviewer).strip() and str(reviewed_at).strip()),
|
||||
"answer_key_hidden": True,
|
||||
"answer_key_path_exposed": False,
|
||||
"ready_to_run_adjudication": bool(cases) and ready_count == len(cases) and invalid_count == 0,
|
||||
"failure_count": len(failures),
|
||||
}
|
||||
|
||||
|
||||
def build_contract(blind_pack_md: Path, decisions_path: Path) -> dict[str, Any]:
|
||||
return {
|
||||
"reviewer_steps": [
|
||||
f"Open {display_path(blind_pack_md)} or this kit and compare Variant A vs Variant B for each case.",
|
||||
f"Record choices in {display_path(decisions_path)} without opening the answer key.",
|
||||
"Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.",
|
||||
"Set reviewer_attestation truthfully after choices are recorded and before adjudication.",
|
||||
"Run python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication after choices are recorded.",
|
||||
"Refresh python3 scripts/yao.py review-studio . before asking for release approval.",
|
||||
],
|
||||
"required_fields": {
|
||||
"reviewer": "Human reviewer name or review group.",
|
||||
"reviewed_at": "Review date or timestamp.",
|
||||
"winner_variant": "A or B for every case.",
|
||||
"confidence": "Optional numeric confidence from 0 to 1.",
|
||||
"reason": "Required rationale based on the rubric, not on hidden labels.",
|
||||
"reviewer_attestation": "Blind-review and answer-key-order attestation after decisions are complete.",
|
||||
"review_integrity": "Blind pack and prompt fingerprints generated by the template.",
|
||||
},
|
||||
"privacy_contract": [
|
||||
"The answer key is intentionally withheld from this kit.",
|
||||
"Do not inspect hidden labels or expected winners before decisions are recorded.",
|
||||
"Do not paste private user data into decision reasons.",
|
||||
"Pending decisions must stay pending instead of being counted as human agreement.",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def render_rubric(rubric: list[dict[str, Any]]) -> list[str]:
|
||||
if not rubric:
|
||||
return ["- No rubric items found."]
|
||||
return [f"- `{item['id']}` ({item['weight']}): {item['description']}" for item in rubric]
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
summary = payload["summary"]
|
||||
contract = payload["review_contract"]
|
||||
lines = [
|
||||
"# Output Review Kit",
|
||||
"",
|
||||
"This reviewer-facing packet contains the blind A/B cases, decision fields, and command flow. It intentionally does not expose the answer key.",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- cases: `{summary['case_count']}`",
|
||||
f"- ready for adjudication: `{summary['ready_for_adjudication_count']}`",
|
||||
f"- pending decisions: `{summary['pending_decision_count']}`",
|
||||
f"- invalid decisions: `{summary['invalid_decision_count']}`",
|
||||
f"- reviewer metadata present: `{str(summary['reviewer_metadata_present']).lower()}`",
|
||||
f"- answer key hidden: `{str(summary['answer_key_hidden']).lower()}`",
|
||||
f"- answer key path exposed: `{str(summary['answer_key_path_exposed']).lower()}`",
|
||||
"",
|
||||
"## Review Flow",
|
||||
"",
|
||||
]
|
||||
lines.extend(f"{index}. {step}" for index, step in enumerate(contract["reviewer_steps"], start=1))
|
||||
lines.extend(["", "## Required Fields", ""])
|
||||
lines.extend(f"- `{key}`: {value}" for key, value in contract["required_fields"].items())
|
||||
lines.extend(["", "## Privacy Contract", ""])
|
||||
lines.extend(f"- {item}" for item in contract["privacy_contract"])
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Decision States",
|
||||
"",
|
||||
"| Case | State | Winner | Confidence | Reason | Blocking Reason |",
|
||||
"| --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for case in payload["cases"]:
|
||||
state = case["decision_state"]
|
||||
lines.append(
|
||||
f"| `{case['case_id']}` | `{state['status']}` | `{str(state['winner_variant_recorded']).lower()}` | "
|
||||
f"`{str(state['confidence_recorded']).lower()}` | `{str(state['reason_recorded']).lower()}` | {state['blocking_reason']} |"
|
||||
)
|
||||
lines.extend(["", "## Blind Cases", ""])
|
||||
for case in payload["cases"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {case['case_id']}",
|
||||
"",
|
||||
f"Prompt: {case['prompt']}",
|
||||
"",
|
||||
"Rubric:",
|
||||
*render_rubric(case["rubric"]),
|
||||
"",
|
||||
"#### Variant A",
|
||||
"",
|
||||
case["variant_a"]["output"] or "_No output found._",
|
||||
"",
|
||||
"#### Variant B",
|
||||
"",
|
||||
case["variant_b"]["output"] or "_No output found._",
|
||||
"",
|
||||
]
|
||||
)
|
||||
if payload.get("failures"):
|
||||
lines.extend(["## Failures", ""])
|
||||
lines.extend(f"- {failure}" for failure in payload["failures"])
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def status_label(status: str) -> str:
|
||||
return {
|
||||
"awaiting-decision": "Awaiting",
|
||||
"needs-fix": "Needs fix",
|
||||
"ready-for-adjudication": "Ready",
|
||||
}.get(status, status)
|
||||
|
||||
|
||||
def render_html_rubric(rubric: list[dict[str, Any]]) -> str:
|
||||
if not rubric:
|
||||
return "<li>No rubric items found.</li>"
|
||||
return "".join(
|
||||
"<li><span>{id}</span><p>{description}</p><small>{weight}</small></li>".format(
|
||||
id=html_text(item.get("id", "")),
|
||||
description=html_text(item.get("description", "")),
|
||||
weight=html_text(item.get("weight", "")),
|
||||
)
|
||||
for item in rubric
|
||||
)
|
||||
|
||||
|
||||
def render_html_cases(cases: list[dict[str, Any]]) -> str:
|
||||
cards = []
|
||||
for index, case in enumerate(cases, start=1):
|
||||
state = case.get("decision_state", {})
|
||||
status = str(state.get("status", "awaiting-decision"))
|
||||
cards.append(
|
||||
f"""
|
||||
<article class="case-card" id="case-{html_text(case.get('case_id', ''))}">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case {index:02d}</span>
|
||||
<h3>{html_text(case.get('case_id', ''))}</h3>
|
||||
</div>
|
||||
<span class="status-pill {html_text(status)}">{html_text(status_label(status))}</span>
|
||||
</header>
|
||||
<p class="prompt">{html_text(case.get('prompt', ''))}</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul>{render_html_rubric(case.get('rubric', []))}</ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>{html_text(case.get('variant_a', {}).get('output', ''))}</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>{html_text(case.get('variant_b', {}).get('output', ''))}</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: {html_text(str(state.get('winner_variant_recorded', False)).lower())}</span>
|
||||
<span>Confidence: {html_text(str(state.get('confidence_recorded', False)).lower())}</span>
|
||||
<span>Reason: {html_text(str(state.get('reason_recorded', False)).lower())}</span>
|
||||
<strong>{html_text(state.get('blocking_reason', ''))}</strong>
|
||||
</footer>
|
||||
</article>"""
|
||||
)
|
||||
return "\n".join(cards)
|
||||
|
||||
|
||||
def decision_template_json(cases: list[dict[str, Any]]) -> str:
|
||||
template = {
|
||||
"schema_version": "1.0",
|
||||
"reviewer": "",
|
||||
"reviewed_at": "",
|
||||
"decisions": [
|
||||
{"case_id": case.get("case_id", ""), "winner_variant": "", "confidence": None, "reason": ""}
|
||||
for case in cases
|
||||
],
|
||||
}
|
||||
return json.dumps(template, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def render_html(payload: dict[str, Any]) -> str:
|
||||
summary = payload["summary"]
|
||||
contract = payload["review_contract"]
|
||||
stats = [
|
||||
("Cases", summary["case_count"]),
|
||||
("Ready", summary["ready_for_adjudication_count"]),
|
||||
("Pending", summary["pending_decision_count"]),
|
||||
("Invalid", summary["invalid_decision_count"]),
|
||||
]
|
||||
stat_html = "".join(f"<div><span>{html_text(label)}</span><strong>{html_text(value)}</strong></div>" for label, value in stats)
|
||||
flow_html = "".join(f"<li>{html_text(step)}</li>" for step in contract["reviewer_steps"])
|
||||
privacy_html = "".join(f"<li>{html_text(item)}</li>" for item in contract["privacy_contract"])
|
||||
decision_json = html_text(decision_template_json(payload["cases"]))
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Output Review Kit</title>
|
||||
<style>
|
||||
:root {{ --ink:#1B365D; --text:#202124; --muted:#6f6a63; --line:#e8e1d8; --soft:#f8f6f2; --warn:#9b4d0f; --ok:#1f6f43; }}
|
||||
* {{ box-sizing:border-box; }}
|
||||
body {{ margin:0; background:#fff; color:var(--text); font:16px/1.55 Georgia, "Times New Roman", serif; }}
|
||||
.shell {{ max-width:1180px; margin:0 auto; padding:36px 24px 72px; }}
|
||||
.topbar {{ position:sticky; top:0; z-index:10; background:rgba(255,255,255,.96); border-bottom:1px solid var(--line); backdrop-filter:blur(8px); }}
|
||||
.topbar-inner {{ max-width:1180px; margin:0 auto; padding:12px 24px; display:flex; justify-content:space-between; gap:18px; align-items:center; }}
|
||||
.brand {{ color:var(--ink); font-weight:700; letter-spacing:0; }}
|
||||
.links {{ display:flex; gap:12px; flex-wrap:wrap; }}
|
||||
.links a {{ color:var(--ink); text-decoration:none; border-bottom:1px solid transparent; }}
|
||||
.links a:hover {{ border-color:var(--ink); }}
|
||||
.hero {{ border-bottom:1px solid var(--line); padding:34px 0 28px; }}
|
||||
.eyebrow {{ color:var(--ink); text-transform:uppercase; font-size:12px; letter-spacing:0; font-weight:700; }}
|
||||
h1 {{ margin:8px 0 12px; color:var(--ink); font-size:58px; line-height:1.02; letter-spacing:0; }}
|
||||
.lede {{ max-width:760px; color:var(--muted); font-size:20px; }}
|
||||
.stats {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin:28px 0; }}
|
||||
.stats div, .panel, .case-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }}
|
||||
.stats div {{ padding:16px; }}
|
||||
.stats span {{ display:block; color:var(--muted); font-size:13px; }}
|
||||
.stats strong {{ color:var(--ink); font-size:34px; line-height:1; }}
|
||||
.grid {{ display:grid; grid-template-columns:minmax(0,1fr) minmax(280px,.42fr); gap:18px; align-items:start; }}
|
||||
.panel {{ padding:20px; }}
|
||||
h2, h3, h4 {{ color:var(--ink); letter-spacing:0; }}
|
||||
h2 {{ margin:0 0 14px; font-size:28px; }}
|
||||
h3 {{ margin:0; font-size:24px; }}
|
||||
h4 {{ margin:0 0 10px; font-size:16px; }}
|
||||
ol, ul {{ padding-left:22px; }}
|
||||
code, pre {{ font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }}
|
||||
pre {{ white-space:pre-wrap; overflow-wrap:anywhere; margin:0; color:#2b2b2b; font-size:13px; line-height:1.5; }}
|
||||
.case-card {{ margin:22px 0; padding:22px; scroll-margin-top:72px; }}
|
||||
.case-head, .case-foot {{ display:flex; gap:14px; justify-content:space-between; align-items:flex-start; flex-wrap:wrap; }}
|
||||
.case-index {{ color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:0; }}
|
||||
.status-pill {{ border:1px solid var(--line); border-radius:999px; padding:4px 10px; color:var(--ink); background:var(--soft); font-size:13px; }}
|
||||
.status-pill.ready-for-adjudication {{ color:var(--ok); }}
|
||||
.status-pill.needs-fix {{ color:var(--warn); }}
|
||||
.prompt {{ color:var(--muted); font-size:18px; }}
|
||||
.rubric {{ margin:18px 0; padding:16px; background:var(--soft); border-radius:8px; }}
|
||||
.rubric ul {{ list-style:none; padding:0; margin:0; display:grid; gap:8px; }}
|
||||
.rubric li {{ display:grid; grid-template-columns:160px 1fr 44px; gap:12px; align-items:start; border-top:1px solid var(--line); padding-top:8px; }}
|
||||
.rubric li:first-child {{ border-top:0; padding-top:0; }}
|
||||
.rubric span {{ color:var(--ink); font-weight:700; overflow-wrap:anywhere; }}
|
||||
.rubric p {{ margin:0; }}
|
||||
.rubric small {{ text-align:right; color:var(--muted); }}
|
||||
.variants {{ display:grid; grid-template-columns:1fr 1fr; gap:16px; }}
|
||||
.variant {{ border:1px solid var(--line); border-radius:8px; padding:16px; min-width:0; }}
|
||||
.case-foot {{ margin-top:16px; padding-top:14px; border-top:1px solid var(--line); color:var(--muted); font-size:13px; }}
|
||||
.case-foot strong {{ flex-basis:100%; color:var(--text); font-weight:400; }}
|
||||
.template {{ background:#101820; color:#f7f2e8; border-radius:8px; padding:16px; }}
|
||||
@media (max-width:820px) {{ .stats, .grid, .variants {{ grid-template-columns:1fr; }} .rubric li {{ grid-template-columns:1fr; }} .rubric small {{ text-align:left; }} h1 {{ font-size:38px; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="topbar"><div class="topbar-inner"><span class="brand">Output Review Kit</span><div class="links"><a href="#flow">Flow</a><a href="#cases">Cases</a><a href="#template">Template</a></div></div></nav>
|
||||
<main class="shell">
|
||||
<section class="hero">
|
||||
<span class="eyebrow">Blind A/B Human Review</span>
|
||||
<h1>Reviewer cockpit for output quality decisions</h1>
|
||||
<p class="lede">Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.</p>
|
||||
<div class="stats">{stat_html}</div>
|
||||
</section>
|
||||
<section class="grid" id="flow">
|
||||
<article class="panel"><h2>Review Flow</h2><ol>{flow_html}</ol></article>
|
||||
<aside class="panel"><h2>Privacy</h2><ul>{privacy_html}</ul></aside>
|
||||
</section>
|
||||
<section id="cases">{render_html_cases(payload["cases"])}</section>
|
||||
<section class="panel" id="template"><h2>Decision Template</h2><p>Use this shape in {html_text(payload['artifacts']['decisions'])}; leave a case blank when the reviewer is not ready.</p><pre class="template">{decision_json}</pre></section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def prepare_output_review_kit(
|
||||
blind_pack_json: Path,
|
||||
blind_pack_md: Path,
|
||||
decisions_path: Path,
|
||||
output_json: Path,
|
||||
output_md: Path,
|
||||
output_html: Path | None = None,
|
||||
write_template: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
blind_pack, failures = load_json(blind_pack_json)
|
||||
decisions_payload, decision_failures = load_optional_decisions(decisions_path)
|
||||
failures.extend(decision_failures)
|
||||
template_written = False
|
||||
if write_template and blind_pack and not decisions_path.exists():
|
||||
decisions_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
decisions_path.write_text(
|
||||
json.dumps(build_decision_template(blind_pack), ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
template_written = True
|
||||
decisions_payload, decision_failures = load_optional_decisions(decisions_path)
|
||||
failures.extend(decision_failures)
|
||||
decisions, index_failures = decision_index(decisions_payload)
|
||||
failures.extend(index_failures)
|
||||
case_ids = {str(pair.get("case_id", "")) for pair in pairs_from_pack(blind_pack)}
|
||||
for case_id in decisions:
|
||||
if case_id not in case_ids:
|
||||
failures.append(f"decision references unknown case_id: {case_id}")
|
||||
cases = build_cases(blind_pack, decisions)
|
||||
summary = build_summary(cases, failures, str(decisions_payload.get("reviewer", "")), str(decisions_payload.get("reviewed_at", "")))
|
||||
payload = {
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
"summary": summary,
|
||||
"artifacts": {
|
||||
"reviewer_kit_json": display_path(output_json),
|
||||
"reviewer_kit_markdown": display_path(output_md),
|
||||
"reviewer_kit_html": display_path(output_html) if output_html else "",
|
||||
"blind_pack_json": display_path(blind_pack_json),
|
||||
"blind_pack_markdown": display_path(blind_pack_md),
|
||||
"decisions": display_path(decisions_path),
|
||||
"adjudication_json": "reports/output_review_adjudication.json",
|
||||
"adjudication_markdown": "reports/output_review_adjudication.md",
|
||||
},
|
||||
"template_written": template_written,
|
||||
"review_contract": build_contract(blind_pack_md, decisions_path),
|
||||
"cases": cases,
|
||||
"failures": failures,
|
||||
}
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(payload), encoding="utf-8")
|
||||
if output_html:
|
||||
output_html.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_html.write_text(render_html(payload), encoding="utf-8")
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Prepare a reviewer-facing blind A/B output review kit.")
|
||||
parser.add_argument("--blind-pack-json", default=str(DEFAULT_BLIND_PACK_JSON))
|
||||
parser.add_argument("--blind-pack-md", default=str(DEFAULT_BLIND_PACK_MD))
|
||||
parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS))
|
||||
parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON))
|
||||
parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD))
|
||||
parser.add_argument("--output-html", default=str(DEFAULT_OUTPUT_HTML))
|
||||
parser.add_argument("--write-template", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = prepare_output_review_kit(
|
||||
Path(args.blind_pack_json).resolve(),
|
||||
Path(args.blind_pack_md).resolve(),
|
||||
Path(args.decisions).resolve(),
|
||||
Path(args.output_json).resolve(),
|
||||
Path(args.output_md).resolve(),
|
||||
Path(args.output_html).resolve() if args.output_html else None,
|
||||
write_template=args.write_template,
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if payload["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,560 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shlex
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_intake import build_intake
|
||||
from world_class_evidence_contract import DISALLOWED_REAL_ARTIFACTS
|
||||
from world_class_phase_queue import build_phase_queue, summarize_phase_queue
|
||||
from world_class_repair_checklist import build_repair_checklist, summarize_repair_checklist
|
||||
from world_class_source_checks import build_source_checklist, summarize_source_checklist
|
||||
from world_class_submission_matrix import build_evidence_matrix, summarize_evidence_matrix
|
||||
from world_class_submission_kit_rendering import render_html, render_readme
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Prepares editable world-class evidence intake packets without counting drafts as accepted evidence."
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def shell_path(path: Path, root: Path) -> str:
|
||||
return shlex.quote(rel_path(path, root))
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def has_glob_pattern(value: str) -> bool:
|
||||
return any(token in value for token in ("*", "?", "["))
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def requested_checklist_items(intake: dict[str, Any], evidence_keys: list[str]) -> list[dict[str, Any]]:
|
||||
items = intake.get("operator_checklist", [])
|
||||
if not evidence_keys:
|
||||
return items
|
||||
requested = set(evidence_keys)
|
||||
return [item for item in items if item.get("evidence_key") in requested]
|
||||
|
||||
|
||||
def template_result_by_key(intake: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
return {str(item.get("evidence_key")): item for item in intake.get("templates", [])}
|
||||
|
||||
|
||||
def artifact_rows_by_key_and_path(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, Any]]:
|
||||
ready: dict[tuple[str, str], dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
key = str(row.get("evidence_key", ""))
|
||||
path = str(row.get("path", ""))
|
||||
if key and path and row.get("artifact_ref_ready"):
|
||||
ready[(key, path)] = row
|
||||
return ready
|
||||
|
||||
|
||||
def template_artifact_ref_paths(skill_dir: Path, item: dict[str, Any]) -> set[str]:
|
||||
template_path = skill_dir / str(item.get("template_path", ""))
|
||||
errors: list[str] = []
|
||||
payload = load_template_payload(template_path, errors)
|
||||
if payload is None:
|
||||
return set()
|
||||
refs = payload.get("artifact_refs", [])
|
||||
if not isinstance(refs, list):
|
||||
return set()
|
||||
paths: set[str] = set()
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict):
|
||||
continue
|
||||
path = str(ref.get("path", "")).strip()
|
||||
if path:
|
||||
paths.add(path)
|
||||
return paths
|
||||
|
||||
|
||||
def load_template_payload(path: Path, errors: list[str]) -> dict[str, Any] | None:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
errors.append("template file is not valid JSON")
|
||||
return None
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def prefill_artifact_refs(
|
||||
payload: dict[str, Any],
|
||||
evidence_key: str,
|
||||
ready_rows: dict[tuple[str, str], dict[str, Any]],
|
||||
) -> dict[str, int]:
|
||||
refs = payload.get("artifact_refs", [])
|
||||
stats = {
|
||||
"prefilled_artifact_ref_count": 0,
|
||||
"unfilled_artifact_ref_count": 0,
|
||||
}
|
||||
if not isinstance(refs, list):
|
||||
return stats
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict):
|
||||
stats["unfilled_artifact_ref_count"] += 1
|
||||
continue
|
||||
path = str(ref.get("path", "")).strip()
|
||||
row = ready_rows.get((evidence_key, path))
|
||||
if row and row.get("sha256"):
|
||||
ref["sha256"] = row["sha256"]
|
||||
ref["contains_raw_content"] = False
|
||||
stats["prefilled_artifact_ref_count"] += 1
|
||||
else:
|
||||
stats["unfilled_artifact_ref_count"] += 1
|
||||
return stats
|
||||
|
||||
|
||||
def copy_template(
|
||||
skill_dir: Path,
|
||||
output_dir: Path,
|
||||
item: dict[str, Any],
|
||||
template_results: dict[str, dict[str, Any]],
|
||||
ready_rows: dict[tuple[str, str], dict[str, Any]],
|
||||
overwrite: bool,
|
||||
prefill_artifacts: bool,
|
||||
) -> dict[str, Any]:
|
||||
key = str(item.get("evidence_key", ""))
|
||||
template_result = template_results.get(key, {})
|
||||
template_path = skill_dir / str(item.get("template_path", ""))
|
||||
output_path = output_dir / f"{key}.json"
|
||||
errors: list[str] = []
|
||||
|
||||
if template_result.get("status") != "pass":
|
||||
errors.append("template failed intake validation")
|
||||
if not template_path.exists():
|
||||
errors.append("template file is missing")
|
||||
|
||||
if errors:
|
||||
return {
|
||||
"evidence_key": key,
|
||||
"status": "skipped",
|
||||
"template_path": rel_path(template_path, skill_dir),
|
||||
"output_path": rel_path(output_path, skill_dir),
|
||||
"prefilled_artifact_ref_count": 0,
|
||||
"unfilled_artifact_ref_count": 0,
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
if output_path.exists() and not overwrite:
|
||||
return {
|
||||
"evidence_key": key,
|
||||
"status": "exists",
|
||||
"template_path": rel_path(template_path, skill_dir),
|
||||
"output_path": rel_path(output_path, skill_dir),
|
||||
"prefilled_artifact_ref_count": 0,
|
||||
"unfilled_artifact_ref_count": 0,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
payload = load_template_payload(template_path, errors)
|
||||
if errors or payload is None:
|
||||
return {
|
||||
"evidence_key": key,
|
||||
"status": "skipped",
|
||||
"template_path": rel_path(template_path, skill_dir),
|
||||
"output_path": rel_path(output_path, skill_dir),
|
||||
"prefilled_artifact_ref_count": 0,
|
||||
"unfilled_artifact_ref_count": 0,
|
||||
"errors": errors or ["template file is not a JSON object"],
|
||||
}
|
||||
|
||||
prefill_stats = (
|
||||
prefill_artifact_refs(payload, key, ready_rows)
|
||||
if prefill_artifacts
|
||||
else {"prefilled_artifact_ref_count": 0, "unfilled_artifact_ref_count": 0}
|
||||
)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json(output_path, payload)
|
||||
return {
|
||||
"evidence_key": key,
|
||||
"status": "written",
|
||||
"template_path": rel_path(template_path, skill_dir),
|
||||
"output_path": rel_path(output_path, skill_dir),
|
||||
**prefill_stats,
|
||||
"errors": [],
|
||||
}
|
||||
|
||||
|
||||
def artifact_row(skill_dir: Path, evidence_key: str, pattern: str, path: Path, status: str) -> dict[str, Any]:
|
||||
exists = path.exists()
|
||||
is_file = exists and path.is_file()
|
||||
relative = rel_path(path, skill_dir) if exists else pattern
|
||||
contains_raw_content = relative in DISALLOWED_REAL_ARTIFACTS
|
||||
digest = sha256_file(path) if is_file and not contains_raw_content else ""
|
||||
row_status = status if status else ("ready" if is_file else "missing")
|
||||
if contains_raw_content:
|
||||
row_status = "raw-content-disallowed"
|
||||
return {
|
||||
"evidence_key": evidence_key,
|
||||
"source_pattern": pattern,
|
||||
"path": relative,
|
||||
"status": row_status,
|
||||
"exists": exists,
|
||||
"is_file": is_file,
|
||||
"sha256": digest,
|
||||
"artifact_ref_ready": bool(is_file and digest and not contains_raw_content),
|
||||
"copy_path": relative if is_file else "",
|
||||
"copy_sha256": digest,
|
||||
"contains_raw_content": contains_raw_content,
|
||||
"concrete_reference_required": has_glob_pattern(pattern),
|
||||
}
|
||||
|
||||
|
||||
def with_artifact_role(row: dict[str, Any], submission_ref_paths: set[str]) -> dict[str, Any]:
|
||||
is_submission_ref = str(row.get("path", "")) in submission_ref_paths
|
||||
return {
|
||||
**row,
|
||||
"artifact_role": "submission-ref" if is_submission_ref else "supporting-evidence",
|
||||
"submission_ref_required": is_submission_ref,
|
||||
}
|
||||
|
||||
|
||||
def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
key = str(item.get("evidence_key", ""))
|
||||
must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {}
|
||||
artifacts = must_collect.get("evidence_artifacts", [])
|
||||
if not isinstance(artifacts, list):
|
||||
return []
|
||||
submission_ref_paths = template_artifact_ref_paths(skill_dir, item)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for artifact in artifacts:
|
||||
pattern = str(artifact or "").strip()
|
||||
if not pattern:
|
||||
continue
|
||||
if Path(pattern).is_absolute() or ".." in Path(pattern).parts:
|
||||
rows.append(
|
||||
with_artifact_role(
|
||||
{
|
||||
"evidence_key": key,
|
||||
"source_pattern": pattern,
|
||||
"path": pattern,
|
||||
"status": "unsafe-path",
|
||||
"exists": False,
|
||||
"is_file": False,
|
||||
"sha256": "",
|
||||
"artifact_ref_ready": False,
|
||||
"copy_path": "",
|
||||
"copy_sha256": "",
|
||||
"contains_raw_content": False,
|
||||
"concrete_reference_required": True,
|
||||
},
|
||||
submission_ref_paths,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if has_glob_pattern(pattern):
|
||||
matches = sorted(path for path in skill_dir.glob(pattern) if path.is_file())
|
||||
if not matches:
|
||||
rows.append(
|
||||
with_artifact_role(
|
||||
{
|
||||
"evidence_key": key,
|
||||
"source_pattern": pattern,
|
||||
"path": pattern,
|
||||
"status": "glob-no-match",
|
||||
"exists": False,
|
||||
"is_file": False,
|
||||
"sha256": "",
|
||||
"artifact_ref_ready": False,
|
||||
"copy_path": "",
|
||||
"copy_sha256": "",
|
||||
"contains_raw_content": False,
|
||||
"concrete_reference_required": True,
|
||||
},
|
||||
submission_ref_paths,
|
||||
)
|
||||
)
|
||||
continue
|
||||
for match in matches:
|
||||
rows.append(with_artifact_role(artifact_row(skill_dir, key, pattern, match, "ready"), submission_ref_paths))
|
||||
continue
|
||||
rows.append(with_artifact_role(artifact_row(skill_dir, key, pattern, skill_dir / pattern, ""), submission_ref_paths))
|
||||
return rows
|
||||
|
||||
|
||||
def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
rows.extend(artifact_checklist_for_item(skill_dir, item))
|
||||
return rows
|
||||
|
||||
|
||||
def build_handoff_steps(
|
||||
commands: dict[str, str],
|
||||
*,
|
||||
files: list[dict[str, Any]],
|
||||
source_blocked_count: int,
|
||||
invalid_draft_count: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
drafts_ready = bool(files) and invalid_draft_count == 0
|
||||
return [
|
||||
{
|
||||
"step_id": "prepare-drafts",
|
||||
"label": "Prepare editable drafts",
|
||||
"status": "ready" if drafts_ready else "fix-required",
|
||||
"command": commands["prepare_submission"],
|
||||
"completion_signal": "JSON drafts, submission_manifest.json, README.md, and index.html are present.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": "One or more drafts were skipped or failed template validation." if invalid_draft_count else "",
|
||||
},
|
||||
{
|
||||
"step_id": "collect-source",
|
||||
"label": "Collect real external or human evidence",
|
||||
"status": "blocked" if source_blocked_count else "ready",
|
||||
"command": "",
|
||||
"completion_signal": "Source aggregate reports satisfy the required provider, human, native, or telemetry checks.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": (
|
||||
f"{source_blocked_count} source check(s) still block ledger review."
|
||||
if source_blocked_count
|
||||
else ""
|
||||
),
|
||||
},
|
||||
{
|
||||
"step_id": "edit-submission",
|
||||
"label": "Edit submission packet",
|
||||
"status": "manual",
|
||||
"command": "",
|
||||
"completion_signal": "template_only is false only after real evidence exists; ledger_reviewer_approved, ledger_reviewer, and ledger_reviewed_at stay unset until reviewer approval.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": "Raw prompts, outputs, transcripts, credentials, or private content must not be included.",
|
||||
},
|
||||
{
|
||||
"step_id": "validate-intake",
|
||||
"label": "Validate intake contract",
|
||||
"status": "pending",
|
||||
"command": commands["validate_intake"],
|
||||
"completion_signal": "world_class_evidence_intake reports valid submissions and no invalid packets before reviewer approval.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": "A valid packet is ready for ledger review but is not accepted evidence by itself.",
|
||||
},
|
||||
{
|
||||
"step_id": "review-submission",
|
||||
"label": "Review submission queue",
|
||||
"status": "pending",
|
||||
"command": commands["review_submission"],
|
||||
"completion_signal": "world_class_submission_review shows ready-for-ledger-review before reviewer identity, timestamp, and approval are set.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": "Reviewer queue output is advisory and cannot accept evidence.",
|
||||
},
|
||||
{
|
||||
"step_id": "refresh-ledger",
|
||||
"label": "Refresh evidence ledger",
|
||||
"status": "pending",
|
||||
"command": commands["refresh_ledger"],
|
||||
"completion_signal": "world_class_evidence_ledger accepts the evidence entry with valid source checks.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": "Ledger remains the only world-class readiness source of truth.",
|
||||
},
|
||||
{
|
||||
"step_id": "guard-claim",
|
||||
"label": "Guard public claim",
|
||||
"status": "pending",
|
||||
"command": commands["guard_claim"],
|
||||
"completion_signal": "world_class_claim_guard allows the public readiness claim.",
|
||||
"counts_as_completion": False,
|
||||
"blocking_condition": "Public world-class claims stay blocked until every ledger entry is accepted.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def build_submission_kit(
|
||||
skill_dir: Path,
|
||||
output_dir: Path,
|
||||
generated_at: str,
|
||||
evidence_keys: list[str] | None = None,
|
||||
overwrite: bool = False,
|
||||
prefill_artifacts: bool = False,
|
||||
output_html: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
intake = build_intake(skill_dir, generated_at, submissions_dir=output_dir)
|
||||
items = requested_checklist_items(intake, evidence_keys or [])
|
||||
valid_keys = {str(item.get("evidence_key")) for item in intake.get("operator_checklist", [])}
|
||||
unknown_keys = sorted(set(evidence_keys or []) - valid_keys)
|
||||
template_results = template_result_by_key(intake)
|
||||
artifact_checklist = build_artifact_checklist(skill_dir, items)
|
||||
ready_rows = artifact_rows_by_key_and_path(artifact_checklist)
|
||||
files = [
|
||||
copy_template(
|
||||
skill_dir,
|
||||
output_dir,
|
||||
item,
|
||||
template_results,
|
||||
ready_rows,
|
||||
overwrite,
|
||||
prefill_artifacts,
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
source_checklist = build_source_checklist(items)
|
||||
evidence_matrix = build_evidence_matrix(items, files, artifact_checklist, source_checklist)
|
||||
repair_checklist = build_repair_checklist(items, files, artifact_checklist, source_checklist, unknown_keys)
|
||||
phase_queue = build_phase_queue(repair_checklist)
|
||||
manifest_path = output_dir / "submission_manifest.json"
|
||||
readme_path = output_dir / "README.md"
|
||||
output_html = output_html or (output_dir / "index.html")
|
||||
written_count = sum(1 for item in files if item["status"] == "written")
|
||||
existing_count = sum(1 for item in files if item["status"] == "exists")
|
||||
skipped_count = sum(1 for item in files if item["status"] == "skipped")
|
||||
prefilled_artifact_ref_count = sum(item.get("prefilled_artifact_ref_count", 0) for item in files)
|
||||
unfilled_artifact_ref_count = sum(item.get("unfilled_artifact_ref_count", 0) for item in files)
|
||||
artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready"))
|
||||
artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready"))
|
||||
artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required"))
|
||||
submission_ref_rows = [item for item in artifact_checklist if item.get("submission_ref_required")]
|
||||
supporting_rows = [item for item in artifact_checklist if not item.get("submission_ref_required")]
|
||||
submission_ref_ready_count = sum(1 for item in submission_ref_rows if item.get("artifact_ref_ready"))
|
||||
supporting_artifact_ready_count = sum(1 for item in supporting_rows if item.get("artifact_ref_ready"))
|
||||
source_summary = summarize_source_checklist(source_checklist)
|
||||
matrix_summary = summarize_evidence_matrix(evidence_matrix)
|
||||
repair_summary = summarize_repair_checklist(repair_checklist)
|
||||
phase_queue_summary = summarize_phase_queue(phase_queue)
|
||||
ok = not unknown_keys and skipped_count == 0
|
||||
output_dir_arg = shell_path(output_dir, skill_dir)
|
||||
requested_key_args = " ".join(f"--evidence-key {shlex.quote(key)}" for key in (evidence_keys or []))
|
||||
prepare_command = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir_arg}"
|
||||
if requested_key_args:
|
||||
prepare_command = f"{prepare_command} {requested_key_args}"
|
||||
if prefill_artifacts:
|
||||
prepare_command = f"{prepare_command} --prefill-artifacts"
|
||||
commands = {
|
||||
"prepare_submission": prepare_command,
|
||||
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir_arg}",
|
||||
"review_submission": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir_arg}",
|
||||
"refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir_arg}",
|
||||
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
}
|
||||
handoff_steps = build_handoff_steps(
|
||||
commands,
|
||||
files=files,
|
||||
source_blocked_count=source_summary["source_blocked_count"],
|
||||
invalid_draft_count=skipped_count + len(unknown_keys),
|
||||
)
|
||||
report = {
|
||||
"schema_version": "1.0",
|
||||
"ok": ok,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"output_dir": rel_path(output_dir, skill_dir),
|
||||
"summary": {
|
||||
"requested_count": len(items) + len(unknown_keys),
|
||||
"prepared_count": len(files),
|
||||
"written_count": written_count,
|
||||
"existing_count": existing_count,
|
||||
"skipped_count": skipped_count,
|
||||
"unknown_key_count": len(unknown_keys),
|
||||
"artifact_checklist_count": len(artifact_checklist),
|
||||
"artifact_ready_count": artifact_ready_count,
|
||||
"artifact_missing_count": artifact_missing_count,
|
||||
"artifact_glob_expansion_count": artifact_glob_count,
|
||||
"submission_ref_count": len(submission_ref_rows),
|
||||
"submission_ref_ready_count": submission_ref_ready_count,
|
||||
"submission_ref_missing_count": len(submission_ref_rows) - submission_ref_ready_count,
|
||||
"supporting_artifact_count": len(supporting_rows),
|
||||
"supporting_artifact_ready_count": supporting_artifact_ready_count,
|
||||
"supporting_artifact_missing_count": len(supporting_rows) - supporting_artifact_ready_count,
|
||||
"artifact_prefill_enabled": prefill_artifacts,
|
||||
"artifact_ref_prefill_count": prefilled_artifact_ref_count,
|
||||
"artifact_ref_unfilled_count": unfilled_artifact_ref_count,
|
||||
**source_summary,
|
||||
**matrix_summary,
|
||||
**repair_summary,
|
||||
**phase_queue_summary,
|
||||
"handoff_step_count": len(handoff_steps),
|
||||
"handoff_blocked_count": sum(1 for item in handoff_steps if item["status"] == "blocked"),
|
||||
"handoff_fix_required_count": sum(1 for item in handoff_steps if item["status"] == "fix-required"),
|
||||
"handoff_counts_as_completion": False,
|
||||
"drafts_count_as_evidence": False,
|
||||
"ledger_counts_submission_as_completion": False,
|
||||
"decision": "submission-kit-ready" if ok else "fix-submission-kit",
|
||||
},
|
||||
"unknown_evidence_keys": unknown_keys,
|
||||
"files": files,
|
||||
"artifact_checklist": artifact_checklist,
|
||||
"source_checklist": source_checklist,
|
||||
"evidence_matrix": evidence_matrix,
|
||||
"repair_checklist": repair_checklist,
|
||||
"phase_queue": phase_queue,
|
||||
"operator_handoff": handoff_steps,
|
||||
"evidence_items": items,
|
||||
"commands": commands,
|
||||
"safety": {
|
||||
"template_only_drafts": True,
|
||||
"artifact_prefill_counts_as_evidence": False,
|
||||
"operator_handoff_counts_as_evidence": False,
|
||||
"real_evidence_required_before_template_only_false": True,
|
||||
"raw_content_allowed": False,
|
||||
"credentials_allowed": False,
|
||||
"drafts_count_as_evidence": False,
|
||||
},
|
||||
"artifacts": {
|
||||
"manifest": rel_path(manifest_path, skill_dir),
|
||||
"readme": rel_path(readme_path, skill_dir),
|
||||
"html": rel_path(output_html, skill_dir),
|
||||
},
|
||||
}
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
write_json(manifest_path, report)
|
||||
readme_path.write_text(render_readme(report), encoding="utf-8")
|
||||
output_html.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_html.write_text(render_html(report), encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Prepare editable world-class evidence submission drafts.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-dir", default="evidence/world_class/submission-kit")
|
||||
parser.add_argument("--evidence-key", action="append", default=[])
|
||||
parser.add_argument("--overwrite", action="store_true")
|
||||
parser.add_argument(
|
||||
"--prefill-artifacts",
|
||||
action="store_true",
|
||||
help="Insert SHA-256 digests for currently available aggregate artifacts while keeping drafts template-only.",
|
||||
)
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
parser.add_argument("--output-html")
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
output_dir = Path(args.output_dir)
|
||||
if not output_dir.is_absolute():
|
||||
output_dir = skill_dir / output_dir
|
||||
report = build_submission_kit(
|
||||
skill_dir,
|
||||
output_dir.resolve(),
|
||||
args.generated_at,
|
||||
evidence_keys=args.evidence_key,
|
||||
overwrite=args.overwrite,
|
||||
prefill_artifacts=args.prefill_artifacts,
|
||||
output_html=Path(args.output_html).resolve() if args.output_html else None,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -10,7 +10,7 @@ except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
|
||||
DEFAULT_TARGETS = ["openai", "claude", "generic"]
|
||||
DEFAULT_TARGETS = ["openai", "claude", "generic", "vscode"]
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path) -> str:
|
||||
@@ -69,6 +69,103 @@ def sorted_strings(value: Any) -> list[str]:
|
||||
return sorted(str(item) for item in value) if isinstance(value, list) else []
|
||||
|
||||
|
||||
def check_index(checks: list[Any]) -> dict[str, str]:
|
||||
indexed: dict[str, str] = {}
|
||||
for item in checks:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
check_id = str(item.get("id") or item.get("key") or "").strip()
|
||||
status = str(item.get("status") or ("pass" if item.get("passed") is True else "")).strip()
|
||||
if check_id:
|
||||
indexed[check_id] = status
|
||||
return indexed
|
||||
|
||||
|
||||
def installer_enforcement_source(
|
||||
skill_dir: Path,
|
||||
package_dir: Path,
|
||||
targets: list[str],
|
||||
expected: list[str],
|
||||
install_simulation_path: Path | None,
|
||||
) -> dict[str, Any]:
|
||||
path = install_simulation_path or skill_dir / "reports" / "install_simulation.json"
|
||||
report = load_json(path)
|
||||
source_display = display_path(path, skill_dir)
|
||||
if not path.exists() or not report:
|
||||
return {
|
||||
"source": source_display,
|
||||
"source_status": "missing",
|
||||
"package_dir_matches": False,
|
||||
"summary": {},
|
||||
"targets": {
|
||||
target: {
|
||||
"target": target,
|
||||
"source_status": "missing",
|
||||
"enforced": False,
|
||||
"enforced_capabilities": [],
|
||||
"missing_capabilities": expected,
|
||||
"failure_details": ["install simulation report is missing"],
|
||||
}
|
||||
for target in targets
|
||||
},
|
||||
}
|
||||
|
||||
expected_package_dir = display_path(package_dir, skill_dir)
|
||||
observed_package_dir = str(report.get("package_dir", "")).strip()
|
||||
package_matches = observed_package_dir in {expected_package_dir, str(package_dir)}
|
||||
summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {}
|
||||
if not package_matches:
|
||||
source_status = "package-mismatch"
|
||||
else:
|
||||
source_status = "present"
|
||||
|
||||
indexed = check_index(report.get("checks", []) if isinstance(report.get("checks"), list) else [])
|
||||
target_results: dict[str, Any] = {}
|
||||
for target in targets:
|
||||
enforced_capabilities = []
|
||||
missing_capabilities = []
|
||||
failure_details = []
|
||||
for capability in expected:
|
||||
approval_id = f"permission-{target}-{capability}-approved"
|
||||
enforcement_id = f"permission-{target}-{capability}-target-enforcement"
|
||||
approval_ok = indexed.get(approval_id) == "pass"
|
||||
enforcement_ok = indexed.get(enforcement_id) == "pass"
|
||||
if source_status == "present" and approval_ok and enforcement_ok:
|
||||
enforced_capabilities.append(capability)
|
||||
else:
|
||||
missing_capabilities.append(capability)
|
||||
if source_status == "present" and not approval_ok:
|
||||
failure_details.append(f"{approval_id} did not pass")
|
||||
if source_status == "present" and not enforcement_ok:
|
||||
failure_details.append(f"{enforcement_id} did not pass")
|
||||
if source_status == "package-mismatch":
|
||||
failure_details.append(
|
||||
f"install simulation package_dir {observed_package_dir or 'missing'} does not match probed package_dir {expected_package_dir}"
|
||||
)
|
||||
target_results[target] = {
|
||||
"target": target,
|
||||
"source_status": source_status,
|
||||
"enforced": bool(expected) and source_status == "present" and not missing_capabilities,
|
||||
"enforced_capabilities": enforced_capabilities,
|
||||
"missing_capabilities": missing_capabilities,
|
||||
"failure_details": failure_details,
|
||||
}
|
||||
|
||||
return {
|
||||
"source": source_display,
|
||||
"source_status": source_status,
|
||||
"package_dir_matches": package_matches,
|
||||
"summary": {
|
||||
"installer_permission_enforced_count": int(summary.get("installer_permission_enforced_count", 0) or 0),
|
||||
"installer_permission_failure_count": int(summary.get("installer_permission_failure_count", 0) or 0),
|
||||
"permission_target_count": int(summary.get("permission_target_count", 0) or 0),
|
||||
"permission_capability_count": int(summary.get("permission_capability_count", 0) or 0),
|
||||
"failure_count": int(summary.get("failure_count", 0) or 0),
|
||||
},
|
||||
"targets": target_results,
|
||||
}
|
||||
|
||||
|
||||
def probe_openai_yaml(package_dir: Path, expected: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
|
||||
checks: list[dict[str, Any]] = []
|
||||
failures: list[str] = []
|
||||
@@ -93,7 +190,13 @@ def probe_openai_yaml(package_dir: Path, expected: list[str]) -> tuple[list[dict
|
||||
return checks, failures
|
||||
|
||||
|
||||
def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list[str]) -> dict[str, Any]:
|
||||
def probe_target(
|
||||
skill_dir: Path,
|
||||
package_dir: Path,
|
||||
target: str,
|
||||
expected: list[str],
|
||||
installer_targets: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
adapter_path = package_dir / "targets" / target / "adapter.json"
|
||||
checks: list[dict[str, Any]] = []
|
||||
failures: list[str] = []
|
||||
@@ -167,6 +270,17 @@ def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list
|
||||
residual_risks = []
|
||||
if native is False:
|
||||
residual_risks.append("Client-native permission enforcement is not provided by this target; installer or operator must honor metadata.")
|
||||
installer_enforcement = installer_targets.get(
|
||||
target,
|
||||
{
|
||||
"target": target,
|
||||
"source_status": "missing",
|
||||
"enforced": False,
|
||||
"enforced_capabilities": [],
|
||||
"missing_capabilities": expected,
|
||||
"failure_details": ["install simulation report is missing"],
|
||||
},
|
||||
)
|
||||
return {
|
||||
"target": target,
|
||||
"status": "pass" if not failures else "fail",
|
||||
@@ -174,6 +288,7 @@ def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list
|
||||
"permission_model": str(target_contract.get("permission_model", "")),
|
||||
"native_enforcement": bool(native) if isinstance(native, bool) else None,
|
||||
"metadata_fallback_explicit": metadata_fallback,
|
||||
"installer_enforcement": installer_enforcement,
|
||||
"assurance": assurance,
|
||||
"declared_capabilities": sorted_strings(target_contract.get("declared_capabilities")),
|
||||
"checks": checks,
|
||||
@@ -184,10 +299,11 @@ def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
installer = report.get("installer_enforcement", {})
|
||||
lines = [
|
||||
"# Runtime Permission Probes",
|
||||
"",
|
||||
"Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit.",
|
||||
"Runtime permission probes verify that generated target adapters expose high-permission capabilities, make native-enforcement limits explicit, and link installer enforcement evidence when available.",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
@@ -197,17 +313,35 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
f"- Failed: `{summary['fail_count']}`",
|
||||
f"- Native enforcement targets: `{summary['native_enforcement_count']}`",
|
||||
f"- Explicit metadata fallbacks: `{summary['metadata_fallback_count']}`",
|
||||
f"- Installer enforcement source: `{summary['installer_enforcement_source_status']}`",
|
||||
f"- Installer-enforced targets: `{summary['installer_enforcement_pass_count']}`",
|
||||
f"- Installer permission failures: `{summary['installer_permission_failure_count']}`",
|
||||
f"- World-class native evidence ready: `{summary['world_class_native_evidence_ready']}`",
|
||||
f"- Required capabilities: `{', '.join(report['expected_capabilities']) or 'none'}`",
|
||||
"",
|
||||
"| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk |",
|
||||
"| --- | --- | --- | --- | --- | --- |",
|
||||
"| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Installer Enforcement | Residual Risk |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for target in report["targets"]:
|
||||
residual = "<br>".join(target["residual_risks"]) if target["residual_risks"] else "None"
|
||||
installer_status = target.get("installer_enforcement", {})
|
||||
installer_label = "pass" if installer_status.get("enforced") else installer_status.get("source_status", "missing")
|
||||
lines.append(
|
||||
f"| `{target['target']}` | `{target['status']}` | `{target['assurance']}` | "
|
||||
f"`{target['native_enforcement']}` | `{target['metadata_fallback_explicit']}` | {residual} |"
|
||||
f"`{target['native_enforcement']}` | `{target['metadata_fallback_explicit']}` | `{installer_label}` | {residual} |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Installer Enforcement",
|
||||
"",
|
||||
f"- Source: `{installer.get('source', '')}`",
|
||||
f"- Source status: `{installer.get('source_status', 'missing')}`",
|
||||
f"- Package dir matches probe: `{installer.get('package_dir_matches', False)}`",
|
||||
"",
|
||||
"Installer enforcement means the package install simulation blocks missing capability approvals or target enforcement notes. It is supporting local distribution evidence, not proof of target-client native enforcement.",
|
||||
]
|
||||
)
|
||||
lines.extend(["", "## Failures", ""])
|
||||
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
|
||||
lines.extend(
|
||||
@@ -227,12 +361,21 @@ def probe_runtime_permissions(
|
||||
targets: list[str],
|
||||
output_json: Path,
|
||||
output_md: Path,
|
||||
install_simulation_json: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
package_dir = package_dir.resolve()
|
||||
expected = expected_capabilities(skill_dir)
|
||||
target_results = [probe_target(skill_dir, package_dir, target, expected) for target in targets]
|
||||
installer = installer_enforcement_source(skill_dir, package_dir, targets, expected, install_simulation_json)
|
||||
installer_targets = installer.get("targets", {}) if isinstance(installer.get("targets"), dict) else {}
|
||||
target_results = [probe_target(skill_dir, package_dir, target, expected, installer_targets) for target in targets]
|
||||
failures = [failure for target in target_results for failure in target["failures"]]
|
||||
installer_summary = installer.get("summary", {}) if isinstance(installer.get("summary"), dict) else {}
|
||||
installer_pass_count = sum(
|
||||
1
|
||||
for item in installer_targets.values()
|
||||
if isinstance(item, dict) and item.get("enforced") is True
|
||||
)
|
||||
summary = {
|
||||
"target_count": len(target_results),
|
||||
"pass_count": sum(1 for item in target_results if item["status"] == "pass"),
|
||||
@@ -242,6 +385,20 @@ def probe_runtime_permissions(
|
||||
"residual_risk_count": sum(len(item["residual_risks"]) for item in target_results),
|
||||
"required_capability_count": len(expected),
|
||||
"failure_count": len(failures),
|
||||
"installer_enforcement_source_status": installer.get("source_status", "missing"),
|
||||
"installer_enforcement_target_count": len(installer_targets),
|
||||
"installer_enforcement_pass_count": installer_pass_count,
|
||||
"installer_permission_enforced_count": int(installer_summary.get("installer_permission_enforced_count", 0) or 0),
|
||||
"installer_permission_failure_count": int(installer_summary.get("installer_permission_failure_count", 0) or 0),
|
||||
"installer_permission_capability_count": int(installer_summary.get("permission_capability_count", 0) or 0),
|
||||
"world_class_native_evidence_ready": sum(1 for item in target_results if item["native_enforcement"] is True) > 0 and not failures,
|
||||
"installer_enforcement_ready": (
|
||||
installer.get("source_status") == "present"
|
||||
and installer_pass_count == len(target_results)
|
||||
and bool(expected)
|
||||
and int(installer_summary.get("failure_count", 0) or 0) == 0
|
||||
and int(installer_summary.get("installer_permission_failure_count", 0) or 0) == 0
|
||||
),
|
||||
}
|
||||
report = {
|
||||
"schema_version": "1.0",
|
||||
@@ -250,6 +407,12 @@ def probe_runtime_permissions(
|
||||
"package_dir": display_path(package_dir, skill_dir),
|
||||
"expected_capabilities": expected,
|
||||
"summary": summary,
|
||||
"installer_enforcement": {
|
||||
"source": installer.get("source", ""),
|
||||
"source_status": installer.get("source_status", "missing"),
|
||||
"package_dir_matches": installer.get("package_dir_matches", False),
|
||||
"summary": installer_summary,
|
||||
},
|
||||
"targets": target_results,
|
||||
"failures": failures,
|
||||
"artifacts": {
|
||||
@@ -271,6 +434,7 @@ def main() -> None:
|
||||
parser.add_argument("--target", action="append", choices=DEFAULT_TARGETS)
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--install-simulation-json")
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
@@ -280,6 +444,7 @@ def main() -> None:
|
||||
args.target or DEFAULT_TARGETS,
|
||||
Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "runtime_permission_probes.json",
|
||||
Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "runtime_permission_probes.md",
|
||||
Path(args.install_simulation_json).resolve() if args.install_simulation_json else None,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Turns redacted repeated preference patterns into proposal-only adaptation plans."
|
||||
|
||||
PROPOSAL_LIBRARY = {
|
||||
"language_default": {
|
||||
"title": "Keep reports Chinese-first with optional English",
|
||||
"change_type": "report-default-language",
|
||||
"risk_level": "low",
|
||||
"target_files": ["scripts/render_skill_overview.py", "references/artifact-design-doctrine.md"],
|
||||
"suggested_changes": [
|
||||
"Keep user-facing report copy Simplified Chinese by default.",
|
||||
"Expose English through the existing language switch instead of mixing languages in the default view.",
|
||||
],
|
||||
"verification_commands": ["python3 tests/verify_skill_overview.py"],
|
||||
"rollback_plan": "Revert report language template changes and rerun the overview verifier.",
|
||||
},
|
||||
"report_ui": {
|
||||
"title": "Improve report layout, visual hierarchy, and chart readability",
|
||||
"change_type": "artifact-ui-polish",
|
||||
"risk_level": "medium",
|
||||
"target_files": ["scripts/render_skill_overview.py", "references/artifact-design-doctrine.md", "tests/verify_skill_overview.py"],
|
||||
"suggested_changes": [
|
||||
"Prefer vertical narrative sections with limited two-column layouts only when content has enough width.",
|
||||
"Keep charts inline SVG, with captions and stable responsive constraints.",
|
||||
],
|
||||
"verification_commands": ["python3 tests/verify_skill_overview.py", "python3 tests/verify_skill_report_charts.py"],
|
||||
"rollback_plan": "Restore the previous report renderer and regenerate the demo report.",
|
||||
},
|
||||
"approval_safety": {
|
||||
"title": "Keep adaptive iteration approval-gated",
|
||||
"change_type": "privacy-governance",
|
||||
"risk_level": "low",
|
||||
"target_files": ["references/user-memory-policy.md", "references/autonomous-adaptation.md", "schemas/adaptation-proposal.schema.json"],
|
||||
"suggested_changes": [
|
||||
"Require explicit source paths for memory scans.",
|
||||
"Generate proposals before any source patching.",
|
||||
"Reserve automatic apply for a future approval ledger and rollback implementation.",
|
||||
],
|
||||
"verification_commands": ["python3 tests/verify_adaptation_safety.py"],
|
||||
"rollback_plan": "Remove the adaptive proposal artifacts and keep feedback/adoption drift as the only iteration inputs.",
|
||||
},
|
||||
"delivery_format": {
|
||||
"title": "Make generated artifact paths explicit in CLI output",
|
||||
"change_type": "artifact-discoverability",
|
||||
"risk_level": "low",
|
||||
"target_files": ["scripts/yao.py", "README.md"],
|
||||
"suggested_changes": [
|
||||
"Include stable report paths in command output.",
|
||||
"Document which artifacts are meant for human review.",
|
||||
],
|
||||
"verification_commands": ["python3 tests/verify_yao_cli.py"],
|
||||
"rollback_plan": "Revert CLI copy/documentation changes and keep artifact paths unchanged.",
|
||||
},
|
||||
"evidence_testing": {
|
||||
"title": "Attach tests and evidence refresh to each upgrade",
|
||||
"change_type": "quality-gate",
|
||||
"risk_level": "medium",
|
||||
"target_files": ["tests/verify_adaptation_safety.py", "scripts/render_skill_os2_coverage.py", "reports/skill_os2_coverage.json"],
|
||||
"suggested_changes": [
|
||||
"Add focused verifier coverage for every new adaptive behavior.",
|
||||
"Refresh Skill OS 2.0 coverage so planned, partial, and covered states remain visible.",
|
||||
],
|
||||
"verification_commands": ["python3 tests/verify_adaptation_safety.py", "python3 tests/verify_skill_os2_coverage.py"],
|
||||
"rollback_plan": "Revert the new verifier and coverage status updates, then regenerate coverage reports.",
|
||||
},
|
||||
}
|
||||
|
||||
TOP_LEVEL_SUMMARY_FIELDS = [
|
||||
"pattern_count",
|
||||
"proposal_count",
|
||||
"apply_supported",
|
||||
"failure_count",
|
||||
]
|
||||
TOP_LEVEL_PROPOSAL_CONTRACT_FIELDS = [
|
||||
"proposal_only",
|
||||
"approval_required",
|
||||
"writes_repository_files",
|
||||
"allowlisted_targets_required",
|
||||
"target_file_sha256_required_for_apply",
|
||||
"approval_draft_supported",
|
||||
"rollback_required_for_apply",
|
||||
"apply_command_available",
|
||||
]
|
||||
|
||||
|
||||
def top_level_mirrors(summary: dict[str, Any], proposal_contract: dict[str, Any]) -> dict[str, Any]:
|
||||
mirrored = {key: summary[key] for key in TOP_LEVEL_SUMMARY_FIELDS if key in summary}
|
||||
mirrored.update({key: proposal_contract[key] for key in TOP_LEVEL_PROPOSAL_CONTRACT_FIELDS if key in proposal_contract})
|
||||
return mirrored
|
||||
|
||||
|
||||
def report_contract() -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"contract": "adaptation-proposals",
|
||||
"top_level_mirrors_summary": True,
|
||||
"top_level_mirrors_proposal_contract": True,
|
||||
"summary_fields": TOP_LEVEL_SUMMARY_FIELDS,
|
||||
"proposal_contract_fields": TOP_LEVEL_PROPOSAL_CONTRACT_FIELDS,
|
||||
"source_of_truth": ["summary", "proposal_contract"],
|
||||
}
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def display_path(path: Path, skill_dir: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def resolve_output(skill_dir: Path, value: str) -> Path:
|
||||
path = Path(value)
|
||||
return path if path.is_absolute() else skill_dir / path
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def proposal_id(pattern_id: str, support_count: int, target_files: list[str]) -> str:
|
||||
digest = hashlib.sha1(f"{pattern_id}:{support_count}:{','.join(target_files)}".encode("utf-8")).hexdigest()
|
||||
return f"adapt-{digest[:10]}"
|
||||
|
||||
|
||||
def proposal_from_pattern(pattern: dict[str, Any]) -> dict[str, Any]:
|
||||
pattern_id = str(pattern.get("pattern_id") or "generic")
|
||||
spec = PROPOSAL_LIBRARY.get(
|
||||
pattern_id,
|
||||
{
|
||||
"title": "Review repeated preference pattern",
|
||||
"change_type": "manual-review",
|
||||
"risk_level": "medium",
|
||||
"target_files": ["README.md"],
|
||||
"suggested_changes": ["Review this repeated signal manually before changing skill behavior."],
|
||||
"verification_commands": ["make ci-test"],
|
||||
"rollback_plan": "Revert the approved patch and rerun the relevant verifier.",
|
||||
},
|
||||
)
|
||||
support_count = int(pattern.get("support_count") or 0)
|
||||
target_files = list(spec["target_files"])
|
||||
return {
|
||||
"proposal_id": proposal_id(pattern_id, support_count, target_files),
|
||||
"pattern_id": pattern_id,
|
||||
"title": spec["title"],
|
||||
"change_type": spec["change_type"],
|
||||
"status": "proposal-only",
|
||||
"requires_approval": True,
|
||||
"write_allowed_without_approval": False,
|
||||
"risk_level": spec["risk_level"],
|
||||
"reason": pattern.get("reason", "Repeated signal detected."),
|
||||
"support_count": support_count,
|
||||
"target_files": target_files,
|
||||
"suggested_changes": spec["suggested_changes"],
|
||||
"verification_commands": spec["verification_commands"],
|
||||
"rollback_plan": spec["rollback_plan"],
|
||||
"evidence_refs": [
|
||||
{
|
||||
"record_id": item.get("record_id", "unknown"),
|
||||
"excerpt": item.get("excerpt", ""),
|
||||
}
|
||||
for item in pattern.get("evidence", [])[:3]
|
||||
if isinstance(item, dict)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def build_report(skill_dir: Path, patterns_json: Path, generated_at: str) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
if not patterns_json.is_absolute():
|
||||
patterns_json = skill_dir / patterns_json
|
||||
patterns_payload = load_json(patterns_json)
|
||||
failures: list[str] = []
|
||||
if not patterns_payload:
|
||||
failures.append(f"Pattern report does not exist or is invalid: {display_path(patterns_json, skill_dir)}")
|
||||
elif patterns_payload.get("ok") is not True:
|
||||
failures.append("Pattern report is not ok; fix scan failures before proposal generation.")
|
||||
patterns = patterns_payload.get("patterns", []) if isinstance(patterns_payload.get("patterns"), list) else []
|
||||
proposals = [proposal_from_pattern(pattern) for pattern in patterns if isinstance(pattern, dict)] if not failures else []
|
||||
summary = {
|
||||
"pattern_count": len(patterns),
|
||||
"proposal_count": len(proposals),
|
||||
"apply_supported": (ROOT / "scripts" / "apply_adaptation.py").exists(),
|
||||
"failure_count": len(failures),
|
||||
}
|
||||
proposal_contract = {
|
||||
"proposal_only": True,
|
||||
"approval_required": True,
|
||||
"writes_repository_files": False,
|
||||
"allowlisted_targets_required": True,
|
||||
"target_file_sha256_required_for_apply": True,
|
||||
"approval_draft_supported": True,
|
||||
"rollback_required_for_apply": True,
|
||||
"apply_command_available": (ROOT / "scripts" / "apply_adaptation.py").exists(),
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
"source_patterns": display_path(patterns_json, skill_dir),
|
||||
**top_level_mirrors(summary, proposal_contract),
|
||||
"summary": summary,
|
||||
"proposal_contract": proposal_contract,
|
||||
"report_contract": report_contract(),
|
||||
"proposals": proposals,
|
||||
"failures": failures,
|
||||
"artifacts": {
|
||||
"json": "reports/adaptation_proposals.json",
|
||||
"markdown": "reports/adaptation_proposals.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# Adaptation Proposals",
|
||||
"",
|
||||
f"- Generated at: `{report['generated_at']}`",
|
||||
f"- Pattern report: `{report['source_patterns']}`",
|
||||
f"- Proposal only: `{str(report['proposal_contract']['proposal_only']).lower()}`",
|
||||
f"- Writes repository files: `{str(report['proposal_contract']['writes_repository_files']).lower()}`",
|
||||
f"- Proposals: `{report['summary']['proposal_count']}`",
|
||||
"",
|
||||
]
|
||||
if not report["proposals"]:
|
||||
lines.append("No proposals were generated.")
|
||||
for proposal in report["proposals"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"## {proposal['title']}",
|
||||
"",
|
||||
f"- ID: `{proposal['proposal_id']}`",
|
||||
f"- Status: `{proposal['status']}`",
|
||||
f"- Pattern: `{proposal['pattern_id']}`",
|
||||
f"- Risk: `{proposal['risk_level']}`",
|
||||
f"- Requires approval: `{str(proposal['requires_approval']).lower()}`",
|
||||
f"- Reason: {proposal['reason']}",
|
||||
"- Target files:",
|
||||
]
|
||||
)
|
||||
lines.extend(f" - `{path}`" for path in proposal["target_files"])
|
||||
lines.append("- Suggested changes:")
|
||||
lines.extend(f" - {item}" for item in proposal["suggested_changes"])
|
||||
lines.append("- Verification:")
|
||||
lines.extend(f" - `{item}`" for item in proposal["verification_commands"])
|
||||
lines.append(f"- Rollback: {proposal['rollback_plan']}")
|
||||
if proposal["evidence_refs"]:
|
||||
lines.append("- Redacted evidence refs:")
|
||||
lines.extend(f" - `{item['record_id']}`: {item['excerpt']}" for item in proposal["evidence_refs"])
|
||||
lines.append("")
|
||||
if report["failures"]:
|
||||
lines.extend(["## Failures", ""])
|
||||
lines.extend(f"- {failure}" for failure in report["failures"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Create proposal-only adaptation plans from summarized user signal patterns.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--patterns-json", default="reports/user_patterns.json")
|
||||
parser.add_argument("--output-json", default="reports/adaptation_proposals.json")
|
||||
parser.add_argument("--output-md", default="reports/adaptation_proposals.md")
|
||||
parser.add_argument("--generated-at", default=utc_now())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(skill_dir, Path(args.patterns_json), args.generated_at)
|
||||
if report["ok"]:
|
||||
output_json = resolve_output(skill_dir, args.output_json)
|
||||
output_md = resolve_output(skill_dir, args.output_md)
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
report["artifacts"] = {
|
||||
"json": display_path(output_json, skill_dir),
|
||||
"markdown": display_path(output_md, skill_dir),
|
||||
}
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1/responses"
|
||||
DEFAULT_DEEPSEEK_BASE_URL = "https://api.deepseek.com/chat/completions"
|
||||
DEFAULT_BASE_URL = DEFAULT_OPENAI_BASE_URL
|
||||
DEFAULT_HOST = urlparse(DEFAULT_OPENAI_BASE_URL).hostname or "api.openai.com"
|
||||
DEFAULT_PROVIDER_CONFIGS = {
|
||||
"openai": {
|
||||
"base_url": DEFAULT_OPENAI_BASE_URL,
|
||||
"api_format": "responses",
|
||||
"api_key_env": "OPENAI_API_KEY",
|
||||
"thinking": "",
|
||||
},
|
||||
"deepseek": {
|
||||
"base_url": DEFAULT_DEEPSEEK_BASE_URL,
|
||||
"api_format": "chat-completions",
|
||||
"api_key_env": "DEEPSEEK_API_KEY",
|
||||
"thinking": "disabled",
|
||||
},
|
||||
}
|
||||
ALLOWED_PATH_PREFIX = "/v1/responses"
|
||||
CHAT_COMPLETIONS_PATHS = {"/chat/completions", "/v1/chat/completions"}
|
||||
LOCAL_HOSTS = {"127.0.0.1", "localhost", "::1"}
|
||||
|
||||
|
||||
def fail(message: str) -> None:
|
||||
print(message, file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
def validate_base_url(
|
||||
base_url: str,
|
||||
allow_insecure_localhost: bool,
|
||||
allow_custom_base_url: bool,
|
||||
api_format: str,
|
||||
) -> None:
|
||||
parsed = urlparse(base_url)
|
||||
host = parsed.hostname or ""
|
||||
if api_format == "responses" and not parsed.path.startswith(ALLOWED_PATH_PREFIX):
|
||||
fail(f"provider endpoint path must start with {ALLOWED_PATH_PREFIX}")
|
||||
if api_format == "chat-completions" and parsed.path not in CHAT_COMPLETIONS_PATHS:
|
||||
fail("chat-completions provider endpoint path must be /chat/completions or /v1/chat/completions")
|
||||
if parsed.scheme == "https" and (
|
||||
base_url in {DEFAULT_OPENAI_BASE_URL, DEFAULT_DEEPSEEK_BASE_URL}
|
||||
or host == DEFAULT_HOST
|
||||
or allow_custom_base_url
|
||||
):
|
||||
return
|
||||
if parsed.scheme == "https":
|
||||
fail("custom provider host requires --allow-custom-base-url")
|
||||
if parsed.scheme == "http" and allow_insecure_localhost and (parsed.hostname or "") in LOCAL_HOSTS:
|
||||
return
|
||||
fail("provider runner requires HTTPS; use --allow-insecure-localhost only for local test servers")
|
||||
|
||||
|
||||
def provider_defaults(provider: str) -> dict[str, str]:
|
||||
return DEFAULT_PROVIDER_CONFIGS.get(provider, DEFAULT_PROVIDER_CONFIGS["openai"])
|
||||
|
||||
|
||||
def load_request() -> dict[str, Any]:
|
||||
raw = sys.stdin.read()
|
||||
if not raw.strip():
|
||||
fail("provider runner requires a JSON request on stdin")
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
fail(f"invalid JSON request: {exc}")
|
||||
if not isinstance(payload, dict):
|
||||
fail("runner request must be a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def safe_relative(path_value: str) -> Path | None:
|
||||
path = Path(path_value)
|
||||
if path.is_absolute() or ".." in path.parts:
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def read_input_files(paths: Any, input_root: Path, max_chars: int) -> list[dict[str, str]]:
|
||||
if not isinstance(paths, list):
|
||||
return []
|
||||
files: list[dict[str, str]] = []
|
||||
for item in paths:
|
||||
rel = safe_relative(str(item))
|
||||
if rel is None:
|
||||
files.append({"path": str(item), "status": "skipped-unsafe-path", "content": ""})
|
||||
continue
|
||||
path = input_root / rel
|
||||
if not path.exists() or not path.is_file():
|
||||
files.append({"path": str(item), "status": "missing", "content": ""})
|
||||
continue
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
files.append({"path": str(item), "status": "loaded", "content": text[:max_chars]})
|
||||
return files
|
||||
|
||||
|
||||
def read_skill_instructions(path: Path, max_chars: int) -> str:
|
||||
if not path.exists() or not path.is_file():
|
||||
return ""
|
||||
return path.read_text(encoding="utf-8", errors="replace")[:max_chars]
|
||||
|
||||
|
||||
def build_provider_input(request: dict[str, Any], skill_text: str, input_files: list[dict[str, str]]) -> str:
|
||||
variant = str(request.get("variant", ""))
|
||||
lines = [
|
||||
"You are producing one output for a Yao Meta Skill output-eval case.",
|
||||
f"Case id: {request.get('case_id', '')}",
|
||||
f"Variant: {variant}",
|
||||
"",
|
||||
"User task:",
|
||||
str(request.get("prompt", "")),
|
||||
"",
|
||||
]
|
||||
if variant == "with_skill":
|
||||
lines.extend(
|
||||
[
|
||||
"Use the skill instructions below as the operating guidance. Preserve concrete evidence paths and boundaries.",
|
||||
"",
|
||||
"Skill instructions:",
|
||||
skill_text or "(Skill instructions were not available.)",
|
||||
"",
|
||||
]
|
||||
)
|
||||
else:
|
||||
lines.extend(
|
||||
[
|
||||
"Produce a direct baseline answer without using the Yao Meta Skill guidance.",
|
||||
"Do not invent files, reports, governance evidence, or hidden review artifacts.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
if input_files:
|
||||
lines.append("Input files:")
|
||||
for item in input_files:
|
||||
lines.append(f"--- {item['path']} ({item['status']}) ---")
|
||||
if item["content"]:
|
||||
lines.append(item["content"])
|
||||
lines.append("")
|
||||
lines.extend(
|
||||
[
|
||||
"Return only the final user-facing answer for this variant.",
|
||||
"Do not mention or copy any fixture output from the eval case.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def response_text(payload: dict[str, Any]) -> str:
|
||||
if isinstance(payload.get("output_text"), str):
|
||||
return str(payload["output_text"])
|
||||
parts: list[str] = []
|
||||
output = payload.get("output")
|
||||
if isinstance(output, list):
|
||||
for item in output:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
content = item.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if not isinstance(block, dict):
|
||||
continue
|
||||
if isinstance(block.get("text"), str):
|
||||
parts.append(str(block["text"]))
|
||||
if parts:
|
||||
return "\n".join(part for part in parts if part).strip()
|
||||
choices = payload.get("choices")
|
||||
if isinstance(choices, list) and choices:
|
||||
first = choices[0]
|
||||
if isinstance(first, dict):
|
||||
message = first.get("message", {})
|
||||
if isinstance(message, dict) and isinstance(message.get("content"), str):
|
||||
return str(message["content"])
|
||||
return ""
|
||||
|
||||
|
||||
def observed_usage(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
usage = payload.get("usage", {})
|
||||
if not isinstance(usage, dict):
|
||||
return {}
|
||||
input_tokens = usage.get("input_tokens", usage.get("prompt_tokens"))
|
||||
output_tokens = usage.get("output_tokens", usage.get("completion_tokens"))
|
||||
total_tokens = usage.get("total_tokens")
|
||||
result: dict[str, Any] = {}
|
||||
if input_tokens is not None:
|
||||
result["input_tokens"] = int(input_tokens)
|
||||
if output_tokens is not None:
|
||||
result["output_tokens"] = int(output_tokens)
|
||||
if total_tokens is not None:
|
||||
result["total_tokens"] = int(total_tokens)
|
||||
if result:
|
||||
result["estimated"] = False
|
||||
return result
|
||||
|
||||
|
||||
def request_body(
|
||||
model: str,
|
||||
provider_input: str,
|
||||
api_format: str,
|
||||
thinking_type: str,
|
||||
temperature: float | None,
|
||||
) -> dict[str, Any]:
|
||||
if api_format == "chat-completions":
|
||||
body: dict[str, Any] = {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": provider_input}],
|
||||
}
|
||||
if thinking_type:
|
||||
body["thinking"] = {"type": thinking_type}
|
||||
if temperature is not None:
|
||||
body["temperature"] = temperature
|
||||
return body
|
||||
body = {"model": model, "input": provider_input}
|
||||
if temperature is not None:
|
||||
body["temperature"] = temperature
|
||||
return body
|
||||
|
||||
|
||||
def call_provider(
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
provider_input: str,
|
||||
timeout_seconds: float,
|
||||
api_format: str,
|
||||
thinking_type: str,
|
||||
temperature: float | None,
|
||||
) -> dict[str, Any]:
|
||||
body = json.dumps(
|
||||
request_body(model, provider_input, api_format, thinking_type, temperature),
|
||||
ensure_ascii=False,
|
||||
).encode("utf-8")
|
||||
request = Request(
|
||||
base_url,
|
||||
data=body,
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=timeout_seconds) as response:
|
||||
response_body = response.read().decode("utf-8", errors="replace")
|
||||
except HTTPError as exc:
|
||||
detail = exc.read().decode("utf-8", errors="replace")[:500]
|
||||
fail(f"provider request failed with HTTP {exc.code}: {detail}")
|
||||
except URLError as exc:
|
||||
fail(f"provider request failed: {exc.reason}")
|
||||
try:
|
||||
payload = json.loads(response_body)
|
||||
except json.JSONDecodeError as exc:
|
||||
fail(f"provider returned invalid JSON: {exc}")
|
||||
if not isinstance(payload, dict):
|
||||
fail("provider response must be a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Provider-backed output-eval runner for run_output_execution.py. "
|
||||
"Requires real model credentials and reports execution_kind=model only after an HTTP provider call."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--provider", default="openai", help="Provider label to write into execution evidence.")
|
||||
parser.add_argument(
|
||||
"--base-url",
|
||||
help="Override the provider endpoint. Defaults are selected from --provider.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--api-format",
|
||||
choices=["responses", "chat-completions"],
|
||||
help="Provider API shape. Defaults are selected from --provider.",
|
||||
)
|
||||
parser.add_argument("--thinking", choices=["enabled", "disabled"], help="Optional chat-completions thinking mode.")
|
||||
parser.add_argument("--temperature", type=float, default=0.0, help="Sampling temperature for provider requests.")
|
||||
parser.add_argument("--model", default=os.environ.get("YAO_OUTPUT_EVAL_MODEL", ""))
|
||||
parser.add_argument("--api-key-env", help="Environment variable that contains the provider API key.")
|
||||
parser.add_argument("--input-root", default=str(ROOT / "evals" / "output"))
|
||||
parser.add_argument("--skill-file", default=str(ROOT / "SKILL.md"))
|
||||
parser.add_argument("--timeout-seconds", type=float, default=60.0)
|
||||
parser.add_argument("--max-input-file-chars", type=int, default=6000)
|
||||
parser.add_argument("--max-skill-chars", type=int, default=8000)
|
||||
parser.add_argument("--allow-insecure-localhost", action="store_true")
|
||||
parser.add_argument("--allow-custom-base-url", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
defaults = provider_defaults(args.provider)
|
||||
base_url = args.base_url or defaults["base_url"]
|
||||
api_format = args.api_format or defaults["api_format"]
|
||||
thinking = args.thinking if args.thinking is not None else defaults["thinking"]
|
||||
api_key_env = args.api_key_env or defaults["api_key_env"]
|
||||
|
||||
validate_base_url(base_url, args.allow_insecure_localhost, args.allow_custom_base_url, api_format)
|
||||
if args.temperature is not None and not 0 <= args.temperature <= 2:
|
||||
fail("--temperature must be between 0 and 2")
|
||||
if not args.model:
|
||||
fail("missing model; pass --model or set YAO_OUTPUT_EVAL_MODEL")
|
||||
api_key = os.environ.get(api_key_env, "")
|
||||
if not api_key:
|
||||
fail(f"missing API key env: {api_key_env}")
|
||||
|
||||
request = load_request()
|
||||
input_files = read_input_files(request.get("input_files", []), Path(args.input_root).resolve(), args.max_input_file_chars)
|
||||
skill_text = read_skill_instructions(Path(args.skill_file).resolve(), args.max_skill_chars)
|
||||
provider_input = build_provider_input(request, skill_text, input_files)
|
||||
response = call_provider(
|
||||
base_url,
|
||||
api_key,
|
||||
args.model,
|
||||
provider_input,
|
||||
args.timeout_seconds,
|
||||
api_format,
|
||||
thinking,
|
||||
args.temperature,
|
||||
)
|
||||
output = response_text(response)
|
||||
if not output:
|
||||
fail("provider response did not contain output text")
|
||||
result = {
|
||||
"output": output,
|
||||
"execution_kind": "model",
|
||||
"provider": args.provider,
|
||||
"model": args.model,
|
||||
"usage": observed_usage(response),
|
||||
"response_id": str(response.get("id", "")),
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,351 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import token
|
||||
import tokenize
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Checks repository Python source for syntax that can pass locally but fail on the supported CI interpreter."
|
||||
|
||||
EXCLUDED_DIRS = {
|
||||
".git",
|
||||
".mypy_cache",
|
||||
".previews",
|
||||
".pytest_cache",
|
||||
".ruff_cache",
|
||||
".venv",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
"node_modules",
|
||||
"venv",
|
||||
}
|
||||
MAX_FILE_BYTES = 1_000_000
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def resolve_path(raw_path: str, root: Path) -> Path:
|
||||
path = Path(raw_path)
|
||||
if not path.is_absolute():
|
||||
path = root / path
|
||||
return path.resolve()
|
||||
|
||||
|
||||
def is_ignored(path: Path, root: Path) -> bool:
|
||||
try:
|
||||
parts = path.resolve().relative_to(root.resolve()).parts
|
||||
except ValueError:
|
||||
return True
|
||||
if any(part in EXCLUDED_DIRS for part in parts):
|
||||
return True
|
||||
if len(parts) >= 2 and parts[0] == "tests" and parts[1].startswith("tmp"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def expand_scan_paths(root: Path, raw_paths: list[str]) -> list[Path]:
|
||||
candidates: list[Path] = []
|
||||
explicit_paths = bool(raw_paths)
|
||||
if raw_paths:
|
||||
for raw_path in raw_paths:
|
||||
path = resolve_path(raw_path, root)
|
||||
if path.is_dir():
|
||||
candidates.extend(path.rglob("*.py"))
|
||||
elif path.exists():
|
||||
candidates.append(path)
|
||||
else:
|
||||
candidates.extend(root.rglob("*.py"))
|
||||
files = []
|
||||
for path in candidates:
|
||||
if not path.is_file() or path.is_symlink() or path.suffix != ".py":
|
||||
continue
|
||||
if not explicit_paths and is_ignored(path, root):
|
||||
continue
|
||||
try:
|
||||
if path.stat().st_size > MAX_FILE_BYTES:
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
files.append(path)
|
||||
return sorted(set(files), key=lambda item: rel_path(item, root))
|
||||
|
||||
|
||||
def issue(path: Path, root: Path, line: int, column: int, rule: str, message: str, excerpt: str = "") -> dict[str, Any]:
|
||||
return {
|
||||
"path": rel_path(path, root),
|
||||
"line": line,
|
||||
"column": column,
|
||||
"rule": rule,
|
||||
"message": message,
|
||||
"excerpt": excerpt.strip()[:220],
|
||||
}
|
||||
|
||||
|
||||
def compile_issues(path: Path, root: Path, source: str) -> list[dict[str, Any]]:
|
||||
try:
|
||||
compile(source, str(path), "exec")
|
||||
except SyntaxError as exc:
|
||||
return [
|
||||
issue(
|
||||
path,
|
||||
root,
|
||||
exc.lineno or 0,
|
||||
exc.offset or 0,
|
||||
"current-python-syntax",
|
||||
str(exc.msg),
|
||||
exc.text or "",
|
||||
)
|
||||
]
|
||||
return []
|
||||
|
||||
|
||||
def token_name(token_type: int) -> str:
|
||||
return token.tok_name.get(token_type, "")
|
||||
|
||||
|
||||
def scan_modern_fstring_tokens(path: Path, root: Path, source: str) -> tuple[bool, list[dict[str, Any]]]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
try:
|
||||
tokens = list(tokenize.generate_tokens(io.StringIO(source).readline))
|
||||
except tokenize.TokenError:
|
||||
return False, findings
|
||||
has_modern_fstring_tokens = any(token_name(item.type) == "FSTRING_START" for item in tokens)
|
||||
if not has_modern_fstring_tokens:
|
||||
return False, findings
|
||||
fstring_depth = 0
|
||||
expression_depth = 0
|
||||
for item in tokens:
|
||||
name = token_name(item.type)
|
||||
if name == "FSTRING_START":
|
||||
fstring_depth += 1
|
||||
expression_depth = 0
|
||||
continue
|
||||
if name == "FSTRING_END":
|
||||
fstring_depth = max(0, fstring_depth - 1)
|
||||
expression_depth = 0
|
||||
continue
|
||||
if not fstring_depth:
|
||||
continue
|
||||
if item.type == token.OP and item.string == "{":
|
||||
expression_depth += 1
|
||||
continue
|
||||
if item.type == token.OP and item.string == "}":
|
||||
expression_depth = max(0, expression_depth - 1)
|
||||
continue
|
||||
if expression_depth > 0 and "\\" in item.string:
|
||||
findings.append(
|
||||
issue(
|
||||
path,
|
||||
root,
|
||||
item.start[0],
|
||||
item.start[1] + 1,
|
||||
"fstring-expression-backslash",
|
||||
"Python 3.11 rejects backslashes inside f-string expressions.",
|
||||
item.line,
|
||||
)
|
||||
)
|
||||
return True, findings
|
||||
|
||||
|
||||
def split_string_token(token_text: str) -> tuple[str, str] | None:
|
||||
index = 0
|
||||
while index < len(token_text) and token_text[index].isalpha():
|
||||
index += 1
|
||||
prefix = token_text[:index].lower()
|
||||
if "f" not in prefix:
|
||||
return None
|
||||
quote = ""
|
||||
for candidate in ('"""', "'''", '"', "'"):
|
||||
if token_text[index:].startswith(candidate):
|
||||
quote = candidate
|
||||
break
|
||||
if not quote or not token_text.endswith(quote):
|
||||
return None
|
||||
return prefix, token_text[index + len(quote) : -len(quote)]
|
||||
|
||||
|
||||
def scan_legacy_fstring_body(path: Path, root: Path, token_text: str, start_line: int, start_column: int) -> list[dict[str, Any]]:
|
||||
parsed = split_string_token(token_text)
|
||||
if not parsed:
|
||||
return []
|
||||
_, body = parsed
|
||||
findings: list[dict[str, Any]] = []
|
||||
expression_depth = 0
|
||||
line = start_line
|
||||
column = start_column
|
||||
index = 0
|
||||
while index < len(body):
|
||||
char = body[index]
|
||||
next_char = body[index + 1] if index + 1 < len(body) else ""
|
||||
if expression_depth == 0:
|
||||
if char == "{" and next_char == "{":
|
||||
index += 2
|
||||
column += 2
|
||||
continue
|
||||
if char == "{" and next_char != "{":
|
||||
expression_depth = 1
|
||||
else:
|
||||
if char == "\\":
|
||||
findings.append(
|
||||
issue(
|
||||
path,
|
||||
root,
|
||||
line,
|
||||
column + 1,
|
||||
"fstring-expression-backslash",
|
||||
"Python 3.11 rejects backslashes inside f-string expressions.",
|
||||
token_text,
|
||||
)
|
||||
)
|
||||
elif char == "{":
|
||||
expression_depth += 1
|
||||
elif char == "}":
|
||||
expression_depth = max(0, expression_depth - 1)
|
||||
if char == "\n":
|
||||
line += 1
|
||||
column = 0
|
||||
else:
|
||||
column += 1
|
||||
index += 1
|
||||
return findings
|
||||
|
||||
|
||||
def fstring_compat_issues(path: Path, root: Path, source: str) -> list[dict[str, Any]]:
|
||||
handled, findings = scan_modern_fstring_tokens(path, root, source)
|
||||
if handled:
|
||||
return findings
|
||||
try:
|
||||
tokens = tokenize.generate_tokens(io.StringIO(source).readline)
|
||||
for item in tokens:
|
||||
if item.type == token.STRING:
|
||||
findings.extend(scan_legacy_fstring_body(path, root, item.string, item.start[0], item.start[1]))
|
||||
except tokenize.TokenError:
|
||||
return findings
|
||||
return findings
|
||||
|
||||
|
||||
def check_file(path: Path, root: Path) -> dict[str, Any]:
|
||||
try:
|
||||
source = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError as exc:
|
||||
issues = [issue(path, root, 0, 0, "read-error", str(exc))]
|
||||
else:
|
||||
issues = compile_issues(path, root, source)
|
||||
issues.extend(fstring_compat_issues(path, root, source))
|
||||
return {
|
||||
"path": rel_path(path, root),
|
||||
"ok": not issues,
|
||||
"issue_count": len(issues),
|
||||
"issues": issues,
|
||||
}
|
||||
|
||||
|
||||
def build_report(root: Path, raw_paths: list[str], target_python: str, generated_at: str) -> dict[str, Any]:
|
||||
files = expand_scan_paths(root, raw_paths)
|
||||
checked = [check_file(path, root) for path in files]
|
||||
issues = [item for file_report in checked for item in file_report["issues"]]
|
||||
syntax_error_count = sum(1 for item in issues if item["rule"] == "current-python-syntax")
|
||||
fstring_count = sum(1 for item in issues if item["rule"] == "fstring-expression-backslash")
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": not issues,
|
||||
"generated_at": generated_at,
|
||||
"root": rel_path(root, ROOT),
|
||||
"summary": {
|
||||
"target_python": target_python,
|
||||
"file_count": len(checked),
|
||||
"issue_count": len(issues),
|
||||
"syntax_error_count": syntax_error_count,
|
||||
"fstring_311_violation_count": fstring_count,
|
||||
"decision": "pass" if not issues else "block-python-compat",
|
||||
},
|
||||
"rules": [
|
||||
{
|
||||
"key": "current-python-syntax",
|
||||
"reason": "Every scanned Python source file must compile under the running interpreter.",
|
||||
},
|
||||
{
|
||||
"key": "fstring-expression-backslash",
|
||||
"reason": "Python 3.11 rejects backslashes inside f-string expressions; keep escaping outside the expression.",
|
||||
},
|
||||
],
|
||||
"files": checked,
|
||||
"issues": issues,
|
||||
"artifacts": {
|
||||
"json": "reports/python_compatibility.json",
|
||||
"markdown": "reports/python_compatibility.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Python Compatibility",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- target python: `{summary['target_python']}`",
|
||||
f"- files scanned: `{summary['file_count']}`",
|
||||
f"- issues: `{summary['issue_count']}`",
|
||||
f"- syntax errors: `{summary['syntax_error_count']}`",
|
||||
f"- f-string 3.11 violations: `{summary['fstring_311_violation_count']}`",
|
||||
"",
|
||||
"This report catches Python syntax and compatibility hazards that can pass on a newer local interpreter but fail in the supported CI/runtime interpreter.",
|
||||
"",
|
||||
"## Issues",
|
||||
"",
|
||||
"| Path | Line | Rule | Message |",
|
||||
"| --- | ---: | --- | --- |",
|
||||
]
|
||||
if report["issues"]:
|
||||
for item in report["issues"]:
|
||||
message = str(item["message"]).replace("|", "\\|")
|
||||
lines.append(f"| `{item['path']}` | {item['line']} | `{item['rule']}` | {message} |")
|
||||
else:
|
||||
lines.append("| `none` | 0 | `none` | none |")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Check Python source compatibility for supported CI/runtime versions.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--path", action="append", default=[], help="Optional file or directory to scan relative to skill_dir.")
|
||||
parser.add_argument("--target-python", default="3.11")
|
||||
parser.add_argument("--output-json", default="reports/python_compatibility.json")
|
||||
parser.add_argument("--output-md", default="reports/python_compatibility.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.skill_dir).resolve()
|
||||
report = build_report(root, args.path, args.target_python, args.generated_at)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = root / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = root / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_reference_synthesis.py to keep reference synthesis modeling separate from Markdown rendering."
|
||||
|
||||
|
||||
def render_markdown(summary: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# Reference Synthesis",
|
||||
"",
|
||||
f"Skill: `{summary['skill_name']}`",
|
||||
f"- Description: {summary['description']}",
|
||||
f"- Intent confidence: `{summary['intent_confidence']['score']}/100` (`{summary['intent_confidence']['band']}`)",
|
||||
"",
|
||||
"## Live GitHub Benchmarks",
|
||||
"",
|
||||
]
|
||||
if summary["github_benchmarks"]:
|
||||
for repo in summary["github_benchmarks"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {repo['name']}",
|
||||
f"- URL: {repo['url']}",
|
||||
f"- Stars: `{repo['stars']}`",
|
||||
]
|
||||
)
|
||||
for item in repo.get("borrow", []):
|
||||
lines.append(f"- Borrow: {item}")
|
||||
lines.append("")
|
||||
else:
|
||||
lines.append("- No live GitHub benchmarks are attached yet.")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(["## Curated World-Class Pattern Tracks", ""])
|
||||
for track in summary["source_tracks"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {track['name']}",
|
||||
f"- Type: `{track['source_type']}`",
|
||||
f"- Evidence mode: `{track['evidence_mode']}`",
|
||||
f"- Why relevant: {track['why_relevant']}",
|
||||
f"- Borrow: {track['borrow']}",
|
||||
f"- Avoid: {track['avoid']}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
lines.extend(["## Borrow Now", ""])
|
||||
for item in summary["synthesis"]["borrow_now"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Avoid Now", ""])
|
||||
for item in summary["synthesis"]["avoid_now"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Pattern Gate", ""])
|
||||
pattern_gate = summary["synthesis"]["pattern_gate"]
|
||||
lines.append(f"- Summary: {pattern_gate['summary']}")
|
||||
lines.append(f"- Acceptance threshold: `{pattern_gate['threshold']}/4`")
|
||||
if pattern_gate["accepted"]:
|
||||
lines.append("- Accepted patterns:")
|
||||
for item in pattern_gate["accepted"][:5]:
|
||||
lines.append(
|
||||
f" - **{item['name']}**: {item['score']}/4 "
|
||||
f"({', '.join(item['passed'])})"
|
||||
)
|
||||
if pattern_gate["deferred"]:
|
||||
lines.append("- Deferred patterns:")
|
||||
for item in pattern_gate["deferred"][:5]:
|
||||
lines.append(
|
||||
f" - **{item['name']}**: missing {', '.join(item['missing']) or 'none'}"
|
||||
)
|
||||
|
||||
lines.extend(["", "## Default Recommendation", ""])
|
||||
lines.append(f"- Summary: {summary['synthesis']['recommendation']['summary']}")
|
||||
lines.append(f"- Why: {summary['synthesis']['recommendation']['why']}")
|
||||
lines.append(f"- User decision required: `{summary['synthesis']['recommendation']['user_decision_required']}`")
|
||||
|
||||
lines.extend(["", "## Visibility Mode", ""])
|
||||
lines.append(f"- Mode: `{summary['synthesis']['visibility']['mode']}`")
|
||||
if summary["synthesis"]["visibility"]["reasons"]:
|
||||
lines.append(f"- Reasons: {', '.join(summary['synthesis']['visibility']['reasons'])}")
|
||||
lines.append(f"- User note: {summary['synthesis']['visibility']['user_note']}")
|
||||
lines.append(f"- Reviewer note: {summary['synthesis']['visibility']['reviewer_note']}")
|
||||
|
||||
lines.extend(["", "## Conflict Check", ""])
|
||||
if summary["synthesis"]["conflicts"]:
|
||||
for conflict in summary["synthesis"]["conflicts"]:
|
||||
lines.append(f"- **{conflict['key']}**: {conflict['summary']}")
|
||||
else:
|
||||
lines.append("- No material design conflict detected. Keep the synthesis silent for the user.")
|
||||
|
||||
lines.extend(["", "## Quality Lift Thesis", ""])
|
||||
for item in summary["synthesis"]["quality_risers"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Decision Prompt", "", summary["synthesis"]["decision_prompt"], ""])
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
@@ -11,6 +11,8 @@ try:
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
from skill_ir_paths import find_skill_ir as find_skill_ir_document
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_REGISTRY_DIR = ROOT / "registry"
|
||||
@@ -73,16 +75,7 @@ def read_frontmatter(path: Path) -> dict[str, Any]:
|
||||
|
||||
|
||||
def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "skill-ir.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{name}.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
|
||||
]
|
||||
for path in candidates:
|
||||
payload = load_json(path)
|
||||
if payload:
|
||||
return payload, display_path(path, skill_dir)
|
||||
return {}, "missing"
|
||||
return find_skill_ir_document(skill_dir, name, fallback_source="missing")
|
||||
|
||||
|
||||
def license_id(skill_dir: Path) -> str:
|
||||
|
||||
@@ -10,6 +10,7 @@ from typing import Any
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
ALLOWED_EVENTS = {"skill_activation", "skill_output", "script_run", "review_event"}
|
||||
ADOPTION_EVENTS = {"skill_activation", "skill_output"}
|
||||
ALLOWED_ACTIVATION_TYPES = {"implicit", "explicit", "manual", "unknown"}
|
||||
ALLOWED_OUTCOMES = {"accepted", "edited", "rejected", "missed", "failed", "reviewed", "unknown"}
|
||||
ALLOWED_FAILURE_TYPES = {
|
||||
@@ -22,14 +23,17 @@ ALLOWED_FAILURE_TYPES = {
|
||||
"review_overdue",
|
||||
}
|
||||
ALLOWED_FIELDS = {
|
||||
"command",
|
||||
"event",
|
||||
"skill",
|
||||
"source",
|
||||
"version",
|
||||
"activation_type",
|
||||
"outcome",
|
||||
"failure_type",
|
||||
"timestamp",
|
||||
}
|
||||
ALLOWED_SOURCES = {"manual", "yao_cli", "external", "unknown"}
|
||||
SENSITIVE_FIELDS = {
|
||||
"prompt",
|
||||
"content",
|
||||
@@ -111,6 +115,8 @@ def normalize_event(raw: dict[str, Any], defaults: dict[str, str], line_label: s
|
||||
timestamp = str(raw.get("timestamp") or utc_now())
|
||||
skill = str(raw.get("skill") or defaults["skill"])
|
||||
version = str(raw.get("version") or defaults["version"])
|
||||
source = str(raw.get("source") or "manual")
|
||||
command = str(raw.get("command") or "unknown")
|
||||
|
||||
if event not in ALLOWED_EVENTS:
|
||||
failures.append(f"{line_label}: unsupported event `{event}`")
|
||||
@@ -120,12 +126,18 @@ def normalize_event(raw: dict[str, Any], defaults: dict[str, str], line_label: s
|
||||
failures.append(f"{line_label}: unsupported outcome `{outcome}`")
|
||||
if failure_type not in ALLOWED_FAILURE_TYPES:
|
||||
failures.append(f"{line_label}: unsupported failure_type `{failure_type}`")
|
||||
if source not in ALLOWED_SOURCES:
|
||||
failures.append(f"{line_label}: unsupported source `{source}`")
|
||||
if not command.replace("-", "").replace("_", "").isalnum() or len(command) > 64:
|
||||
failures.append(f"{line_label}: command must use only letters, numbers, hyphens, or underscores and stay under 64 chars")
|
||||
|
||||
if failures:
|
||||
return None, failures
|
||||
return {
|
||||
"command": command,
|
||||
"event": event,
|
||||
"skill": skill,
|
||||
"source": source,
|
||||
"version": version,
|
||||
"activation_type": activation_type,
|
||||
"outcome": outcome,
|
||||
@@ -165,21 +177,27 @@ def append_event(path: Path, event: dict[str, str]) -> None:
|
||||
|
||||
def adoption_by_skill(events: list[dict[str, str]]) -> list[dict[str, Any]]:
|
||||
grouped: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
adoption_grouped: dict[str, Counter[str]] = defaultdict(Counter)
|
||||
for event in events:
|
||||
grouped[event["skill"]][event["outcome"]] += 1
|
||||
skill = event["skill"]
|
||||
grouped[skill]["events"] += 1
|
||||
if event["event"] in ADOPTION_EVENTS:
|
||||
adoption_grouped[skill][event["outcome"]] += 1
|
||||
rows = []
|
||||
for skill, counts in sorted(grouped.items()):
|
||||
total = sum(counts.values())
|
||||
adopted = counts["accepted"] + counts["edited"]
|
||||
adoption_counts = adoption_grouped[skill]
|
||||
adoption_total = sum(adoption_counts.values())
|
||||
adopted = adoption_counts["accepted"] + adoption_counts["edited"]
|
||||
rows.append(
|
||||
{
|
||||
"skill": skill,
|
||||
"events": total,
|
||||
"accepted": counts["accepted"],
|
||||
"edited": counts["edited"],
|
||||
"rejected": counts["rejected"],
|
||||
"missed": counts["missed"],
|
||||
"adoption_rate": round(adopted / total * 100, 1) if total else 0,
|
||||
"events": counts["events"],
|
||||
"adoption_events": adoption_total,
|
||||
"accepted": adoption_counts["accepted"],
|
||||
"edited": adoption_counts["edited"],
|
||||
"rejected": adoption_counts["rejected"],
|
||||
"missed": adoption_counts["missed"],
|
||||
"adoption_rate": round(adopted / adoption_total * 100, 1) if adoption_total else 0,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
@@ -243,11 +261,15 @@ def next_candidates(summary: dict[str, Any]) -> list[dict[str, str]]:
|
||||
|
||||
|
||||
def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[str, Any]:
|
||||
outcomes = Counter(event["outcome"] for event in events)
|
||||
adoption_events = [event for event in events if event["event"] in ADOPTION_EVENTS]
|
||||
outcomes = Counter(event["outcome"] for event in adoption_events)
|
||||
failures = Counter(event["failure_type"] for event in events if event["failure_type"] != "none")
|
||||
event_types = Counter(event["event"] for event in events)
|
||||
source_types = Counter(event.get("source", "manual") for event in events)
|
||||
command_counts = Counter(event.get("command", "unknown") for event in events if event.get("command", "unknown") != "unknown")
|
||||
adopted = outcomes["accepted"] + outcomes["edited"]
|
||||
event_count = len(events)
|
||||
adoption_sample_count = len(adoption_events)
|
||||
missed_trigger = outcomes["missed"] + failures["under_trigger"]
|
||||
bad_output = failures["bad_output"]
|
||||
script_error = failures["script_error"]
|
||||
@@ -263,13 +285,14 @@ def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[s
|
||||
risk_band = "low"
|
||||
return {
|
||||
"event_count": event_count,
|
||||
"adoption_sample_count": adoption_sample_count,
|
||||
"activation_count": event_types["skill_activation"],
|
||||
"accepted_count": outcomes["accepted"],
|
||||
"edited_count": outcomes["edited"],
|
||||
"rejected_count": outcomes["rejected"],
|
||||
"missed_count": outcomes["missed"],
|
||||
"failed_count": outcomes["failed"],
|
||||
"adoption_rate": round(adopted / event_count * 100, 1) if event_count else 0,
|
||||
"adoption_rate": round(adopted / adoption_sample_count * 100, 1) if adoption_sample_count else 0,
|
||||
"missed_trigger_count": missed_trigger,
|
||||
"wrong_trigger_count": wrong_trigger,
|
||||
"bad_output_count": bad_output,
|
||||
@@ -279,6 +302,8 @@ def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[s
|
||||
"risk_band": risk_band,
|
||||
"event_types": dict(sorted(event_types.items())),
|
||||
"failure_types": dict(sorted(failures.items())),
|
||||
"source_types": dict(sorted(source_types.items())),
|
||||
"command_counts": dict(sorted(command_counts.items())),
|
||||
}
|
||||
|
||||
|
||||
@@ -292,6 +317,7 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
"## Summary",
|
||||
"",
|
||||
f"- Events: `{summary['event_count']}`",
|
||||
f"- Adoption samples: `{summary['adoption_sample_count']}`",
|
||||
f"- Activation events: `{summary['activation_count']}`",
|
||||
f"- Adoption rate: `{summary['adoption_rate']}`",
|
||||
f"- Missed trigger signals: `{summary['missed_trigger_count']}`",
|
||||
@@ -309,16 +335,16 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
"",
|
||||
"## Adoption By Skill",
|
||||
"",
|
||||
"| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
"| Skill | Events | Adoption Samples | Accepted | Edited | Rejected | Missed | Adoption Rate |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for row in report["adoption_by_skill"]:
|
||||
lines.append(
|
||||
f"| `{row['skill']}` | {row['events']} | {row['accepted']} | {row['edited']} | "
|
||||
f"| `{row['skill']}` | {row['events']} | {row['adoption_events']} | {row['accepted']} | {row['edited']} | "
|
||||
f"{row['rejected']} | {row['missed']} | {row['adoption_rate']} |"
|
||||
)
|
||||
if not report["adoption_by_skill"]:
|
||||
lines.append("| `none` | 0 | 0 | 0 | 0 | 0 | 0 |")
|
||||
lines.append("| `none` | 0 | 0 | 0 | 0 | 0 | 0 | 0 |")
|
||||
lines.extend(["", "## Next Iteration Candidates", ""])
|
||||
for item in report["next_iteration_candidates"]:
|
||||
lines.append(f"- `{item['signal']}`: {item['recommendation']}")
|
||||
@@ -328,6 +354,7 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
for event in report["recent_events"]:
|
||||
lines.append(
|
||||
f"- `{event['timestamp']}` `{event['skill']}` event=`{event['event']}` "
|
||||
f"source=`{event.get('source', 'manual')}` command=`{event.get('command', 'unknown')}` "
|
||||
f"activation=`{event['activation_type']}` outcome=`{event['outcome']}` failure=`{event['failure_type']}`"
|
||||
)
|
||||
if not report["recent_events"]:
|
||||
@@ -404,6 +431,8 @@ def main() -> None:
|
||||
parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="unknown")
|
||||
parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown")
|
||||
parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none")
|
||||
parser.add_argument("--source", choices=sorted(ALLOWED_SOURCES), default="manual")
|
||||
parser.add_argument("--command", default="unknown")
|
||||
parser.add_argument("--timestamp")
|
||||
parser.add_argument("--skill-name")
|
||||
parser.add_argument("--version")
|
||||
@@ -416,6 +445,8 @@ def main() -> None:
|
||||
"activation_type": args.activation_type,
|
||||
"outcome": args.outcome,
|
||||
"failure_type": args.failure_type,
|
||||
"source": args.source,
|
||||
"command": args.command,
|
||||
}
|
||||
if args.timestamp:
|
||||
record_event["timestamp"] = args.timestamp
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render a maintainability audit for the skill code surface."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
EXCLUDED_PARTS = {
|
||||
".git",
|
||||
".mypy_cache",
|
||||
".pytest_cache",
|
||||
"__pycache__",
|
||||
"dist",
|
||||
".previews",
|
||||
}
|
||||
|
||||
|
||||
def iter_python_files(skill_dir: Path) -> list[Path]:
|
||||
roots = [skill_dir / "scripts", skill_dir / "tests"]
|
||||
files: list[Path] = []
|
||||
for root in roots:
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in root.rglob("*.py"):
|
||||
rel_parts = path.relative_to(skill_dir).parts
|
||||
if any(part in EXCLUDED_PARTS for part in rel_parts):
|
||||
continue
|
||||
if len(rel_parts) >= 2 and rel_parts[0] == "tests" and rel_parts[1].startswith("tmp"):
|
||||
continue
|
||||
files.append(path)
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def rel(skill_dir: Path, path: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def classify_python_file(path: Path, text: str) -> str:
|
||||
if 'SCRIPT_INTERFACE = "internal-module"' in text or "SCRIPT_INTERFACE = 'internal-module'" in text:
|
||||
return "internal-module"
|
||||
if "argparse.ArgumentParser" in text or ".add_argument(" in text or "argparse" in text:
|
||||
return "cli-script"
|
||||
if path.parts[-2:-1] == ("tests",) or "/tests/" in path.as_posix():
|
||||
return "test"
|
||||
return "module"
|
||||
|
||||
|
||||
def recommendation_for(path: str) -> str:
|
||||
if path == "scripts/yao.py":
|
||||
return "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator."
|
||||
if path == "scripts/render_review_studio.py":
|
||||
return "Move data loading and large section renderers into focused review_studio_* modules."
|
||||
if path == "scripts/render_review_viewer.py":
|
||||
return "Split viewer data assembly from HTML section rendering."
|
||||
if path.startswith("tests/"):
|
||||
return "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
return "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
|
||||
|
||||
|
||||
def count_handlers_in_file(path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
return sum(1 for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.startswith("def command_"))
|
||||
|
||||
|
||||
def command_module_paths(skill_dir: Path) -> list[Path]:
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
return []
|
||||
paths = [scripts_dir / "yao.py"]
|
||||
paths.extend(sorted(scripts_dir.glob("yao_cli_*commands.py")))
|
||||
return [path for path in paths if path.exists()]
|
||||
|
||||
|
||||
def count_cli_command_handlers(skill_dir: Path) -> int:
|
||||
return sum(count_handlers_in_file(path) for path in command_module_paths(skill_dir))
|
||||
|
||||
|
||||
def build_report(skill_dir: Path, warn_lines: int, block_lines: int, trend_lines: int, generated_at: str) -> dict[str, Any]:
|
||||
files = iter_python_files(skill_dir)
|
||||
watch_lines = max(1, int(warn_lines * 0.8))
|
||||
early_watch_lines = max(1, min(trend_lines, watch_lines))
|
||||
records: list[dict[str, Any]] = []
|
||||
internal_count = 0
|
||||
cli_count = 0
|
||||
test_count = 0
|
||||
script_count = 0
|
||||
for path in files:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
line_count = len(text.splitlines())
|
||||
kind = classify_python_file(path, text)
|
||||
rel_path = rel(skill_dir, path)
|
||||
if kind == "internal-module":
|
||||
internal_count += 1
|
||||
if kind == "cli-script":
|
||||
cli_count += 1
|
||||
if rel_path.startswith("tests/"):
|
||||
test_count += 1
|
||||
if rel_path.startswith("scripts/"):
|
||||
script_count += 1
|
||||
severity = "pass"
|
||||
if line_count >= block_lines:
|
||||
severity = "block"
|
||||
elif line_count >= warn_lines:
|
||||
severity = "warn"
|
||||
early_watch = severity == "pass" and line_count >= early_watch_lines
|
||||
records.append(
|
||||
{
|
||||
"path": rel_path,
|
||||
"lines": line_count,
|
||||
"kind": kind,
|
||||
"severity": severity,
|
||||
"early_watch": early_watch,
|
||||
"recommendation": recommendation_for(rel_path),
|
||||
}
|
||||
)
|
||||
records.sort(key=lambda item: (-int(item["lines"]), str(item["path"])))
|
||||
hotspots = [item for item in records if item["severity"] in {"warn", "block"}]
|
||||
watchlist = [item for item in records if item["severity"] == "pass" and int(item["lines"]) >= watch_lines]
|
||||
early_watchlist = [
|
||||
item
|
||||
for item in records
|
||||
if item["severity"] == "pass" and int(item["lines"]) >= early_watch_lines and item not in watchlist
|
||||
]
|
||||
blockers = [item for item in records if item["severity"] == "block"]
|
||||
summary = {
|
||||
"python_file_count": len(records),
|
||||
"script_file_count": script_count,
|
||||
"test_file_count": test_count,
|
||||
"internal_module_count": internal_count,
|
||||
"cli_script_count": cli_count,
|
||||
"command_handler_count": count_cli_command_handlers(skill_dir),
|
||||
"entrypoint_command_handler_count": count_handlers_in_file(skill_dir / "scripts" / "yao.py"),
|
||||
"command_module_count": len(command_module_paths(skill_dir)),
|
||||
"warn_line_threshold": warn_lines,
|
||||
"watch_line_threshold": watch_lines,
|
||||
"early_watch_line_threshold": early_watch_lines,
|
||||
"block_line_threshold": block_lines,
|
||||
"largest_file_lines": records[0]["lines"] if records else 0,
|
||||
"watchlist_count": len(watchlist),
|
||||
"early_watchlist_count": len(early_watchlist),
|
||||
"hotspot_count": len(hotspots),
|
||||
"blocker_count": len(blockers),
|
||||
"decision": "block-maintainability" if blockers else ("watch-maintainability-hotspots" if hotspots else "pass"),
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": not blockers,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": ".",
|
||||
"summary": summary,
|
||||
"largest_files": records[:12],
|
||||
"watchlist": watchlist[:12],
|
||||
"early_watchlist": early_watchlist[:12],
|
||||
"hotspots": hotspots,
|
||||
"actions": [
|
||||
{
|
||||
"path": item["path"],
|
||||
"severity": item["severity"],
|
||||
"action": item["recommendation"],
|
||||
}
|
||||
for item in hotspots[:8]
|
||||
],
|
||||
"artifacts": {
|
||||
"json": "reports/architecture_maintainability.json",
|
||||
"markdown": "reports/architecture_maintainability.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Architecture Maintainability",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- python files: `{summary['python_file_count']}`",
|
||||
f"- scripts: `{summary['script_file_count']}`",
|
||||
f"- tests: `{summary['test_file_count']}`",
|
||||
f"- internal modules: `{summary['internal_module_count']}`",
|
||||
f"- CLI scripts: `{summary['cli_script_count']}`",
|
||||
f"- Yao CLI command handlers: `{summary['command_handler_count']}`",
|
||||
f"- entrypoint command handlers: `{summary['entrypoint_command_handler_count']}`",
|
||||
f"- command modules: `{summary['command_module_count']}`",
|
||||
f"- largest file lines: `{summary['largest_file_lines']}`",
|
||||
f"- early watch threshold lines: `{summary['early_watch_line_threshold']}`",
|
||||
f"- early watchlist: `{summary['early_watchlist_count']}`",
|
||||
f"- watch threshold lines: `{summary['watch_line_threshold']}`",
|
||||
f"- watchlist: `{summary['watchlist_count']}`",
|
||||
f"- hotspots: `{summary['hotspot_count']}`",
|
||||
f"- blockers: `{summary['blocker_count']}`",
|
||||
"",
|
||||
"This report keeps maintainability risk visible before the Meta Skill grows more gates, renderers, and CLI commands.",
|
||||
"",
|
||||
"## Hotspots",
|
||||
"",
|
||||
]
|
||||
hotspots = report.get("hotspots", [])
|
||||
if hotspots:
|
||||
lines.extend(["| File | Lines | Kind | Severity | Recommended action |", "| --- | ---: | --- | --- | --- |"])
|
||||
for item in hotspots:
|
||||
lines.append(
|
||||
f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | `{item['severity']}` | {item['recommendation']} |"
|
||||
)
|
||||
else:
|
||||
lines.append("No file-size hotspots found.")
|
||||
lines.extend(["", "## Watchlist", ""])
|
||||
watchlist = report.get("watchlist", [])
|
||||
if watchlist:
|
||||
lines.extend(["| File | Lines | Kind | Recommended next split |", "| --- | ---: | --- | --- |"])
|
||||
for item in watchlist:
|
||||
lines.append(f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | {item['recommendation']} |")
|
||||
else:
|
||||
lines.append("No near-threshold files found.")
|
||||
lines.extend(["", "## Early Watchlist", ""])
|
||||
early_watchlist = report.get("early_watchlist", [])
|
||||
if early_watchlist:
|
||||
lines.extend(["| File | Lines | Kind | Recommended next split |", "| --- | ---: | --- | --- |"])
|
||||
for item in early_watchlist:
|
||||
lines.append(f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | {item['recommendation']} |")
|
||||
else:
|
||||
lines.append("No early watch files found.")
|
||||
lines.extend(["", "## Largest Files", ""])
|
||||
if report.get("largest_files"):
|
||||
lines.extend(["| File | Lines | Kind | Severity |", "| --- | ---: | --- | --- |"])
|
||||
for item in report["largest_files"]:
|
||||
lines.append(f"| `{item['path']}` | `{item['lines']}` | `{item['kind']}` | `{item['severity']}` |")
|
||||
else:
|
||||
lines.append("No Python files found under scripts/ or tests/.")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Release Rule",
|
||||
"",
|
||||
"- `block` hotspots should be split before governed release.",
|
||||
"- `warn` hotspots can ship only when Review Studio keeps them visible and a reviewer accepts the modularization plan.",
|
||||
"- Do not split a file only for line count; split when a stable responsibility boundary is clear.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render architecture maintainability evidence for a skill package.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--warn-lines", type=int, default=900)
|
||||
parser.add_argument("--block-lines", type=int, default=1500)
|
||||
parser.add_argument("--trend-lines", type=int, default=600)
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(skill_dir, args.warn_lines, args.block_lines, args.trend_lines, args.generated_at)
|
||||
output_json = Path(args.output_json) if args.output_json else skill_dir / "reports" / "architecture_maintainability.json"
|
||||
output_md = Path(args.output_md) if args.output_md else skill_dir / "reports" / "architecture_maintainability.md"
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,685 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from benchmark_release_lock import git_status, release_lock_status
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a release-facing benchmark reproducibility manifest and Markdown report."
|
||||
|
||||
METHODOLOGY_SECTIONS = [
|
||||
"## Benchmark Types",
|
||||
"## Sample Sources",
|
||||
"## Evaluation Dimensions",
|
||||
"## Weighting Rule",
|
||||
"## Failure Disclosure",
|
||||
"## Reproduction",
|
||||
]
|
||||
|
||||
REQUIRED_ARTIFACTS = [
|
||||
("methodology", "reports/benchmark_methodology.md"),
|
||||
("failure_disclosure", "evals/failure-cases.md"),
|
||||
("output_cases", "evals/output/cases.jsonl"),
|
||||
("output_schema", "evals/output/schema.json"),
|
||||
("output_scorecard", "reports/output_quality_scorecard.json"),
|
||||
("output_execution", "reports/output_execution_runs.json"),
|
||||
("blind_review", "reports/output_blind_review_pack.json"),
|
||||
("review_adjudication", "reports/output_review_adjudication.json"),
|
||||
("trigger_scorecard", "reports/route_scorecard.json"),
|
||||
("runtime_conformance", "reports/conformance_matrix.json"),
|
||||
("trust_report", "reports/security_trust_report.json"),
|
||||
("python_compatibility", "reports/python_compatibility.json"),
|
||||
("registry_audit", "reports/registry_audit.json"),
|
||||
("package_verification", "reports/package_verification.json"),
|
||||
("install_simulation", "reports/install_simulation.json"),
|
||||
("skill_os2_audit", "reports/skill_os2_audit.json"),
|
||||
("world_class_evidence_plan", "reports/world_class_evidence_plan.json"),
|
||||
("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"),
|
||||
("world_class_evidence_intake", "reports/world_class_evidence_intake.json"),
|
||||
("world_class_evidence_preflight", "reports/world_class_evidence_preflight.json"),
|
||||
("world_class_submission_review", "reports/world_class_submission_review.json"),
|
||||
("world_class_operator_runbook", "reports/world_class_operator_runbook.json"),
|
||||
("world_class_operator_runbook_markdown", "reports/world_class_operator_runbook.md"),
|
||||
("world_class_operator_runbook_html", "reports/world_class_operator_runbook.html"),
|
||||
("world_class_claim_guard", "reports/world_class_claim_guard.json"),
|
||||
]
|
||||
|
||||
REPRODUCTION_COMMANDS = [
|
||||
{
|
||||
"label": "source commit",
|
||||
"command": "git rev-parse HEAD",
|
||||
"evidence": "git commit hash",
|
||||
},
|
||||
{
|
||||
"label": "trigger eval",
|
||||
"command": "make eval-suite",
|
||||
"evidence": "reports/eval_suite.json",
|
||||
},
|
||||
{
|
||||
"label": "output eval",
|
||||
"command": "python3 scripts/yao.py output-eval",
|
||||
"evidence": "reports/output_quality_scorecard.json",
|
||||
},
|
||||
{
|
||||
"label": "output execution",
|
||||
"command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'",
|
||||
"evidence": "reports/output_execution_runs.json",
|
||||
},
|
||||
{
|
||||
"label": "blind review adjudication",
|
||||
"command": "python3 scripts/yao.py output-review",
|
||||
"evidence": "reports/output_review_adjudication.json",
|
||||
},
|
||||
{
|
||||
"label": "skill ir",
|
||||
"command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json",
|
||||
"evidence": "skill-ir/examples/yao-meta-skill.json",
|
||||
},
|
||||
{
|
||||
"label": "runtime conformance",
|
||||
"command": "python3 scripts/yao.py conformance .",
|
||||
"evidence": "reports/conformance_matrix.json",
|
||||
},
|
||||
{
|
||||
"label": "trust report",
|
||||
"command": "python3 scripts/yao.py trust .",
|
||||
"evidence": "reports/security_trust_report.json",
|
||||
},
|
||||
{
|
||||
"label": "python compatibility",
|
||||
"command": "python3 scripts/yao.py python-compat .",
|
||||
"evidence": "reports/python_compatibility.json",
|
||||
},
|
||||
{
|
||||
"label": "package",
|
||||
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip",
|
||||
"evidence": "dist/yao-meta-skill.zip",
|
||||
},
|
||||
{
|
||||
"label": "package verify",
|
||||
"command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip",
|
||||
"evidence": "reports/package_verification.json",
|
||||
},
|
||||
{
|
||||
"label": "install simulate",
|
||||
"command": "python3 scripts/yao.py install-simulate . --package-dir dist",
|
||||
"evidence": "reports/install_simulation.json",
|
||||
},
|
||||
{
|
||||
"label": "registry audit",
|
||||
"command": "python3 scripts/yao.py registry-audit .",
|
||||
"evidence": "reports/registry_audit.json",
|
||||
},
|
||||
{
|
||||
"label": "skill os audit",
|
||||
"command": "python3 scripts/yao.py skill-os2-audit .",
|
||||
"evidence": "reports/skill_os2_audit.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class evidence plan",
|
||||
"command": "python3 scripts/yao.py world-class-evidence .",
|
||||
"evidence": "reports/world_class_evidence_plan.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class evidence ledger",
|
||||
"command": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions",
|
||||
"evidence": "reports/world_class_evidence_ledger.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class evidence intake",
|
||||
"command": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
|
||||
"evidence": "reports/world_class_evidence_intake.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class evidence preflight",
|
||||
"command": "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions",
|
||||
"evidence": "reports/world_class_evidence_preflight.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class submission review",
|
||||
"command": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
|
||||
"evidence": "reports/world_class_submission_review.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class operator runbook",
|
||||
"command": "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions",
|
||||
"evidence": "reports/world_class_operator_runbook.json",
|
||||
},
|
||||
{
|
||||
"label": "world-class claim guard",
|
||||
"command": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
"evidence": "reports/world_class_claim_guard.json",
|
||||
},
|
||||
{
|
||||
"label": "evidence consistency",
|
||||
"command": "python3 scripts/yao.py evidence-consistency .",
|
||||
"evidence": "reports/evidence_consistency.json",
|
||||
},
|
||||
{
|
||||
"label": "full ci",
|
||||
"command": "make ci-test",
|
||||
"evidence": "CI target output",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def git_commit(skill_dir: Path) -> str:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["git", "rev-parse", "HEAD"],
|
||||
cwd=skill_dir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return "unknown"
|
||||
return proc.stdout.strip() or "unknown"
|
||||
|
||||
|
||||
def count_jsonl(path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip())
|
||||
|
||||
|
||||
def count_failure_cases(path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.startswith("### "))
|
||||
|
||||
|
||||
def methodology_check(path: Path) -> dict[str, Any]:
|
||||
text = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
sections = [{"heading": heading, "exists": heading in text} for heading in METHODOLOGY_SECTIONS]
|
||||
return {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
"exists": path.exists(),
|
||||
"sections": sections,
|
||||
"missing_sections": [item["heading"] for item in sections if not item["exists"]],
|
||||
}
|
||||
|
||||
|
||||
def artifact_record(skill_dir: Path, label: str, rel: str) -> dict[str, Any]:
|
||||
path = skill_dir / rel
|
||||
record: dict[str, Any] = {
|
||||
"label": label,
|
||||
"path": rel,
|
||||
"exists": path.exists(),
|
||||
}
|
||||
if path.exists() and path.is_file():
|
||||
record["bytes"] = path.stat().st_size
|
||||
record["sha256"] = sha256_file(path)
|
||||
return record
|
||||
|
||||
|
||||
def evidence_bundle_fingerprint(artifacts: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
digest = hashlib.sha256()
|
||||
existing_count = 0
|
||||
missing_paths = []
|
||||
for artifact in sorted(artifacts, key=lambda item: str(item.get("path", ""))):
|
||||
path = str(artifact.get("path", ""))
|
||||
digest.update(path.encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(str(artifact.get("label", "")).encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(str(bool(artifact.get("exists"))).encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
digest.update(str(artifact.get("sha256", "")).encode("utf-8"))
|
||||
digest.update(b"\0")
|
||||
if artifact.get("exists"):
|
||||
existing_count += 1
|
||||
else:
|
||||
missing_paths.append(path)
|
||||
return {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
"artifact_count": len(artifacts),
|
||||
"existing_count": existing_count,
|
||||
"missing_count": len(missing_paths),
|
||||
"missing_paths": missing_paths,
|
||||
"sha256": digest.hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def public_claim_blockers(
|
||||
local_reproducibility_ready: bool,
|
||||
release_lock_ready: bool,
|
||||
provider_evidence_complete: bool,
|
||||
human_review_complete: bool,
|
||||
world_class_ready: bool,
|
||||
world_class_open_gap_count: int,
|
||||
world_class_ledger_pending_count: int,
|
||||
world_class_source_check_count: int,
|
||||
world_class_source_pass_count: int,
|
||||
world_class_source_blocked_count: int,
|
||||
) -> list[str]:
|
||||
blockers = []
|
||||
if not local_reproducibility_ready:
|
||||
blockers.append("local benchmark reproducibility is incomplete")
|
||||
if not release_lock_ready:
|
||||
blockers.append("release lock is not clean or commit is unavailable")
|
||||
if not provider_evidence_complete:
|
||||
blockers.append("provider-backed model holdout evidence is incomplete")
|
||||
if not human_review_complete:
|
||||
blockers.append("human blind-review adjudication is incomplete")
|
||||
if not world_class_ready:
|
||||
blockers.append(
|
||||
f"world-class evidence is not accepted yet ({world_class_open_gap_count} open gaps, "
|
||||
f"{world_class_ledger_pending_count} ledger pending)"
|
||||
)
|
||||
if (
|
||||
world_class_source_check_count == 0
|
||||
or world_class_source_pass_count != world_class_source_check_count
|
||||
or world_class_source_blocked_count > 0
|
||||
):
|
||||
blockers.append(
|
||||
"world-class source checks are not all accepted "
|
||||
f"({world_class_source_pass_count}/{world_class_source_check_count} pass, "
|
||||
f"{world_class_source_blocked_count} blocked)"
|
||||
)
|
||||
return blockers
|
||||
|
||||
|
||||
def beta_test_blockers(
|
||||
local_reproducibility_ready: bool,
|
||||
release_lock_ready: bool,
|
||||
provider_evidence_complete: bool,
|
||||
) -> list[str]:
|
||||
blockers = []
|
||||
if not local_reproducibility_ready:
|
||||
blockers.append("local benchmark reproducibility is incomplete")
|
||||
if not release_lock_ready:
|
||||
blockers.append("release lock is not clean or commit is unavailable")
|
||||
if not provider_evidence_complete:
|
||||
blockers.append("provider-backed model holdout source evidence is incomplete")
|
||||
return blockers
|
||||
|
||||
|
||||
BETA_DEFERRED_EVIDENCE = {
|
||||
"provider-holdout": {
|
||||
"label": "Provider holdout ledger review",
|
||||
"reason": "Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims.",
|
||||
},
|
||||
"human-adjudication": {
|
||||
"label": "Human blind-review adjudication",
|
||||
"reason": "Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims.",
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"label": "Native permission enforcement evidence",
|
||||
"reason": "Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims.",
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"label": "Real client telemetry evidence",
|
||||
"reason": "Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def beta_deferred_evidence(world_class_ledger: dict[str, Any]) -> list[dict[str, str]]:
|
||||
deferred = []
|
||||
entries = world_class_ledger.get("entries", [])
|
||||
if not isinstance(entries, list):
|
||||
return deferred
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict) or entry.get("status") != "pending":
|
||||
continue
|
||||
key = str(entry.get("key") or "")
|
||||
if not key:
|
||||
continue
|
||||
fallback = BETA_DEFERRED_EVIDENCE.get(key, {})
|
||||
deferred.append(
|
||||
{
|
||||
"key": key,
|
||||
"label": str(entry.get("label") or fallback.get("label") or key),
|
||||
"reason": str(
|
||||
fallback.get("reason")
|
||||
or "Formal evidence is still pending; deferred for beta/public testing and still required before public claims."
|
||||
),
|
||||
}
|
||||
)
|
||||
return deferred
|
||||
|
||||
|
||||
def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
reports = skill_dir / "reports"
|
||||
output_quality = load_json(reports / "output_quality_scorecard.json")
|
||||
output_execution = load_json(reports / "output_execution_runs.json")
|
||||
output_review = load_json(reports / "output_review_adjudication.json")
|
||||
skill_os2 = load_json(reports / "skill_os2_audit.json")
|
||||
world_class_plan = load_json(reports / "world_class_evidence_plan.json")
|
||||
world_class_ledger = load_json(reports / "world_class_evidence_ledger.json")
|
||||
trust = load_json(reports / "security_trust_report.json")
|
||||
package_verification = load_json(reports / "package_verification.json")
|
||||
methodology = methodology_check(reports / "benchmark_methodology.md")
|
||||
artifacts = [artifact_record(skill_dir, label, rel) for label, rel in REQUIRED_ARTIFACTS]
|
||||
evidence_bundle = evidence_bundle_fingerprint(artifacts)
|
||||
missing_artifacts = [item["path"] for item in artifacts if not item["exists"]]
|
||||
output_summary = output_quality.get("summary", {})
|
||||
execution_summary = output_execution.get("summary", {})
|
||||
review_summary = output_review.get("summary", {})
|
||||
failure_case_count = count_failure_cases(skill_dir / "evals" / "failure-cases.md")
|
||||
output_case_count = count_jsonl(skill_dir / "evals" / "output" / "cases.jsonl")
|
||||
status = git_status(skill_dir)
|
||||
commit = git_commit(skill_dir)
|
||||
release_lock = release_lock_status(status, commit)
|
||||
local_reproducibility_ready = (
|
||||
not methodology["missing_sections"]
|
||||
and not missing_artifacts
|
||||
and output_case_count >= 5
|
||||
and failure_case_count > 0
|
||||
and output_summary.get("gate_pass") is True
|
||||
and execution_summary.get("command_executed_count", 0) > 0
|
||||
and execution_summary.get("timing_observed_count", 0) > 0
|
||||
)
|
||||
human_review_complete = review_summary.get("pair_count", 0) > 0 and review_summary.get("pending_count", 0) == 0
|
||||
provider_evidence_complete = execution_summary.get("model_executed_count", 0) > 0 and execution_summary.get("token_observed_count", 0) > 0
|
||||
world_class_ready = bool(skill_os2.get("summary", {}).get("world_class_ready", False))
|
||||
world_class_open_gap_count = int(skill_os2.get("summary", {}).get("open_gap_count", 0) or 0)
|
||||
world_class_summary = world_class_ledger.get("summary", {})
|
||||
world_class_ledger_pending_count = int(world_class_summary.get("pending_count", 0) or 0)
|
||||
world_class_source_check_count = int(world_class_summary.get("source_check_count", 0) or 0)
|
||||
world_class_source_pass_count = int(world_class_summary.get("source_pass_count", 0) or 0)
|
||||
world_class_source_blocked_count = int(world_class_summary.get("source_blocked_count", 0) or 0)
|
||||
claim_blockers = public_claim_blockers(
|
||||
local_reproducibility_ready,
|
||||
release_lock["ready"],
|
||||
provider_evidence_complete,
|
||||
human_review_complete,
|
||||
world_class_ready,
|
||||
world_class_open_gap_count,
|
||||
world_class_ledger_pending_count,
|
||||
world_class_source_check_count,
|
||||
world_class_source_pass_count,
|
||||
world_class_source_blocked_count,
|
||||
)
|
||||
public_claim_ready = not claim_blockers
|
||||
beta_blockers = beta_test_blockers(
|
||||
local_reproducibility_ready,
|
||||
release_lock["ready"],
|
||||
provider_evidence_complete,
|
||||
)
|
||||
beta_deferred = beta_deferred_evidence(world_class_ledger)
|
||||
beta_test_ready = not beta_blockers
|
||||
limitations = [
|
||||
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
|
||||
"Pending blind-review decisions are visible but do not count as human adjudication.",
|
||||
"World-class readiness remains false until external and human evidence gaps close.",
|
||||
"Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims.",
|
||||
]
|
||||
if provider_evidence_complete:
|
||||
limitations.insert(
|
||||
1,
|
||||
"Provider-backed model holdout source evidence is complete, but ledger acceptance still requires a valid independently reviewed submission packet.",
|
||||
)
|
||||
else:
|
||||
limitations.insert(
|
||||
1,
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
)
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": local_reproducibility_ready,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"commit": commit,
|
||||
"git_status": status,
|
||||
"summary": {
|
||||
"reproducibility_ready": local_reproducibility_ready,
|
||||
"release_lock_ready": release_lock["ready"],
|
||||
"methodology_complete": not methodology["missing_sections"],
|
||||
"required_artifact_count": len(artifacts),
|
||||
"missing_artifact_count": len(missing_artifacts),
|
||||
"evidence_bundle_sha256": evidence_bundle["sha256"],
|
||||
"source_contract_sha256": trust.get("summary", {}).get("package_sha256", ""),
|
||||
"archive_sha256": package_verification.get("summary", {}).get("archive_sha256", ""),
|
||||
"output_case_count": output_case_count,
|
||||
"failure_disclosure_count": failure_case_count,
|
||||
"command_count": len(REPRODUCTION_COMMANDS),
|
||||
"command_executed_count": execution_summary.get("command_executed_count", 0),
|
||||
"timing_observed_count": execution_summary.get("timing_observed_count", 0),
|
||||
"model_executed_count": execution_summary.get("model_executed_count", 0),
|
||||
"token_observed_count": execution_summary.get("token_observed_count", 0),
|
||||
"human_review_complete": human_review_complete,
|
||||
"provider_evidence_complete": provider_evidence_complete,
|
||||
"world_class_ready": world_class_ready,
|
||||
"world_class_open_gap_count": world_class_open_gap_count,
|
||||
"world_class_task_count": world_class_plan.get("summary", {}).get("task_count", 0),
|
||||
"world_class_ledger_pending_count": world_class_ledger_pending_count,
|
||||
"world_class_source_check_count": world_class_source_check_count,
|
||||
"world_class_source_pass_count": world_class_source_pass_count,
|
||||
"world_class_source_blocked_count": world_class_source_blocked_count,
|
||||
"beta_test_ready": beta_test_ready,
|
||||
"beta_test_blocker_count": len(beta_blockers),
|
||||
"beta_test_deferred_evidence_count": len(beta_deferred),
|
||||
"public_claim_ready": public_claim_ready,
|
||||
"public_claim_blocker_count": len(claim_blockers),
|
||||
"working_tree_dirty": status.get("dirty"),
|
||||
"changed_file_count": status.get("changed_file_count"),
|
||||
"source_tree_dirty": status.get("source_dirty"),
|
||||
"source_changed_file_count": status.get("source_changed_file_count"),
|
||||
"generated_tree_dirty": status.get("generated_dirty"),
|
||||
"generated_changed_file_count": status.get("generated_changed_file_count"),
|
||||
},
|
||||
"beta_test_release": {
|
||||
"ready": beta_test_ready,
|
||||
"scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
|
||||
"blockers": beta_blockers,
|
||||
"allowed_deferred_evidence": beta_deferred,
|
||||
"policy": "Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted.",
|
||||
"required_wording": "Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline.",
|
||||
},
|
||||
"public_claim": {
|
||||
"ready": public_claim_ready,
|
||||
"scope": "public benchmark or world-class readiness claim",
|
||||
"blockers": claim_blockers,
|
||||
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks.",
|
||||
},
|
||||
"release_lock": release_lock,
|
||||
"evidence_bundle": evidence_bundle,
|
||||
"methodology": methodology,
|
||||
"artifacts_checked": artifacts,
|
||||
"missing_artifacts": missing_artifacts,
|
||||
"reproduction_commands": REPRODUCTION_COMMANDS,
|
||||
"failure_disclosure": {
|
||||
"path": "evals/failure-cases.md",
|
||||
"case_count": failure_case_count,
|
||||
"policy": "Keep representative failures visible and tied to regression checks.",
|
||||
},
|
||||
"limitations": limitations,
|
||||
"artifacts": {
|
||||
"json": "reports/benchmark_reproducibility.json",
|
||||
"markdown": "reports/benchmark_reproducibility.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Benchmark Reproducibility",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
f"Commit: `{report['commit']}`",
|
||||
f"Working tree dirty at generation: `{str(summary.get('working_tree_dirty')).lower()}`",
|
||||
f"Source tree dirty at generation: `{str(summary.get('source_tree_dirty')).lower()}`",
|
||||
f"Generated evidence dirty at generation: `{str(summary.get('generated_tree_dirty')).lower()}`",
|
||||
f"Evidence bundle SHA256: `{summary.get('evidence_bundle_sha256', '')}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- reproducibility ready: `{str(summary['reproducibility_ready']).lower()}`",
|
||||
f"- release lock ready: `{str(summary.get('release_lock_ready')).lower()}`",
|
||||
f"- methodology complete: `{str(summary['methodology_complete']).lower()}`",
|
||||
f"- required artifacts: `{summary['required_artifact_count']}`",
|
||||
f"- missing artifacts: `{summary['missing_artifact_count']}`",
|
||||
f"- source contract sha256: `{summary.get('source_contract_sha256', '')[:12]}`",
|
||||
f"- archive sha256: `{summary.get('archive_sha256', '')[:12]}`",
|
||||
f"- output cases: `{summary['output_case_count']}`",
|
||||
f"- disclosed failure cases: `{summary['failure_disclosure_count']}`",
|
||||
f"- reproduction commands: `{summary['command_count']}`",
|
||||
f"- provider evidence complete: `{str(summary['provider_evidence_complete']).lower()}`",
|
||||
f"- human review complete: `{str(summary['human_review_complete']).lower()}`",
|
||||
f"- world-class ready: `{str(summary['world_class_ready']).lower()}`",
|
||||
f"- world-class source checks: `{summary.get('world_class_source_pass_count', 0)}` pass / `{summary.get('world_class_source_check_count', 0)}` total; `{summary.get('world_class_source_blocked_count', 0)}` blocked",
|
||||
f"- beta test ready: `{str(summary['beta_test_ready']).lower()}`",
|
||||
f"- beta test blockers: `{summary['beta_test_blocker_count']}`",
|
||||
f"- beta deferred evidence: `{summary['beta_test_deferred_evidence_count']}`",
|
||||
f"- public claim ready: `{str(summary['public_claim_ready']).lower()}`",
|
||||
f"- public claim blockers: `{summary['public_claim_blocker_count']}`",
|
||||
f"- changed files at generation: `{summary.get('changed_file_count')}`",
|
||||
f"- source changed files at generation: `{summary.get('source_changed_file_count')}`",
|
||||
f"- generated changed files at generation: `{summary.get('generated_changed_file_count')}`",
|
||||
"",
|
||||
"This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.",
|
||||
"",
|
||||
]
|
||||
beta_release = report.get("beta_test_release", {})
|
||||
lines.extend(
|
||||
[
|
||||
"## Beta Test Boundary",
|
||||
"",
|
||||
f"- ready: `{str(beta_release.get('ready')).lower()}`",
|
||||
f"- scope: {beta_release.get('scope', 'beta/public test release')}",
|
||||
f"- policy: {beta_release.get('policy', '')}",
|
||||
f"- required wording: {beta_release.get('required_wording', '')}",
|
||||
"",
|
||||
"| Blocker |",
|
||||
"| --- |",
|
||||
]
|
||||
)
|
||||
beta_blockers = beta_release.get("blockers", [])
|
||||
if beta_blockers:
|
||||
lines.extend(f"| {item} |" for item in beta_blockers)
|
||||
else:
|
||||
lines.append("| none |")
|
||||
lines.extend(["", "| Deferred evidence | Reason |", "| --- | --- |"])
|
||||
deferred = beta_release.get("allowed_deferred_evidence", [])
|
||||
if deferred:
|
||||
lines.extend(f"| `{item.get('key', '')}` | {item.get('reason', '')} |" for item in deferred)
|
||||
else:
|
||||
lines.append("| none | none |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Public Claim Boundary",
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`",
|
||||
f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}",
|
||||
f"- policy: {report.get('public_claim', {}).get('policy', '')}",
|
||||
"",
|
||||
"| Blocker |",
|
||||
"| --- |",
|
||||
]
|
||||
)
|
||||
blockers = report.get("public_claim", {}).get("blockers", [])
|
||||
if blockers:
|
||||
lines.extend(f"| {item} |" for item in blockers)
|
||||
else:
|
||||
lines.append("| none |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Release Lock",
|
||||
"",
|
||||
f"- ready: `{str(report.get('release_lock', {}).get('ready')).lower()}`",
|
||||
f"- reason: {report.get('release_lock', {}).get('reason', 'unknown')}",
|
||||
f"- status scope: {report.get('release_lock', {}).get('status_scope', 'generation-time status')}",
|
||||
"",
|
||||
"## Evidence Bundle",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
f"- algorithm: `{report.get('evidence_bundle', {}).get('algorithm', '')}`",
|
||||
f"- artifacts: `{report.get('evidence_bundle', {}).get('existing_count', 0)}` / `{report.get('evidence_bundle', {}).get('artifact_count', 0)}`",
|
||||
f"- sha256: `{report.get('evidence_bundle', {}).get('sha256', '')}`",
|
||||
"",
|
||||
"## Methodology Sections",
|
||||
"",
|
||||
"| Section | Status |",
|
||||
"| --- | --- |",
|
||||
]
|
||||
)
|
||||
for section in report["methodology"]["sections"]:
|
||||
lines.append(f"| `{section['heading']}` | {'present' if section['exists'] else 'missing'} |")
|
||||
lines.extend(["", "## Required Artifacts", "", "| Label | Path | Status | SHA256 |", "| --- | --- | --- | --- |"])
|
||||
for artifact in report["artifacts_checked"]:
|
||||
digest = artifact.get("sha256", "")
|
||||
lines.append(
|
||||
f"| {artifact['label']} | `{artifact['path']}` | {'present' if artifact['exists'] else 'missing'} | `{digest[:12]}` |"
|
||||
)
|
||||
lines.extend(["", "## Reproduction Commands", ""])
|
||||
for command in report["reproduction_commands"]:
|
||||
lines.append(f"- `{command['command']}`")
|
||||
lines.append(f" - evidence: `{command['evidence']}`")
|
||||
lines.extend(["", "## Failure Disclosure", ""])
|
||||
disclosure = report["failure_disclosure"]
|
||||
lines.append(f"- path: `{disclosure['path']}`")
|
||||
lines.append(f"- disclosed cases: `{disclosure['case_count']}`")
|
||||
lines.append(f"- policy: {disclosure['policy']}")
|
||||
lines.extend(["", "## Limits", ""])
|
||||
lines.extend(f"- {item}" for item in report["limitations"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render benchmark reproducibility evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json", default="reports/benchmark_reproducibility.json")
|
||||
parser.add_argument("--output-md", default="reports/benchmark_reproducibility.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(skill_dir, args.generated_at)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -52,6 +52,9 @@ def main() -> None:
|
||||
"budget_limit": stats.get("context_budget_limit"),
|
||||
"initial_tokens": stats.get("estimated_initial_load_tokens"),
|
||||
"skill_body_tokens": stats.get("skill_body_tokens"),
|
||||
"deferred_resource_tokens": stats.get("deferred_resource_tokens"),
|
||||
"large_deferred_resource_dirs": stats.get("large_deferred_resource_dirs", []),
|
||||
"deferred_resource_governance": stats.get("deferred_resource_governance", {}),
|
||||
"quality_density": stats.get("quality_density"),
|
||||
"unused_resource_dirs": stats.get("unused_resource_dirs", []),
|
||||
"ok": report.get("ok"),
|
||||
@@ -72,14 +75,20 @@ def main() -> None:
|
||||
lines = [
|
||||
"# Context Budget Summary",
|
||||
"",
|
||||
"| Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status |",
|
||||
"| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |",
|
||||
"| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |",
|
||||
"| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |",
|
||||
]
|
||||
for row in rows:
|
||||
unused = ", ".join(row["unused_resource_dirs"]) if row["unused_resource_dirs"] else "-"
|
||||
governance = row.get("deferred_resource_governance", {}) or {}
|
||||
governance_status = governance.get("status", "unknown")
|
||||
large_dirs = ", ".join(
|
||||
f"{item['path']}:{item['estimated_tokens']}"
|
||||
for item in row.get("large_deferred_resource_dirs", [])
|
||||
) or "-"
|
||||
status = "ok" if row["ok"] else "fail"
|
||||
lines.append(
|
||||
f"| {row['label']} | `{row['path']}` | `{row['budget_tier']}` | {row['budget_limit']} | {row['initial_tokens']} | {row['skill_body_tokens']} | {row['quality_density']} | {unused} | {status} |"
|
||||
f"| {row['label']} | `{row['path']}` | `{row['budget_tier']}` | {row['budget_limit']} | {row['initial_tokens']} | {row['skill_body_tokens']} | {row['deferred_resource_tokens']} | `{governance_status}` | {large_dirs} | {row['quality_density']} | {unused} | {status} |"
|
||||
)
|
||||
lines.extend(["", "Per-target JSON reports are written beside each target and under `reports/`."])
|
||||
(ROOT / "reports" / "context_budget.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import Any
|
||||
|
||||
from propose_adaptation import build_report as build_proposal_report
|
||||
from propose_adaptation import render_markdown as render_proposals_markdown
|
||||
from skillops_opportunity import build_opportunities, decision_policy_contract, summarize_opportunities
|
||||
from summarize_user_signals import build_report as build_pattern_report
|
||||
from summarize_user_signals import render_markdown as render_patterns_markdown
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a Daily SkillOps report that summarizes explicit-source patterns, proposals, approval state, and release evidence without scanning private logs or applying patches."
|
||||
|
||||
SUMMARY_FIELDS = [
|
||||
"decision",
|
||||
"source_supplied",
|
||||
"pattern_count",
|
||||
"proposal_count",
|
||||
"approval_count",
|
||||
"pending_review_count",
|
||||
"applied_count",
|
||||
"rollback_count",
|
||||
"local_blueprint_ready",
|
||||
"public_world_class_ready",
|
||||
"world_class_pending_count",
|
||||
"release_lock_ready",
|
||||
"evidence_consistency_ok",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
"failure_count",
|
||||
]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def report_date(generated_at: str) -> str:
|
||||
if len(generated_at) >= 10:
|
||||
candidate = generated_at[:10]
|
||||
try:
|
||||
date.fromisoformat(candidate)
|
||||
return candidate
|
||||
except ValueError:
|
||||
pass
|
||||
return date.today().isoformat()
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return f"[external-explicit-source]/{path.name}"
|
||||
|
||||
|
||||
def resolve_output(skill_dir: Path, value: str) -> Path:
|
||||
path = Path(value)
|
||||
return path if path.is_absolute() else skill_dir / path
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def default_daily_path(skill_dir: Path, generated_at: str, suffix: str) -> Path:
|
||||
return skill_dir / "reports" / "skillops" / "daily" / f"{report_date(generated_at)}.{suffix}"
|
||||
|
||||
|
||||
def source_summary(patterns: dict[str, Any]) -> dict[str, Any]:
|
||||
source = patterns.get("source", {}) if isinstance(patterns.get("source"), dict) else {}
|
||||
privacy = patterns.get("privacy_contract", {}) if isinstance(patterns.get("privacy_contract"), dict) else {}
|
||||
return {
|
||||
"path": source.get("path", ""),
|
||||
"fingerprint_sha256": source.get("fingerprint_sha256", ""),
|
||||
"record_count": source.get("record_count", 0),
|
||||
"explicit_source": privacy.get("explicit_source_required") is True,
|
||||
"raw_content_stored": privacy.get("raw_content_stored") is True,
|
||||
"redacted_excerpts_only": privacy.get("redacted_excerpts_only") is True,
|
||||
}
|
||||
|
||||
|
||||
def compact_proposals(proposals: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
rows = []
|
||||
for item in proposals.get("proposals", []):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"proposal_id": item.get("proposal_id", ""),
|
||||
"pattern_id": item.get("pattern_id", ""),
|
||||
"title": item.get("title", ""),
|
||||
"risk_level": item.get("risk_level", ""),
|
||||
"status": item.get("status", ""),
|
||||
"requires_approval": item.get("requires_approval") is True,
|
||||
"target_files": item.get("target_files", []),
|
||||
"verification_commands": item.get("verification_commands", []),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def build_actions(summary: dict[str, Any], proposal_rows: list[dict[str, Any]]) -> list[dict[str, str]]:
|
||||
actions: list[dict[str, str]] = []
|
||||
if summary["source_supplied"] is False and summary["pattern_count"] == 0:
|
||||
actions.append(
|
||||
{
|
||||
"key": "provide-explicit-source",
|
||||
"priority": "medium",
|
||||
"action": "Run daily-skillops with --source pointing to a curated JSONL or Markdown conversation summary.",
|
||||
}
|
||||
)
|
||||
if proposal_rows:
|
||||
actions.append(
|
||||
{
|
||||
"key": "review-adaptation-proposals",
|
||||
"priority": "high",
|
||||
"action": "Review proposal-only adaptation items before preparing any approval ledger entry.",
|
||||
}
|
||||
)
|
||||
if summary["pending_review_count"]:
|
||||
actions.append(
|
||||
{
|
||||
"key": "resolve-pending-approvals",
|
||||
"priority": "high",
|
||||
"action": "Approve, reject, or expire pending adaptive approval ledger entries before applying patches.",
|
||||
}
|
||||
)
|
||||
if summary["world_class_pending_count"]:
|
||||
actions.append(
|
||||
{
|
||||
"key": "close-world-class-evidence",
|
||||
"priority": "high",
|
||||
"action": "Collect accepted external or human evidence for the pending world-class ledger entries.",
|
||||
}
|
||||
)
|
||||
if summary["evidence_consistency_ok"] is not True:
|
||||
actions.append(
|
||||
{
|
||||
"key": "refresh-evidence-consistency",
|
||||
"priority": "high",
|
||||
"action": "Regenerate evidence-consistency before using this report for release decisions.",
|
||||
}
|
||||
)
|
||||
if not actions:
|
||||
actions.append(
|
||||
{
|
||||
"key": "monitor",
|
||||
"priority": "low",
|
||||
"action": "No action required beyond routine daily monitoring.",
|
||||
}
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
def build_report(
|
||||
skill_dir: Path,
|
||||
generated_at: str,
|
||||
source: Path | None = None,
|
||||
min_support: int = 2,
|
||||
allow_history_source: bool = False,
|
||||
patterns_json: Path | None = None,
|
||||
proposals_json: Path | None = None,
|
||||
write_source_reports: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
reports = skill_dir / "reports"
|
||||
patterns_path = resolve_output(skill_dir, str(patterns_json or "reports/user_patterns.json"))
|
||||
proposals_path = resolve_output(skill_dir, str(proposals_json or "reports/adaptation_proposals.json"))
|
||||
failures: list[str] = []
|
||||
|
||||
if source is not None:
|
||||
pattern_report = build_pattern_report(
|
||||
skill_dir,
|
||||
source,
|
||||
min_support=max(2, min_support),
|
||||
generated_at=generated_at,
|
||||
allow_history_source=allow_history_source,
|
||||
)
|
||||
if pattern_report.get("ok") is True and write_source_reports:
|
||||
pattern_report["artifacts"] = {
|
||||
"json": display_path(patterns_path, skill_dir),
|
||||
"markdown": display_path(patterns_path.with_suffix(".md"), skill_dir),
|
||||
}
|
||||
write_json(patterns_path, pattern_report)
|
||||
write_text(patterns_path.with_suffix(".md"), render_patterns_markdown(pattern_report))
|
||||
else:
|
||||
failures.extend(pattern_report.get("failures", []))
|
||||
else:
|
||||
pattern_report = load_json(patterns_path)
|
||||
|
||||
if pattern_report and pattern_report.get("ok") is True:
|
||||
proposal_source_path = patterns_path
|
||||
with TemporaryDirectory() as temp_dir:
|
||||
if source is not None and not write_source_reports:
|
||||
proposal_source_path = Path(temp_dir) / "user_patterns.json"
|
||||
write_json(proposal_source_path, pattern_report)
|
||||
proposal_report = build_proposal_report(skill_dir, proposal_source_path, generated_at)
|
||||
if proposal_report.get("ok") is True and write_source_reports:
|
||||
proposal_report["artifacts"] = {
|
||||
"json": display_path(proposals_path, skill_dir),
|
||||
"markdown": display_path(proposals_path.with_suffix(".md"), skill_dir),
|
||||
}
|
||||
write_json(proposals_path, proposal_report)
|
||||
write_text(proposals_path.with_suffix(".md"), render_proposals_markdown(proposal_report))
|
||||
elif proposal_report.get("ok") is not True:
|
||||
failures.extend(proposal_report.get("failures", []))
|
||||
else:
|
||||
proposal_report = load_json(proposals_path)
|
||||
if source is not None and not pattern_report:
|
||||
failures.append("Pattern report could not be built from the explicit source.")
|
||||
|
||||
approval = load_json(reports / "adaptation_approval_ledger.json")
|
||||
regression = load_json(reports / "adaptation_regression_report.json")
|
||||
coverage = load_json(reports / "skill_os2_coverage.json")
|
||||
ledger = load_json(reports / "world_class_evidence_ledger.json")
|
||||
consistency = load_json(reports / "evidence_consistency.json")
|
||||
benchmark = load_json(reports / "benchmark_reproducibility.json")
|
||||
|
||||
pattern_summary = pattern_report.get("summary", {}) if isinstance(pattern_report.get("summary"), dict) else {}
|
||||
proposal_summary = proposal_report.get("summary", {}) if isinstance(proposal_report.get("summary"), dict) else {}
|
||||
approval_summary = approval.get("summary", {}) if isinstance(approval.get("summary"), dict) else {}
|
||||
regression_summary = regression.get("summary", {}) if isinstance(regression.get("summary"), dict) else {}
|
||||
coverage_summary = coverage.get("summary", {}) if isinstance(coverage.get("summary"), dict) else {}
|
||||
ledger_summary = ledger.get("summary", {}) if isinstance(ledger.get("summary"), dict) else {}
|
||||
benchmark_summary = benchmark.get("summary", {}) if isinstance(benchmark.get("summary"), dict) else {}
|
||||
proposal_contract = proposal_report.get("proposal_contract", {}) if isinstance(proposal_report.get("proposal_contract"), dict) else {}
|
||||
|
||||
proposal_rows = compact_proposals(proposal_report)
|
||||
pattern_rows = pattern_report.get("patterns", []) if isinstance(pattern_report.get("patterns"), list) else []
|
||||
opportunities = build_opportunities(pattern_rows, proposal_rows)
|
||||
opportunity_summary = summarize_opportunities(opportunities)
|
||||
failure_count = len(failures)
|
||||
summary = {
|
||||
"decision": "blocked" if failure_count else "proposal-review" if proposal_rows else "monitor",
|
||||
"source_supplied": source is not None,
|
||||
"pattern_count": int(pattern_summary.get("pattern_count", 0) or 0),
|
||||
"proposal_count": int(proposal_summary.get("proposal_count", 0) or len(proposal_rows)),
|
||||
"approval_count": int(approval_summary.get("approval_count", 0) or 0),
|
||||
"pending_review_count": int(approval_summary.get("pending_review_count", 0) or 0),
|
||||
"applied_count": int(approval_summary.get("applied_count", 0) or regression_summary.get("applied_count", 0) or 0),
|
||||
"rollback_count": int(approval_summary.get("rollback_count", 0) or regression_summary.get("rollback_count", 0) or 0),
|
||||
"local_blueprint_ready": coverage_summary.get("local_blueprint_ready") is True,
|
||||
"public_world_class_ready": coverage_summary.get("public_world_class_ready") is True,
|
||||
"world_class_pending_count": int(ledger_summary.get("pending_count", coverage_summary.get("world_class_evidence_pending_count", 0)) or 0),
|
||||
"release_lock_ready": benchmark_summary.get("release_lock_ready") is True,
|
||||
"evidence_consistency_ok": consistency.get("ok") is True,
|
||||
"writes_source_files": False,
|
||||
"auto_patch_enabled": False,
|
||||
"failure_count": failure_count,
|
||||
}
|
||||
actions = build_actions(summary, proposal_rows)
|
||||
operations_contract = {
|
||||
"schema_version": "1.0",
|
||||
"contract": "daily-skillops-report",
|
||||
"explicit_source_required_for_scan": True,
|
||||
"implicit_private_log_scan": False,
|
||||
"raw_content_stored": False,
|
||||
"redacted_excerpts_only": True,
|
||||
"proposal_only": proposal_contract.get("proposal_only") is not False,
|
||||
"approval_required_for_writes": True,
|
||||
"writes_source_files": False,
|
||||
"auto_patch_enabled": False,
|
||||
"daily_report_counts_as_world_class_evidence": False,
|
||||
}
|
||||
report = {
|
||||
"schema_version": "1.0",
|
||||
"ok": failure_count == 0,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
**{key: summary[key] for key in SUMMARY_FIELDS},
|
||||
"summary": summary,
|
||||
"operations_contract": operations_contract,
|
||||
"report_contract": {
|
||||
"schema_version": "1.0",
|
||||
"contract": "daily-skillops-report",
|
||||
"top_level_mirrors_summary": True,
|
||||
"summary_fields": SUMMARY_FIELDS,
|
||||
"source_of_truth": ["summary", "operations_contract"],
|
||||
},
|
||||
"source": source_summary(pattern_report) if pattern_report else {},
|
||||
"patterns": pattern_report.get("patterns", []) if isinstance(pattern_report.get("patterns"), list) else [],
|
||||
"proposals": proposal_rows,
|
||||
"opportunity_summary": opportunity_summary,
|
||||
"opportunities": opportunities,
|
||||
"decision_policy": decision_policy_contract(),
|
||||
"approval": {
|
||||
"approval_count": summary["approval_count"],
|
||||
"pending_review_count": summary["pending_review_count"],
|
||||
"applied_count": summary["applied_count"],
|
||||
"rollback_count": summary["rollback_count"],
|
||||
},
|
||||
"release_state": {
|
||||
"local_blueprint_ready": summary["local_blueprint_ready"],
|
||||
"public_world_class_ready": summary["public_world_class_ready"],
|
||||
"world_class_pending_count": summary["world_class_pending_count"],
|
||||
"release_lock_ready": summary["release_lock_ready"],
|
||||
"evidence_consistency_ok": summary["evidence_consistency_ok"],
|
||||
},
|
||||
"actions": actions,
|
||||
"failures": failures,
|
||||
"source_reports": {
|
||||
"patterns": display_path(patterns_path, skill_dir),
|
||||
"proposals": display_path(proposals_path, skill_dir),
|
||||
"approval_ledger": "reports/adaptation_approval_ledger.json",
|
||||
"regression": "reports/adaptation_regression_report.json",
|
||||
"skill_os2_coverage": "reports/skill_os2_coverage.json",
|
||||
"world_class_ledger": "reports/world_class_evidence_ledger.json",
|
||||
"evidence_consistency": "reports/evidence_consistency.json",
|
||||
"benchmark_reproducibility": "reports/benchmark_reproducibility.json",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": str(default_daily_path(skill_dir, generated_at, "json").relative_to(skill_dir)),
|
||||
"markdown": str(default_daily_path(skill_dir, generated_at, "md").relative_to(skill_dir)),
|
||||
},
|
||||
}
|
||||
return report
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Daily SkillOps Report",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- source supplied: `{str(summary['source_supplied']).lower()}`",
|
||||
f"- patterns: `{summary['pattern_count']}`",
|
||||
f"- proposals: `{summary['proposal_count']}`",
|
||||
f"- pending approvals: `{summary['pending_review_count']}`",
|
||||
f"- applied patches: `{summary['applied_count']}`",
|
||||
f"- rollbacks: `{summary['rollback_count']}`",
|
||||
f"- local blueprint ready: `{str(summary['local_blueprint_ready']).lower()}`",
|
||||
f"- public world-class ready: `{str(summary['public_world_class_ready']).lower()}`",
|
||||
f"- world-class pending: `{summary['world_class_pending_count']}`",
|
||||
f"- release lock ready: `{str(summary['release_lock_ready']).lower()}`",
|
||||
f"- evidence consistency ok: `{str(summary['evidence_consistency_ok']).lower()}`",
|
||||
"",
|
||||
"This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence.",
|
||||
"",
|
||||
"## Privacy Boundary",
|
||||
"",
|
||||
]
|
||||
contract = report["operations_contract"]
|
||||
for key in [
|
||||
"explicit_source_required_for_scan",
|
||||
"implicit_private_log_scan",
|
||||
"raw_content_stored",
|
||||
"redacted_excerpts_only",
|
||||
"proposal_only",
|
||||
"approval_required_for_writes",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
"daily_report_counts_as_world_class_evidence",
|
||||
]:
|
||||
lines.append(f"- {key}: `{str(contract[key]).lower()}`")
|
||||
lines.extend(["", "## Actions", ""])
|
||||
for action in report["actions"]:
|
||||
lines.append(f"- `{action['priority']}` {action['action']}")
|
||||
lines.extend(["", "## Opportunities", ""])
|
||||
opportunity_summary = report.get("opportunity_summary", {})
|
||||
lines.extend(
|
||||
[
|
||||
f"- count: `{opportunity_summary.get('opportunity_count', 0)}`",
|
||||
f"- top score: `{opportunity_summary.get('top_score', 0)}`",
|
||||
f"- ready for approval review: `{opportunity_summary.get('ready_for_approval_review_count', 0)}`",
|
||||
]
|
||||
)
|
||||
for opportunity in report.get("opportunities", []):
|
||||
lines.extend(
|
||||
[
|
||||
f"### {opportunity.get('title', '')}",
|
||||
"",
|
||||
f"- ID: `{opportunity.get('opportunity_id', '')}`",
|
||||
f"- Action: `{opportunity.get('action_type', '')}`",
|
||||
f"- Decision: `{opportunity.get('decision', '')}`",
|
||||
f"- Score: `{opportunity.get('score', 0)}`",
|
||||
f"- Risk: `{opportunity.get('risk_level', '')}`",
|
||||
f"- Policy: {opportunity.get('policy_reason', '')}",
|
||||
]
|
||||
)
|
||||
lines.extend(["", "## Patterns", ""])
|
||||
patterns = report.get("patterns", [])
|
||||
if not patterns:
|
||||
lines.append("- No repeated pattern met the support threshold.")
|
||||
for pattern in patterns:
|
||||
lines.append(
|
||||
f"- `{pattern.get('pattern_id', '')}`: {pattern.get('label', '')} "
|
||||
f"(support `{pattern.get('support_count', 0)}`, confidence `{pattern.get('confidence', '')}`)"
|
||||
)
|
||||
lines.extend(["", "## Proposals", ""])
|
||||
proposals = report.get("proposals", [])
|
||||
if not proposals:
|
||||
lines.append("- No adaptation proposals are waiting for review.")
|
||||
for proposal in proposals:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {proposal['title']}",
|
||||
"",
|
||||
f"- ID: `{proposal['proposal_id']}`",
|
||||
f"- Pattern: `{proposal['pattern_id']}`",
|
||||
f"- Status: `{proposal['status']}`",
|
||||
f"- Risk: `{proposal['risk_level']}`",
|
||||
f"- Requires approval: `{str(proposal['requires_approval']).lower()}`",
|
||||
"- Target files:",
|
||||
]
|
||||
)
|
||||
lines.extend(f" - `{path}`" for path in proposal.get("target_files", []))
|
||||
lines.append("- Verification:")
|
||||
lines.extend(f" - `{command}`" for command in proposal.get("verification_commands", []))
|
||||
lines.append("")
|
||||
lines.extend(["## Evidence", ""])
|
||||
for label, path in report["source_reports"].items():
|
||||
lines.append(f"- {label}: `{path}`")
|
||||
if report["failures"]:
|
||||
lines.extend(["", "## Failures", ""])
|
||||
lines.extend(f"- {failure}" for failure in report["failures"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a Daily SkillOps report from explicit-source adaptive evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--source", help="Explicit curated source file to scan before rendering the daily report.")
|
||||
parser.add_argument("--patterns-json", default="reports/user_patterns.json")
|
||||
parser.add_argument("--proposals-json", default="reports/adaptation_proposals.json")
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--min-support", type=int, default=2)
|
||||
parser.add_argument("--generated-at", default=utc_now())
|
||||
parser.add_argument("--allow-history-source", action="store_true")
|
||||
parser.add_argument("--no-refresh-source-reports", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(
|
||||
skill_dir,
|
||||
args.generated_at,
|
||||
source=Path(args.source) if args.source else None,
|
||||
min_support=args.min_support,
|
||||
allow_history_source=args.allow_history_source,
|
||||
patterns_json=Path(args.patterns_json),
|
||||
proposals_json=Path(args.proposals_json),
|
||||
write_source_reports=not args.no_refresh_source_reports,
|
||||
)
|
||||
output_json = resolve_output(skill_dir, args.output_json) if args.output_json else default_daily_path(skill_dir, args.generated_at, "json")
|
||||
output_md = resolve_output(skill_dir, args.output_md) if args.output_md else default_daily_path(skill_dir, args.generated_at, "md")
|
||||
report["artifacts"] = {
|
||||
"json": display_path(output_json, skill_dir),
|
||||
"markdown": display_path(output_md, skill_dir),
|
||||
}
|
||||
write_json(output_json, report)
|
||||
write_text(output_md, render_markdown(report))
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,676 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from evidence_consistency_artifact_roles import build_preflight_artifact_role_handoff_checks
|
||||
from evidence_consistency_core import (
|
||||
ADOPTION_SUMMARY_KEYS,
|
||||
BENCHMARK_SUMMARY_KEYS,
|
||||
LEDGER_SUMMARY_KEYS,
|
||||
LOCKSTEP_SECTIONS,
|
||||
REQUIRED_REPORTS,
|
||||
REQUIRED_TEXT_REPORTS,
|
||||
add_check,
|
||||
as_int,
|
||||
beta_public_claim_split_values,
|
||||
compare_summary_keys,
|
||||
compare_values,
|
||||
gate_by_key,
|
||||
load_json,
|
||||
load_text,
|
||||
nested,
|
||||
rel_path,
|
||||
render_markdown,
|
||||
report_contract,
|
||||
scanned_surface_paths,
|
||||
)
|
||||
from evidence_consistency_phase_queue import build_phase_queue_consistency_check
|
||||
from evidence_consistency_release import build_release_evidence_flow_check
|
||||
from evidence_consistency_skill_os2_review import build_skill_os2_review_current_evidence_check
|
||||
from evidence_consistency_world_class import build_world_class_workflow_check
|
||||
from skill_ir_paths import find_skill_ir_path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a cross-report evidence consistency gate for generated Skill OS reports."
|
||||
|
||||
|
||||
def git_worktree_status(skill_dir: Path) -> dict[str, Any]:
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["git", "-C", str(skill_dir), "status", "--porcelain=v1", "-uall"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
except OSError:
|
||||
return {"available": False, "clean": None, "changed_file_count": None}
|
||||
if proc.returncode != 0:
|
||||
return {"available": False, "clean": None, "changed_file_count": None}
|
||||
changes = [line for line in proc.stdout.splitlines() if line.strip()]
|
||||
return {"available": True, "clean": not changes, "changed_file_count": len(changes)}
|
||||
|
||||
|
||||
def build_review_studio_gate_action_mirror_check(review_studio: dict[str, Any]) -> dict[str, Any]:
|
||||
gates = review_studio.get("gates", []) if isinstance(review_studio.get("gates", []), list) else []
|
||||
actions = (
|
||||
review_studio.get("review_actions", [])
|
||||
if isinstance(review_studio.get("review_actions", []), list)
|
||||
else []
|
||||
)
|
||||
non_pass_gates = [gate for gate in gates if isinstance(gate, dict) and gate.get("status") != "pass"]
|
||||
pass_gates = [gate for gate in gates if isinstance(gate, dict) and gate.get("status") == "pass"]
|
||||
actions_by_gate = {
|
||||
str(action.get("gate_key", "")): action
|
||||
for action in actions
|
||||
if isinstance(action, dict) and str(action.get("gate_key", ""))
|
||||
}
|
||||
expected = {
|
||||
"non_pass_gate_keys": sorted(str(gate.get("key", "")) for gate in non_pass_gates),
|
||||
"action_gate_keys": sorted(str(gate.get("key", "")) for gate in non_pass_gates),
|
||||
"pass_gate_action_ids_empty": True,
|
||||
"gate_action_mirrors": {
|
||||
str(gate.get("key", "")): {
|
||||
"review_action_id": f"review-action-{gate.get('key', '')}",
|
||||
"review_action_status": gate.get("status", ""),
|
||||
"title_present": True,
|
||||
"next_step_present": True,
|
||||
"reason_present": True,
|
||||
"source_ref_present": True,
|
||||
"verification_command_present": True,
|
||||
}
|
||||
for gate in non_pass_gates
|
||||
},
|
||||
}
|
||||
actual = {
|
||||
"non_pass_gate_keys": sorted(str(gate.get("key", "")) for gate in non_pass_gates),
|
||||
"action_gate_keys": sorted(actions_by_gate),
|
||||
"pass_gate_action_ids_empty": all(not gate.get("review_action_id") for gate in pass_gates),
|
||||
"gate_action_mirrors": {},
|
||||
}
|
||||
for gate in non_pass_gates:
|
||||
key = str(gate.get("key", ""))
|
||||
action = actions_by_gate.get(key, {})
|
||||
actual["gate_action_mirrors"][key] = {
|
||||
"review_action_id": gate.get("review_action_id", ""),
|
||||
"review_action_status": gate.get("review_action_status", ""),
|
||||
"title_present": bool(gate.get("review_action_title")),
|
||||
"next_step_present": bool(gate.get("review_action_next_step")),
|
||||
"reason_present": bool(gate.get("review_action_reason")),
|
||||
"source_ref_present": bool(gate.get("review_action_source_ref_count")),
|
||||
"verification_command_present": bool(gate.get("review_action_verification_command")),
|
||||
"top_level_action_id": action.get("action_id", ""),
|
||||
"top_level_action_status": action.get("status", ""),
|
||||
"top_level_title_present": bool(action.get("title")),
|
||||
"top_level_next_step_present": bool(action.get("next_step")),
|
||||
"top_level_reason_present": bool(action.get("reason")),
|
||||
"top_level_source_ref_present": bool(action.get("source_refs")),
|
||||
"top_level_verification_command_present": bool(action.get("verification_command")),
|
||||
}
|
||||
expected["gate_action_mirrors"][key]["top_level_action_id"] = f"review-action-{key}"
|
||||
expected["gate_action_mirrors"][key]["top_level_action_status"] = gate.get("status", "")
|
||||
expected["gate_action_mirrors"][key]["top_level_title_present"] = True
|
||||
expected["gate_action_mirrors"][key]["top_level_next_step_present"] = True
|
||||
expected["gate_action_mirrors"][key]["top_level_reason_present"] = True
|
||||
expected["gate_action_mirrors"][key]["top_level_source_ref_present"] = True
|
||||
expected["gate_action_mirrors"][key]["top_level_verification_command_present"] = True
|
||||
return {
|
||||
"key": "review-studio-gate-action-mirror",
|
||||
"label": "Review Studio gates mirror review actions",
|
||||
"status": "pass" if expected == actual else "fail",
|
||||
"expected": expected,
|
||||
"actual": actual,
|
||||
"paths": [REQUIRED_REPORTS["review_studio"]],
|
||||
"detail": (
|
||||
"Every non-pass Review Studio gate must have exactly one top-level review action and a gate-local "
|
||||
"action summary with source refs and a verification command."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
reports: dict[str, dict[str, Any]] = {}
|
||||
text_reports: dict[str, str] = {}
|
||||
checks: list[dict[str, Any]] = []
|
||||
load_failures: dict[str, str] = {}
|
||||
for name, relative in REQUIRED_REPORTS.items():
|
||||
payload, failure = load_json(skill_dir / relative)
|
||||
reports[name] = payload
|
||||
if failure:
|
||||
load_failures[relative] = failure
|
||||
for name, relative in REQUIRED_TEXT_REPORTS.items():
|
||||
text, failure = load_text(skill_dir / relative)
|
||||
text_reports[name] = text
|
||||
if failure:
|
||||
load_failures[relative] = failure
|
||||
add_check(
|
||||
checks,
|
||||
key="required-report-artifacts",
|
||||
label="Required report artifacts are readable",
|
||||
status="pass" if not load_failures else "fail",
|
||||
expected="all required JSON and Markdown reports exist and parse",
|
||||
actual=load_failures or "all readable",
|
||||
paths=list(REQUIRED_REPORTS.values()) + list(REQUIRED_TEXT_REPORTS.values()),
|
||||
detail="The consistency gate can only be trusted when every source JSON report parses and every source Markdown report is readable.",
|
||||
)
|
||||
checks.append(build_release_evidence_flow_check(skill_dir))
|
||||
|
||||
benchmark = reports["benchmark"]
|
||||
overview = reports["overview"]
|
||||
interpretation = reports["interpretation"]
|
||||
adoption = reports["adoption"]
|
||||
ledger = reports["world_class_ledger"]
|
||||
world_class_plan = reports["world_class_plan"]
|
||||
world_class_intake = reports["world_class_intake"]
|
||||
world_class_preflight = reports["world_class_preflight"]
|
||||
world_class_submission_review = reports["world_class_submission_review"]
|
||||
world_class_operator_runbook = reports["world_class_operator_runbook"]
|
||||
coverage = reports["skill_os2_coverage"]
|
||||
review_studio = reports["review_studio"]
|
||||
package_verification = reports["package_verification"]
|
||||
install_simulation = reports["install_simulation"]
|
||||
trust = reports["trust"]
|
||||
context_budget = reports["context_budget"]
|
||||
claim_guard = reports["world_class_claim_guard"]
|
||||
|
||||
checks.append(build_review_studio_gate_action_mirror_check(review_studio))
|
||||
|
||||
benchmark_summary = nested(benchmark, ["summary"], {})
|
||||
adoption_summary = nested(adoption, ["summary"], {})
|
||||
ledger_summary = nested(ledger, ["summary"], {})
|
||||
coverage_summary = nested(coverage, ["summary"], {})
|
||||
studio_summary = nested(review_studio, ["summary"], {})
|
||||
package_summary = nested(package_verification, ["summary"], {})
|
||||
install_summary = nested(install_simulation, ["summary"], {})
|
||||
trust_summary = nested(trust, ["summary"], {})
|
||||
context_stats = nested(context_budget, ["stats"], {})
|
||||
claim_guard_summary = nested(claim_guard, ["summary"], {})
|
||||
preflight_summary = nested(world_class_preflight, ["summary"], {})
|
||||
context_governance = (
|
||||
context_stats.get("deferred_resource_governance", {}) if isinstance(context_stats, dict) else {}
|
||||
)
|
||||
if not isinstance(context_governance, dict):
|
||||
context_governance = {}
|
||||
large_deferred_dirs = context_stats.get("large_deferred_resource_dirs", []) if isinstance(context_stats, dict) else []
|
||||
top_deferred = "none"
|
||||
if isinstance(large_deferred_dirs, list) and large_deferred_dirs:
|
||||
first = large_deferred_dirs[0] if isinstance(large_deferred_dirs[0], dict) else {}
|
||||
top_deferred = f"{first.get('path', 'resource')} {first.get('estimated_tokens', 'n/a')}"
|
||||
context_expected_status = "pass" if context_budget.get("ok") else "block"
|
||||
if context_budget.get("warnings") and context_expected_status == "pass":
|
||||
context_expected_status = "warn"
|
||||
context_expected_detail = (
|
||||
f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/"
|
||||
f"{context_stats.get('context_budget_limit', 'n/a')}; "
|
||||
f"deferred {context_stats.get('deferred_resource_tokens', 'n/a')}/"
|
||||
f"{context_stats.get('deferred_resource_warn_threshold', 'n/a')}; "
|
||||
f"top deferred {top_deferred}; "
|
||||
f"resource governance {context_governance.get('status', 'unknown')}; "
|
||||
f"quality density {context_stats.get('quality_density', 'n/a')}"
|
||||
)
|
||||
context_gate = gate_by_key(review_studio, "context-budget")
|
||||
compare_values(
|
||||
checks,
|
||||
key="review-studio-context-budget-mirror",
|
||||
label="Review Studio mirrors context budget governance",
|
||||
expected={
|
||||
"status": context_expected_status,
|
||||
"detail": context_expected_detail,
|
||||
"evidence": REQUIRED_REPORTS["context_budget"],
|
||||
},
|
||||
actual={
|
||||
"status": context_gate.get("status"),
|
||||
"detail": context_gate.get("detail"),
|
||||
"evidence": context_gate.get("evidence"),
|
||||
},
|
||||
paths=[REQUIRED_REPORTS["context_budget"], REQUIRED_REPORTS["review_studio"]],
|
||||
detail=(
|
||||
"Review Studio must not keep stale context warnings after context reports prove large deferred resources are governed."
|
||||
),
|
||||
)
|
||||
if isinstance(benchmark_summary, dict):
|
||||
compare_values(
|
||||
checks,
|
||||
key="benchmark-release-lock-self-consistency",
|
||||
label="Benchmark release lock matches source dirty state",
|
||||
expected=not bool(nested(benchmark, ["git_status", "source_dirty"], True)),
|
||||
actual=benchmark_summary.get("release_lock_ready"),
|
||||
paths=[REQUIRED_REPORTS["benchmark"]],
|
||||
detail=(
|
||||
"The benchmark release lock must be blocked by source changes, while generated evidence artifacts are "
|
||||
"tracked as generation context."
|
||||
),
|
||||
)
|
||||
current_git_status = git_worktree_status(skill_dir)
|
||||
if current_git_status.get("available") and current_git_status.get("clean") is True:
|
||||
compare_values(
|
||||
checks,
|
||||
key="benchmark-clean-worktree-release-lock",
|
||||
label="Clean worktree keeps a clean benchmark release lock",
|
||||
expected=True,
|
||||
actual=benchmark_summary.get("release_lock_ready"),
|
||||
paths=[REQUIRED_REPORTS["benchmark"]],
|
||||
detail=(
|
||||
"If the current worktree is clean, the committed benchmark report must not still carry a dirty release lock from an earlier generation."
|
||||
),
|
||||
)
|
||||
else:
|
||||
add_check(
|
||||
checks,
|
||||
key="benchmark-clean-worktree-release-lock",
|
||||
label="Clean worktree keeps a clean benchmark release lock",
|
||||
status="pass",
|
||||
expected="checked only when git is available and the current worktree is clean",
|
||||
actual=current_git_status,
|
||||
paths=[REQUIRED_REPORTS["benchmark"]],
|
||||
detail=(
|
||||
"Dirty or non-git worktrees cannot prove final release-lock freshness, so this check is advisory until the final clean-lock pass."
|
||||
),
|
||||
)
|
||||
skill_name = str(overview.get("name") or nested(review_studio, ["data", "frontmatter", "name"]) or skill_dir.name)
|
||||
expected_skill_ir_path = find_skill_ir_path(skill_dir, skill_name, require_schema=True)
|
||||
expected_skill_ir = {
|
||||
"source_path": expected_skill_ir_path,
|
||||
"exists": bool(expected_skill_ir_path and (skill_dir / expected_skill_ir_path).exists()),
|
||||
"schema_version": load_json(skill_dir / expected_skill_ir_path)[0].get("schema_version")
|
||||
if expected_skill_ir_path
|
||||
else "",
|
||||
}
|
||||
actual_skill_ir = {
|
||||
"overview_source_path": nested(overview, ["skill_ir", "source_path"], ""),
|
||||
"interpretation_source_path": nested(interpretation, ["skill_ir", "source_path"], ""),
|
||||
"review_studio_evidence_path": nested(review_studio, ["evidence_paths", "skill_ir"], ""),
|
||||
"overview_deliverable": expected_skill_ir_path in nested(overview, ["skill_summary", "deliverables"], []),
|
||||
"interpretation_deliverable": expected_skill_ir_path
|
||||
in nested(interpretation, ["skill_summary", "deliverables"], []),
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key="skill-ir-evidence-path-contract",
|
||||
label="Human-facing reports expose the canonical Skill IR artifact",
|
||||
expected={
|
||||
"overview_source_path": expected_skill_ir["source_path"],
|
||||
"interpretation_source_path": expected_skill_ir["source_path"],
|
||||
"review_studio_evidence_path": expected_skill_ir["source_path"],
|
||||
"overview_deliverable": True,
|
||||
"interpretation_deliverable": True,
|
||||
"exists": True,
|
||||
"schema_version": "2.0.0",
|
||||
},
|
||||
actual={**actual_skill_ir, **{key: expected_skill_ir[key] for key in ("exists", "schema_version")}},
|
||||
paths=[
|
||||
REQUIRED_REPORTS["overview"],
|
||||
REQUIRED_REPORTS["interpretation"],
|
||||
REQUIRED_REPORTS["review_studio"],
|
||||
expected_skill_ir_path or "skill-ir/examples",
|
||||
],
|
||||
detail="Skill IR is the 2.0 platform-neutral semantic source, so user-facing reports must link to the artifact that actually exists.",
|
||||
)
|
||||
for report_key, payload in [("overview", overview), ("interpretation", interpretation)]:
|
||||
embedded_benchmark = nested(payload, ["benchmark_reproducibility"], {})
|
||||
compare_values(
|
||||
checks,
|
||||
key=f"{report_key}-benchmark-commit",
|
||||
label=f"{report_key} embeds the benchmark commit",
|
||||
expected=benchmark.get("commit"),
|
||||
actual=nested(embedded_benchmark, ["commit"]),
|
||||
paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]],
|
||||
detail="Human-facing reports must point to the same benchmark release-lock commit.",
|
||||
)
|
||||
compare_summary_keys(
|
||||
checks,
|
||||
key_prefix=f"{report_key}-benchmark-summary",
|
||||
label=f"{report_key} embeds benchmark summary fields",
|
||||
source_summary=benchmark_summary if isinstance(benchmark_summary, dict) else {},
|
||||
embedded_summary=nested(embedded_benchmark, ["summary"], {}),
|
||||
keys=BENCHMARK_SUMMARY_KEYS,
|
||||
paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS[report_key]],
|
||||
)
|
||||
compare_summary_keys(
|
||||
checks,
|
||||
key_prefix=f"{report_key}-adoption-summary",
|
||||
label=f"{report_key} embeds adoption drift summary fields",
|
||||
source_summary=adoption_summary if isinstance(adoption_summary, dict) else {},
|
||||
embedded_summary=nested(payload, ["adoption_drift", "summary"], {}),
|
||||
keys=ADOPTION_SUMMARY_KEYS,
|
||||
paths=[REQUIRED_REPORTS["adoption"], REQUIRED_REPORTS[report_key]],
|
||||
)
|
||||
compare_summary_keys(
|
||||
checks,
|
||||
key_prefix=f"{report_key}-world-class-ledger-summary",
|
||||
label=f"{report_key} embeds world-class ledger summary fields",
|
||||
source_summary=ledger_summary if isinstance(ledger_summary, dict) else {},
|
||||
embedded_summary=nested(payload, ["world_class_evidence_ledger", "summary"], {}),
|
||||
keys=LEDGER_SUMMARY_KEYS,
|
||||
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]],
|
||||
)
|
||||
readiness_expected = {
|
||||
"ready": ledger_summary.get("ready_to_claim_world_class") if isinstance(ledger_summary, dict) else None,
|
||||
"decision": ledger_summary.get("decision") if isinstance(ledger_summary, dict) else None,
|
||||
"pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None,
|
||||
"accepted_count": ledger_summary.get("accepted_count") if isinstance(ledger_summary, dict) else None,
|
||||
"source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None,
|
||||
"source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None,
|
||||
}
|
||||
readiness = nested(payload, ["world_class_readiness"], {})
|
||||
readiness_actual = {key: readiness.get(key) if isinstance(readiness, dict) else None for key in readiness_expected}
|
||||
compare_values(
|
||||
checks,
|
||||
key=f"{report_key}-world-class-readiness",
|
||||
label=f"{report_key} derives readiness from the ledger",
|
||||
expected=readiness_expected,
|
||||
actual=readiness_actual,
|
||||
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS[report_key]],
|
||||
detail="Readiness summaries must be derived from the evidence ledger, not hand-maintained copy.",
|
||||
)
|
||||
|
||||
for section in LOCKSTEP_SECTIONS:
|
||||
compare_values(
|
||||
checks,
|
||||
key=f"overview-interpretation-lockstep-{section.replace('_', '-')}",
|
||||
label=f"Overview and interpretation share {section}",
|
||||
expected=overview.get(section),
|
||||
actual=interpretation.get(section),
|
||||
paths=[REQUIRED_REPORTS["overview"], REQUIRED_REPORTS["interpretation"]],
|
||||
detail="The first-class interpretation report must stay in lockstep with the canonical overview model.",
|
||||
)
|
||||
|
||||
for report_key, expected_html in [
|
||||
("overview", "reports/skill-overview.html"),
|
||||
("interpretation", "reports/skill-interpretation.html"),
|
||||
]:
|
||||
contract = report_contract(reports[report_key])
|
||||
expected = {
|
||||
"schema_version": "2.0",
|
||||
"default_language": "zh-CN",
|
||||
"layout": "kami-white-audit-v2",
|
||||
"html_report": expected_html,
|
||||
"html_exists": True,
|
||||
}
|
||||
actual = {
|
||||
"schema_version": contract.get("schema_version"),
|
||||
"default_language": contract.get("default_language"),
|
||||
"layout": contract.get("layout"),
|
||||
"html_report": contract.get("html_report"),
|
||||
"html_exists": (skill_dir / expected_html).exists(),
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key=f"{report_key}-html-contract",
|
||||
label=f"{report_key} has a stable HTML contract",
|
||||
expected=expected,
|
||||
actual=actual,
|
||||
paths=[REQUIRED_REPORTS[report_key], expected_html],
|
||||
detail="Report output paths and language defaults are part of the user-facing contract.",
|
||||
)
|
||||
|
||||
if isinstance(ledger_summary, dict):
|
||||
expected_boundary = {
|
||||
"world_class_evidence_pending_count": ledger_summary.get("pending_count"),
|
||||
"public_world_class_ready": ledger_summary.get("ready_to_claim_world_class"),
|
||||
}
|
||||
actual_boundary = {
|
||||
"world_class_evidence_pending_count": coverage_summary.get("world_class_evidence_pending_count")
|
||||
if isinstance(coverage_summary, dict)
|
||||
else None,
|
||||
"public_world_class_ready": coverage_summary.get("public_world_class_ready")
|
||||
if isinstance(coverage_summary, dict)
|
||||
else None,
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key="coverage-world-class-boundary",
|
||||
label="Coverage report mirrors world-class evidence boundary",
|
||||
expected=expected_boundary,
|
||||
actual=actual_boundary,
|
||||
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["skill_os2_coverage"]],
|
||||
detail="Blueprint coverage can be locally complete while public world-class evidence remains pending.",
|
||||
)
|
||||
benchmark_boundary = {
|
||||
"world_class_ledger_pending_count": ledger_summary.get("pending_count"),
|
||||
"world_class_source_check_count": ledger_summary.get("source_check_count"),
|
||||
"world_class_source_pass_count": ledger_summary.get("source_pass_count"),
|
||||
"world_class_source_blocked_count": ledger_summary.get("source_blocked_count"),
|
||||
"public_claim_ready": ledger_summary.get("ready_to_claim_world_class"),
|
||||
}
|
||||
actual_benchmark_boundary = {
|
||||
key: benchmark_summary.get(key) if isinstance(benchmark_summary, dict) else None
|
||||
for key in benchmark_boundary
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key="benchmark-world-class-boundary",
|
||||
label="Benchmark report mirrors world-class evidence boundary",
|
||||
expected=benchmark_boundary,
|
||||
actual=actual_benchmark_boundary,
|
||||
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]],
|
||||
detail="Benchmark reproducibility must not overstate public claim readiness.",
|
||||
)
|
||||
beta_boundary, actual_beta_boundary = beta_public_claim_split_values(
|
||||
benchmark,
|
||||
ledger,
|
||||
review_studio,
|
||||
)
|
||||
compare_values(
|
||||
checks,
|
||||
key="benchmark-beta-public-claim-split",
|
||||
label="Benchmark separates beta testing from public claims",
|
||||
expected=beta_boundary,
|
||||
actual=actual_beta_boundary,
|
||||
paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS["world_class_ledger"]],
|
||||
detail=(
|
||||
"Beta/public testing may proceed with human blind review deferred, but public world-class or superiority claims must remain blocked."
|
||||
),
|
||||
)
|
||||
preflight_boundary = {
|
||||
"pending_count": ledger_summary.get("pending_count"),
|
||||
"source_check_count": ledger_summary.get("source_check_count"),
|
||||
"source_pass_count": ledger_summary.get("source_pass_count"),
|
||||
"source_blocked_count": ledger_summary.get("source_blocked_count"),
|
||||
"ready_to_claim_world_class": ledger_summary.get("ready_to_claim_world_class"),
|
||||
"preflight_counts_as_evidence": False,
|
||||
"credential_value_exposed": False,
|
||||
}
|
||||
actual_preflight_boundary = {
|
||||
key: preflight_summary.get(key) if isinstance(preflight_summary, dict) else None
|
||||
for key in preflight_boundary
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key="preflight-world-class-boundary",
|
||||
label="Preflight mirrors ledger without accepting evidence",
|
||||
expected=preflight_boundary,
|
||||
actual=actual_preflight_boundary,
|
||||
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["world_class_preflight"]],
|
||||
detail="Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness.",
|
||||
)
|
||||
checks.extend(
|
||||
build_preflight_artifact_role_handoff_checks(
|
||||
skill_dir=skill_dir,
|
||||
world_class_preflight=world_class_preflight,
|
||||
review_studio=review_studio,
|
||||
report_paths=REQUIRED_REPORTS,
|
||||
)
|
||||
)
|
||||
checks.append(
|
||||
build_phase_queue_consistency_check(
|
||||
world_class_preflight=world_class_preflight,
|
||||
world_class_operator_runbook=world_class_operator_runbook,
|
||||
review_studio=review_studio,
|
||||
report_paths=REQUIRED_REPORTS,
|
||||
)
|
||||
)
|
||||
|
||||
public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False
|
||||
compare_values(
|
||||
checks,
|
||||
key="review-studio-no-overclaim",
|
||||
label="Review Studio does not overclaim pending world-class evidence",
|
||||
expected=False if not public_ready else True,
|
||||
actual=studio_summary.get("decision") in {"pass", "release", "ready", "world-class-ready"}
|
||||
if isinstance(studio_summary, dict)
|
||||
else None,
|
||||
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["review_studio"]],
|
||||
detail="When world-class evidence is pending, Review Studio must stay in a review or warning posture.",
|
||||
)
|
||||
claim_surface_paths = scanned_surface_paths(claim_guard)
|
||||
required_claim_surfaces = [
|
||||
"README.md",
|
||||
"SKILL.md",
|
||||
"manifest.json",
|
||||
"agents/interface.yaml",
|
||||
"evidence/world_class/README.md",
|
||||
"security/permission_policy.json",
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
]
|
||||
optional_claim_surfaces = [
|
||||
"dist/manifest.json",
|
||||
"dist/targets/openai/adapter.json",
|
||||
]
|
||||
required_claim_surfaces.extend(
|
||||
path for path in optional_claim_surfaces if (skill_dir / path).exists()
|
||||
)
|
||||
prohibited_claim_surface_prefixes = [
|
||||
"dist/install-simulation/",
|
||||
"evidence/world_class/submissions/",
|
||||
]
|
||||
json_claim_surface_count = as_int(claim_guard_summary.get("json_claim_surface_count"))
|
||||
metadata_claim_surface_count = as_int(claim_guard_summary.get("metadata_claim_surface_count"))
|
||||
package_claim_surface_count = as_int(claim_guard_summary.get("package_claim_surface_count"))
|
||||
claim_surface_count = as_int(claim_guard_summary.get("claim_surface_count"))
|
||||
expected_claim_guard_surface = {
|
||||
"overclaim_guard_active": True,
|
||||
"violation_count": 0,
|
||||
"ledger_ready_to_claim_world_class": ledger_summary.get("ready_to_claim_world_class")
|
||||
if isinstance(ledger_summary, dict)
|
||||
else None,
|
||||
"ledger_pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None,
|
||||
"metadata_covers_json": True,
|
||||
"package_surface_minimum": True,
|
||||
"claim_surface_covers_package": True,
|
||||
"required_surfaces": {path: True for path in required_claim_surfaces},
|
||||
"prohibited_surfaces": [],
|
||||
}
|
||||
actual_claim_guard_surface = {
|
||||
"overclaim_guard_active": claim_guard_summary.get("overclaim_guard_active"),
|
||||
"violation_count": claim_guard_summary.get("violation_count"),
|
||||
"ledger_ready_to_claim_world_class": claim_guard_summary.get("ledger_ready_to_claim_world_class"),
|
||||
"ledger_pending_count": claim_guard_summary.get("ledger_pending_count"),
|
||||
"metadata_covers_json": (
|
||||
metadata_claim_surface_count is not None
|
||||
and json_claim_surface_count is not None
|
||||
and metadata_claim_surface_count >= json_claim_surface_count
|
||||
),
|
||||
"package_surface_minimum": package_claim_surface_count is not None and package_claim_surface_count >= 5,
|
||||
"claim_surface_covers_package": (
|
||||
claim_surface_count is not None
|
||||
and package_claim_surface_count is not None
|
||||
and claim_surface_count >= package_claim_surface_count
|
||||
),
|
||||
"required_surfaces": {path: path in claim_surface_paths for path in required_claim_surfaces},
|
||||
"prohibited_surfaces": sorted(
|
||||
path
|
||||
for path in claim_surface_paths
|
||||
if any(path.startswith(prefix) for prefix in prohibited_claim_surface_prefixes)
|
||||
),
|
||||
}
|
||||
compare_values(
|
||||
checks,
|
||||
key="claim-guard-package-runtime-surface",
|
||||
label="Claim guard covers package and runtime claim surfaces",
|
||||
expected=expected_claim_guard_surface,
|
||||
actual=actual_claim_guard_surface,
|
||||
paths=[
|
||||
REQUIRED_REPORTS["world_class_claim_guard"],
|
||||
"manifest.json",
|
||||
"agents/interface.yaml",
|
||||
"dist/manifest.json",
|
||||
"dist/targets/openai/adapter.json",
|
||||
"evidence/world_class/README.md",
|
||||
"security/permission_policy.json",
|
||||
REQUIRED_REPORTS["world_class_ledger"],
|
||||
],
|
||||
detail="The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted.",
|
||||
)
|
||||
checks.append(
|
||||
build_world_class_workflow_check(
|
||||
ledger=ledger,
|
||||
world_class_plan=world_class_plan,
|
||||
world_class_intake=world_class_intake,
|
||||
world_class_submission_review=world_class_submission_review,
|
||||
world_class_operator_runbook=world_class_operator_runbook,
|
||||
review_studio=review_studio,
|
||||
report_paths=REQUIRED_REPORTS,
|
||||
)
|
||||
)
|
||||
skill_os2_review = text_reports.get("skill_os2_review", "")
|
||||
checks.append(
|
||||
build_skill_os2_review_current_evidence_check(
|
||||
skill_dir=skill_dir,
|
||||
skill_os2_review=skill_os2_review,
|
||||
studio_summary=studio_summary if isinstance(studio_summary, dict) else {},
|
||||
trust_summary=trust_summary if isinstance(trust_summary, dict) else {},
|
||||
package_summary=package_summary if isinstance(package_summary, dict) else {},
|
||||
install_summary=install_summary if isinstance(install_summary, dict) else {},
|
||||
benchmark_summary=benchmark_summary if isinstance(benchmark_summary, dict) else {},
|
||||
context_stats=context_stats if isinstance(context_stats, dict) else {},
|
||||
required_text_reports=REQUIRED_TEXT_REPORTS,
|
||||
required_reports=REQUIRED_REPORTS,
|
||||
)
|
||||
)
|
||||
status_counts: dict[str, int] = {"pass": 0, "warn": 0, "fail": 0}
|
||||
for check in checks:
|
||||
status_counts[check["status"]] = status_counts.get(check["status"], 0) + 1
|
||||
summary = {
|
||||
"check_count": len(checks),
|
||||
"pass_count": status_counts.get("pass", 0),
|
||||
"warn_count": status_counts.get("warn", 0),
|
||||
"fail_count": status_counts.get("fail", 0),
|
||||
"decision": "consistent" if status_counts.get("fail", 0) == 0 else "evidence-drift-detected",
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": summary["fail_count"] == 0,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": summary,
|
||||
"status_counts": status_counts,
|
||||
"checks": checks,
|
||||
"artifacts": {
|
||||
"json": "reports/evidence_consistency.json",
|
||||
"markdown": "reports/evidence_consistency.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render cross-report evidence consistency checks.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json", default="reports/evidence_consistency.json")
|
||||
parser.add_argument("--output-md", default="reports/evidence_consistency.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(skill_dir, args.generated_at)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,6 +5,8 @@ from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from skill_ir_paths import find_skill_ir
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
@@ -18,14 +20,7 @@ def load_yaml(path: Path) -> dict:
|
||||
|
||||
|
||||
def find_ir(root: Path) -> tuple[dict, str]:
|
||||
candidates = [
|
||||
root / "reports" / "skill-ir.json",
|
||||
root / "skill-ir" / "examples" / f"{root.name}.json",
|
||||
]
|
||||
for path in candidates:
|
||||
if path.exists():
|
||||
return load_json(path), str(path.relative_to(root))
|
||||
return {}, "missing"
|
||||
return find_skill_ir(root, root.name, fallback_source="missing")
|
||||
|
||||
|
||||
def band_for(score: int) -> str:
|
||||
@@ -129,13 +124,21 @@ def render_markdown(report: dict) -> str:
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a portability score from neutral metadata, contracts, and snapshots.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=str(ROOT), help="Skill directory to inspect. Defaults to this repo root.")
|
||||
parser.add_argument("--output-json", default="reports/portability_score.json")
|
||||
parser.add_argument("--output-md", default="reports/portability_score.md")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = build_report(ROOT)
|
||||
output_json = ROOT / args.output_json
|
||||
output_md = ROOT / args.output_md
|
||||
skill_root = Path(args.skill_dir).resolve()
|
||||
report = build_report(skill_root)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_root / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_root / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
@@ -5,6 +5,8 @@ import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from reference_synthesis_markdown import render_markdown
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
@@ -507,100 +509,6 @@ def build_summary(skill_dir: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(summary: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# Reference Synthesis",
|
||||
"",
|
||||
f"Skill: `{summary['skill_name']}`",
|
||||
f"- Description: {summary['description']}",
|
||||
f"- Intent confidence: `{summary['intent_confidence']['score']}/100` (`{summary['intent_confidence']['band']}`)",
|
||||
"",
|
||||
"## Live GitHub Benchmarks",
|
||||
"",
|
||||
]
|
||||
if summary["github_benchmarks"]:
|
||||
for repo in summary["github_benchmarks"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {repo['name']}",
|
||||
f"- URL: {repo['url']}",
|
||||
f"- Stars: `{repo['stars']}`",
|
||||
]
|
||||
)
|
||||
for item in repo.get("borrow", []):
|
||||
lines.append(f"- Borrow: {item}")
|
||||
lines.append("")
|
||||
else:
|
||||
lines.append("- No live GitHub benchmarks are attached yet.")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(["## Curated World-Class Pattern Tracks", ""])
|
||||
for track in summary["source_tracks"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {track['name']}",
|
||||
f"- Type: `{track['source_type']}`",
|
||||
f"- Evidence mode: `{track['evidence_mode']}`",
|
||||
f"- Why relevant: {track['why_relevant']}",
|
||||
f"- Borrow: {track['borrow']}",
|
||||
f"- Avoid: {track['avoid']}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
|
||||
lines.extend(["## Borrow Now", ""])
|
||||
for item in summary["synthesis"]["borrow_now"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Avoid Now", ""])
|
||||
for item in summary["synthesis"]["avoid_now"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Pattern Gate", ""])
|
||||
pattern_gate = summary["synthesis"]["pattern_gate"]
|
||||
lines.append(f"- Summary: {pattern_gate['summary']}")
|
||||
lines.append(f"- Acceptance threshold: `{pattern_gate['threshold']}/4`")
|
||||
if pattern_gate["accepted"]:
|
||||
lines.append("- Accepted patterns:")
|
||||
for item in pattern_gate["accepted"][:5]:
|
||||
lines.append(
|
||||
f" - **{item['name']}**: {item['score']}/4 "
|
||||
f"({', '.join(item['passed'])})"
|
||||
)
|
||||
if pattern_gate["deferred"]:
|
||||
lines.append("- Deferred patterns:")
|
||||
for item in pattern_gate["deferred"][:5]:
|
||||
lines.append(
|
||||
f" - **{item['name']}**: missing {', '.join(item['missing']) or 'none'}"
|
||||
)
|
||||
|
||||
lines.extend(["", "## Default Recommendation", ""])
|
||||
lines.append(f"- Summary: {summary['synthesis']['recommendation']['summary']}")
|
||||
lines.append(f"- Why: {summary['synthesis']['recommendation']['why']}")
|
||||
lines.append(f"- User decision required: `{summary['synthesis']['recommendation']['user_decision_required']}`")
|
||||
|
||||
lines.extend(["", "## Visibility Mode", ""])
|
||||
lines.append(f"- Mode: `{summary['synthesis']['visibility']['mode']}`")
|
||||
if summary["synthesis"]["visibility"]["reasons"]:
|
||||
lines.append(f"- Reasons: {', '.join(summary['synthesis']['visibility']['reasons'])}")
|
||||
lines.append(f"- User note: {summary['synthesis']['visibility']['user_note']}")
|
||||
lines.append(f"- Reviewer note: {summary['synthesis']['visibility']['reviewer_note']}")
|
||||
|
||||
lines.extend(["", "## Conflict Check", ""])
|
||||
if summary["synthesis"]["conflicts"]:
|
||||
for conflict in summary["synthesis"]["conflicts"]:
|
||||
lines.append(f"- **{conflict['key']}**: {conflict['summary']}")
|
||||
else:
|
||||
lines.append("- No material design conflict detected. Keep the synthesis silent for the user.")
|
||||
|
||||
lines.extend(["", "## Quality Lift Thesis", ""])
|
||||
for item in summary["synthesis"]["quality_risers"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Decision Prompt", "", summary["synthesis"]["decision_prompt"], ""])
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def render_reference_synthesis(
|
||||
skill_dir: Path,
|
||||
output_md: Path | None = None,
|
||||
|
||||
@@ -18,11 +18,14 @@ VALID_GATES = {
|
||||
"context-budget",
|
||||
"runtime-matrix",
|
||||
"trust-report",
|
||||
"python-compat",
|
||||
"architecture-maintainability",
|
||||
"permission-gates",
|
||||
"permission-runtime",
|
||||
"skill-atlas",
|
||||
"operations-loop",
|
||||
"review-waivers",
|
||||
"world-class-evidence",
|
||||
"registry-audit",
|
||||
"release-notes",
|
||||
}
|
||||
|
||||
+234
-1060
File diff suppressed because it is too large
Load Diff
+13
-479
@@ -2,315 +2,24 @@
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from render_intent_confidence import render_intent_confidence
|
||||
from render_intent_dialogue import render_intent_dialogue
|
||||
from render_iteration_directions import render_iteration_directions
|
||||
from render_artifact_design_profile import render_artifact_design_profile
|
||||
from render_output_risk_profile import render_output_risk_profile
|
||||
from render_prompt_quality_profile import render_prompt_quality_profile
|
||||
from render_reference_scan import render_reference_scan
|
||||
from render_reference_synthesis import render_reference_synthesis
|
||||
from render_skill_overview import render_skill_overview
|
||||
from review_viewer_data import (
|
||||
architecture_steps,
|
||||
benchmark_cards,
|
||||
compare_rows,
|
||||
ensure_report_inputs,
|
||||
evidence_readiness,
|
||||
synthesis_cards,
|
||||
variant_diff_cards,
|
||||
)
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
ASSETS_DIR = Path(__file__).resolve().parent.parent / "assets"
|
||||
|
||||
|
||||
def load_feedback_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "feedback-log.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_baseline_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "baseline-compare.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_specific_compare(skill_dir: Path) -> dict:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "description_optimization.json",
|
||||
skill_dir.parent / "optimization" / "reports" / "description_optimization.json",
|
||||
]
|
||||
for path in candidates:
|
||||
payload = load_json(path)
|
||||
if isinstance(payload, dict) and payload:
|
||||
return payload
|
||||
return {}
|
||||
|
||||
|
||||
def load_specific_promotion(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "promotion_decisions.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_benchmark_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "github-benchmark-scan.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_reference_synthesis_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "reference-synthesis.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_output_risk_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "output-risk-profile.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_artifact_design_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "artifact-design-profile.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_prompt_quality_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "prompt-quality-profile.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def ensure_report_inputs(skill_dir: Path) -> dict:
|
||||
overview_json = skill_dir / "reports" / "skill-overview.json"
|
||||
intent_confidence_json = skill_dir / "reports" / "intent-confidence.json"
|
||||
intent_json = skill_dir / "reports" / "intent-dialogue.json"
|
||||
reference_json = skill_dir / "reports" / "reference-scan.json"
|
||||
reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json"
|
||||
output_risk_json = skill_dir / "reports" / "output-risk-profile.json"
|
||||
artifact_design_json = skill_dir / "reports" / "artifact-design-profile.json"
|
||||
prompt_quality_json = skill_dir / "reports" / "prompt-quality-profile.json"
|
||||
directions_json = skill_dir / "reports" / "iteration-directions.json"
|
||||
|
||||
overview_payload = load_json(overview_json) if overview_json.exists() else {}
|
||||
intent_confidence_payload = load_json(intent_confidence_json) if intent_confidence_json.exists() else {}
|
||||
intent_payload = load_json(intent_json) if intent_json.exists() else {}
|
||||
reference_payload = load_json(reference_json) if reference_json.exists() else {}
|
||||
reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {}
|
||||
output_risk_payload = load_json(output_risk_json) if output_risk_json.exists() else {}
|
||||
artifact_design_payload = load_json(artifact_design_json) if artifact_design_json.exists() else {}
|
||||
prompt_quality_payload = load_json(prompt_quality_json) if prompt_quality_json.exists() else {}
|
||||
directions_payload = load_json(directions_json) if directions_json.exists() else {}
|
||||
|
||||
intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"]
|
||||
intent = intent_payload or render_intent_dialogue(skill_dir)["summary"]
|
||||
reference = reference_payload or render_reference_scan(skill_dir, [])["summary"]
|
||||
reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"]
|
||||
output_risk = output_risk_payload or render_output_risk_profile(skill_dir)["summary"]
|
||||
artifact_design = artifact_design_payload or render_artifact_design_profile(skill_dir)["summary"]
|
||||
prompt_quality = prompt_quality_payload or render_prompt_quality_profile(skill_dir)["summary"]
|
||||
iteration_payload = directions_payload or render_iteration_directions(skill_dir)
|
||||
iteration = iteration_payload.get("summary", {})
|
||||
overview = overview_payload or render_skill_overview(skill_dir)["summary"]
|
||||
feedback = load_feedback_summary(skill_dir)
|
||||
baseline = load_baseline_summary(skill_dir)
|
||||
compare = load_specific_compare(skill_dir)
|
||||
promotion = load_specific_promotion(skill_dir)
|
||||
benchmark = load_benchmark_summary(skill_dir)
|
||||
reference_synthesis = load_reference_synthesis_summary(skill_dir)
|
||||
output_risk = load_output_risk_summary(skill_dir) or output_risk
|
||||
artifact_design = load_artifact_design_summary(skill_dir) or artifact_design
|
||||
prompt_quality = load_prompt_quality_summary(skill_dir) or prompt_quality
|
||||
return {
|
||||
"overview": overview,
|
||||
"intent_confidence": intent_confidence,
|
||||
"intent": intent,
|
||||
"reference": reference,
|
||||
"iteration": iteration_payload,
|
||||
"feedback": feedback,
|
||||
"baseline": baseline,
|
||||
"compare": compare,
|
||||
"promotion": promotion,
|
||||
"benchmark": benchmark,
|
||||
"reference_synthesis": reference_synthesis,
|
||||
"output_risk": output_risk,
|
||||
"artifact_design": artifact_design,
|
||||
"prompt_quality": prompt_quality,
|
||||
}
|
||||
|
||||
|
||||
def architecture_steps(overview: dict) -> list[dict]:
|
||||
logic = overview.get("logic_steps", [])[:3]
|
||||
usage = overview.get("usage_steps", [])[:2]
|
||||
return [
|
||||
{"label": "Inputs", "detail": "workflow, prompt, transcript, docs, or notes"},
|
||||
{"label": "Boundary", "detail": overview.get("description", "Define the recurring job and exclusions.")},
|
||||
{"label": "Logic", "detail": "; ".join(logic) if logic else "Understand, execute, and validate."},
|
||||
{"label": "Usage", "detail": "; ".join(usage) if usage else "Load the skill and follow the workflow."},
|
||||
{"label": "Next", "detail": "Review the top iteration directions before growing the package."},
|
||||
]
|
||||
|
||||
|
||||
def compare_rows(compare: dict) -> list[dict]:
|
||||
if not compare:
|
||||
return []
|
||||
rows = []
|
||||
items = [
|
||||
("Baseline", compare.get("baseline", {})),
|
||||
("Current", compare.get("current_candidate", {})),
|
||||
(compare.get("winner", {}).get("label", "Winner"), compare.get("winner", {})),
|
||||
]
|
||||
for label, payload in items:
|
||||
if not payload:
|
||||
continue
|
||||
dev = payload.get("dev", {})
|
||||
holdout = payload.get("holdout", {})
|
||||
rows.append(
|
||||
{
|
||||
"label": label,
|
||||
"tokens": payload.get("estimated_tokens", 0),
|
||||
"dev_errors": dev.get("total_errors", 0),
|
||||
"holdout_errors": holdout.get("total_errors", 0),
|
||||
"strategy": payload.get("strategy", "existing"),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def benchmark_cards(benchmark: dict) -> list[dict]:
|
||||
cards = []
|
||||
for repo in benchmark.get("repositories", [])[:3]:
|
||||
cards.append(
|
||||
{
|
||||
"name": repo.get("full_name", "Unknown repo"),
|
||||
"borrow": repo.get("borrow", [])[:2],
|
||||
"avoid": repo.get("avoid", [])[:1],
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def synthesis_cards(reference_synthesis: dict) -> list[dict]:
|
||||
cards = []
|
||||
for track in reference_synthesis.get("source_tracks", [])[:3]:
|
||||
cards.append(
|
||||
{
|
||||
"name": track.get("name", "Unknown track"),
|
||||
"borrow": [track.get("borrow", "")] if track.get("borrow") else [],
|
||||
"avoid": [track.get("avoid", "")] if track.get("avoid") else [],
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def split_sentences(text: str) -> list[str]:
|
||||
if not text:
|
||||
return []
|
||||
parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", " ".join(text.split())) if item.strip()]
|
||||
return parts
|
||||
|
||||
|
||||
def metric_delta(current: int | float, baseline: int | float) -> str:
|
||||
delta = current - baseline
|
||||
if delta == 0:
|
||||
return "0"
|
||||
return f"{delta:+}"
|
||||
|
||||
|
||||
def variant_diff_cards(compare: dict) -> list[dict]:
|
||||
baseline = compare.get("baseline", {})
|
||||
current = compare.get("current_candidate", {})
|
||||
winner = compare.get("winner", {})
|
||||
variants = [
|
||||
("Baseline", baseline),
|
||||
("Current", current),
|
||||
(f"Winner — {winner.get('label', 'Winner')}", winner),
|
||||
]
|
||||
baseline_sentences = split_sentences(baseline.get("description", ""))
|
||||
baseline_set = set(baseline_sentences)
|
||||
baseline_dev = baseline.get("dev", {}).get("total_errors", 0)
|
||||
baseline_holdout = baseline.get("holdout", {}).get("total_errors", 0)
|
||||
cards = []
|
||||
seen = set()
|
||||
for label, payload in variants:
|
||||
if not payload:
|
||||
continue
|
||||
unique_key = (payload.get("description"), payload.get("strategy"), label)
|
||||
if unique_key in seen:
|
||||
continue
|
||||
seen.add(unique_key)
|
||||
description = payload.get("description", "")
|
||||
sentences = split_sentences(description)
|
||||
sentence_set = set(sentences)
|
||||
added = [item for item in sentences if item not in baseline_set][:3]
|
||||
removed = [item for item in baseline_sentences if item not in sentence_set][:2]
|
||||
dev_errors = payload.get("dev", {}).get("total_errors", 0)
|
||||
holdout_errors = payload.get("holdout", {}).get("total_errors", 0)
|
||||
cards.append(
|
||||
{
|
||||
"label": label,
|
||||
"strategy": payload.get("strategy", "existing"),
|
||||
"description": description,
|
||||
"tokens": payload.get("estimated_tokens", 0),
|
||||
"dev_errors": dev_errors,
|
||||
"holdout_errors": holdout_errors,
|
||||
"token_delta": metric_delta(payload.get("estimated_tokens", 0), baseline.get("estimated_tokens", 0)),
|
||||
"dev_delta": metric_delta(dev_errors, baseline_dev),
|
||||
"holdout_delta": metric_delta(holdout_errors, baseline_holdout),
|
||||
"added": added if label != "Baseline" else baseline_sentences[:3],
|
||||
"removed": removed,
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def evidence_readiness(report: dict) -> dict:
|
||||
intent_confidence = report.get("intent_confidence", {})
|
||||
reference_synthesis = report.get("reference_synthesis", {})
|
||||
output_risk = report.get("output_risk", {})
|
||||
artifact_design = report.get("artifact_design", {})
|
||||
prompt_quality = report.get("prompt_quality", {})
|
||||
benchmark = report.get("benchmark", {})
|
||||
synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {}
|
||||
pattern_gate = synthesis.get("pattern_gate", {}) if isinstance(synthesis, dict) else {}
|
||||
accepted_patterns = pattern_gate.get("accepted", []) if isinstance(pattern_gate, dict) else []
|
||||
conflicts = synthesis.get("conflicts", []) if isinstance(synthesis, dict) else []
|
||||
checks = [
|
||||
{
|
||||
"label": "Intent clarity",
|
||||
"status": "ready" if intent_confidence.get("gate_passed") else "needs review",
|
||||
"detail": f"{intent_confidence.get('score', 0)}/100 intent confidence.",
|
||||
},
|
||||
{
|
||||
"label": "Benchmark coverage",
|
||||
"status": "ready" if len(benchmark.get("repositories", [])) >= 2 else "needs evidence",
|
||||
"detail": f"{len(benchmark.get('repositories', []))} GitHub benchmark repositories attached.",
|
||||
},
|
||||
{
|
||||
"label": "Pattern gate",
|
||||
"status": "ready" if accepted_patterns else "needs review",
|
||||
"detail": pattern_gate.get("summary", "No pattern gate summary attached."),
|
||||
},
|
||||
{
|
||||
"label": "Conflict handling",
|
||||
"status": "ready" if not conflicts else "decision needed",
|
||||
"detail": "No material conflicts detected." if not conflicts else conflicts[0].get("summary", "Conflict detected."),
|
||||
},
|
||||
{
|
||||
"label": "Output risk profile",
|
||||
"status": "ready" if output_risk.get("risk_families") else "needs review",
|
||||
"detail": f"{len(output_risk.get('risk_families', []))} output risk families attached.",
|
||||
},
|
||||
{
|
||||
"label": "Artifact design profile",
|
||||
"status": "ready" if artifact_design.get("primary_artifact") else "needs review",
|
||||
"detail": artifact_design.get("primary_artifact", {}).get("direction", "No artifact design profile attached."),
|
||||
},
|
||||
{
|
||||
"label": "Prompt quality profile",
|
||||
"status": "ready" if prompt_quality.get("quality_matrix") else "needs review",
|
||||
"detail": f"{prompt_quality.get('overall_quality_score', 0)}/100 prompt-facing quality score.",
|
||||
},
|
||||
]
|
||||
ready_count = sum(1 for item in checks if item["status"] == "ready")
|
||||
return {
|
||||
"score": int(ready_count / len(checks) * 100),
|
||||
"checks": checks,
|
||||
"reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery.",
|
||||
}
|
||||
def review_viewer_css() -> str:
|
||||
return (ASSETS_DIR / "review-viewer.css").read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def render_html(report: dict) -> str:
|
||||
@@ -566,182 +275,7 @@ def render_html(report: dict) -> str:
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{html.escape(overview.get('display_name', overview.get('title', 'Skill Review')))} Review Viewer</title>
|
||||
<style>
|
||||
:root {{
|
||||
--text: #111111;
|
||||
--muted: #666666;
|
||||
--line: #e8e8e8;
|
||||
--soft: #f6f6f4;
|
||||
--white: #ffffff;
|
||||
}}
|
||||
* {{ box-sizing: border-box; }}
|
||||
body {{
|
||||
margin: 0;
|
||||
background: var(--white);
|
||||
color: var(--text);
|
||||
font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
}}
|
||||
.page {{
|
||||
max-width: 1120px;
|
||||
margin: 0 auto;
|
||||
padding: 48px 32px 72px;
|
||||
}}
|
||||
.hero {{
|
||||
padding-bottom: 28px;
|
||||
border-bottom: 1px solid var(--line);
|
||||
margin-bottom: 28px;
|
||||
}}
|
||||
h1, h2, h3 {{
|
||||
margin: 0 0 12px;
|
||||
letter-spacing: -0.02em;
|
||||
font-weight: 600;
|
||||
}}
|
||||
h1 {{ font-size: 40px; line-height: 1.08; }}
|
||||
h2 {{ font-size: 22px; margin-top: 34px; }}
|
||||
h3 {{ font-size: 16px; }}
|
||||
p, li, span {{ font-size: 15px; }}
|
||||
.lede {{
|
||||
max-width: 860px;
|
||||
font-size: 18px;
|
||||
color: var(--muted);
|
||||
margin: 0 0 18px;
|
||||
}}
|
||||
.meta {{
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
flex-wrap: wrap;
|
||||
margin-top: 16px;
|
||||
}}
|
||||
.meta span {{
|
||||
border: 1px solid var(--line);
|
||||
padding: 6px 10px;
|
||||
background: var(--soft);
|
||||
}}
|
||||
.arch-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||
gap: 12px;
|
||||
margin-top: 16px;
|
||||
}}
|
||||
.arch-step, .panel, .direction-card, .baseline-box {{
|
||||
border: 1px solid var(--line);
|
||||
background: var(--white);
|
||||
}}
|
||||
.arch-step {{
|
||||
padding: 14px;
|
||||
min-height: 132px;
|
||||
}}
|
||||
.step-label {{
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.08em;
|
||||
color: var(--muted);
|
||||
margin-bottom: 10px;
|
||||
}}
|
||||
.step-detail {{
|
||||
font-size: 14px;
|
||||
}}
|
||||
.grid {{
|
||||
display: grid;
|
||||
grid-template-columns: 1.1fr 0.9fr;
|
||||
gap: 18px;
|
||||
margin-top: 16px;
|
||||
}}
|
||||
.panel {{
|
||||
padding: 18px;
|
||||
}}
|
||||
.panel ul {{
|
||||
margin: 0;
|
||||
padding-left: 18px;
|
||||
}}
|
||||
.direction-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
margin-top: 16px;
|
||||
}}
|
||||
.variant-grid {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
margin-top: 16px;
|
||||
}}
|
||||
.direction-card {{
|
||||
padding: 18px;
|
||||
}}
|
||||
.direction-card ul {{
|
||||
margin: 12px 0;
|
||||
padding-left: 18px;
|
||||
}}
|
||||
.minor {{
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}}
|
||||
.variant-card {{
|
||||
border: 1px solid var(--line);
|
||||
background: var(--white);
|
||||
padding: 18px;
|
||||
}}
|
||||
.variant-head {{
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
align-items: baseline;
|
||||
}}
|
||||
.variant-head span {{
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
}}
|
||||
.variant-description {{
|
||||
margin: 14px 0;
|
||||
padding-left: 14px;
|
||||
border-left: 2px solid var(--line);
|
||||
color: var(--text);
|
||||
}}
|
||||
.variant-metrics {{
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-bottom: 14px;
|
||||
}}
|
||||
.variant-metrics span {{
|
||||
border: 1px solid var(--line);
|
||||
background: var(--soft);
|
||||
padding: 6px 10px;
|
||||
font-size: 12px;
|
||||
}}
|
||||
.variant-cues p {{
|
||||
margin: 8px 0 6px;
|
||||
}}
|
||||
.variant-cues ul {{
|
||||
margin: 0 0 12px;
|
||||
padding-left: 18px;
|
||||
}}
|
||||
table {{
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
margin-top: 14px;
|
||||
font-size: 14px;
|
||||
}}
|
||||
th, td {{
|
||||
border-top: 1px solid var(--line);
|
||||
text-align: left;
|
||||
padding: 10px 8px;
|
||||
vertical-align: top;
|
||||
}}
|
||||
th {{
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.06em;
|
||||
color: var(--muted);
|
||||
}}
|
||||
@media (max-width: 1000px) {{
|
||||
.arch-grid, .direction-grid, .variant-grid, .grid {{
|
||||
grid-template-columns: 1fr;
|
||||
}}
|
||||
}}
|
||||
{review_viewer_css()}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -2,26 +2,17 @@
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import date
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from review_studio_gates import REVIEW_STUDIO_GATE_KEYS
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
KNOWN_GATE_KEYS = {
|
||||
"intent-canvas",
|
||||
"trigger-lab",
|
||||
"output-lab",
|
||||
"context-budget",
|
||||
"runtime-matrix",
|
||||
"trust-report",
|
||||
"permission-gates",
|
||||
"permission-runtime",
|
||||
"skill-atlas",
|
||||
"operations-loop",
|
||||
"registry-audit",
|
||||
"release-notes",
|
||||
}
|
||||
NON_WAIVABLE_GATE_KEYS = {"review-waivers", "world-class-evidence"}
|
||||
WAIVERABLE_GATE_KEYS = REVIEW_STUDIO_GATE_KEYS - NON_WAIVABLE_GATE_KEYS
|
||||
KNOWN_GATE_KEYS = WAIVERABLE_GATE_KEYS
|
||||
VALID_DECISIONS = {"accepted-risk", "false-positive", "temporary-exception"}
|
||||
MIN_REASON_CHARS = 20
|
||||
|
||||
@@ -139,6 +130,88 @@ def validate_waivers(waivers: list[dict[str, Any]], today: date) -> tuple[list[d
|
||||
return normalized, failures, warnings
|
||||
|
||||
|
||||
def output_lab_candidate(skill_dir: Path, covered_gate_keys: set[str], today: date) -> dict[str, Any] | None:
|
||||
output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json").get("summary", {})
|
||||
output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {})
|
||||
output_review = load_json(skill_dir / "reports" / "output_review_adjudication.json").get("summary", {})
|
||||
if not output_quality and not output_execution and not output_review:
|
||||
return None
|
||||
pending = int(output_review.get("pending_count", 0) or 0)
|
||||
model_executed = int(output_execution.get("model_executed_count", 0) or 0)
|
||||
failure_count = int(output_quality.get("failure_count", 0) or 0)
|
||||
if pending == 0 and model_executed > 0 and failure_count == 0:
|
||||
return None
|
||||
status = "covered" if "output-lab" in covered_gate_keys else "needs-reviewer-decision"
|
||||
return {
|
||||
"gate_key": "output-lab",
|
||||
"label": "Output Lab",
|
||||
"status": status,
|
||||
"waiver_allowed": True,
|
||||
"decision_options": sorted(VALID_DECISIONS),
|
||||
"risk_summary": (
|
||||
f"review pending {pending}; model-executed {model_executed}; "
|
||||
f"output failures {failure_count}"
|
||||
),
|
||||
"required_review": [
|
||||
"Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.",
|
||||
"Reviewer names the release scope and expiry date.",
|
||||
"Reviewer links output_review_adjudication or output_execution evidence.",
|
||||
],
|
||||
"suggested_evidence": "reports/output_review_adjudication.md",
|
||||
"suggested_command": (
|
||||
"python3 scripts/yao.py review-waivers . --add-waiver "
|
||||
"--gate-key output-lab --reviewer \"<reviewer>\" "
|
||||
"--reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" "
|
||||
f"--expires-at {(today + timedelta(days=365)).isoformat()} "
|
||||
"--evidence reports/output_review_adjudication.md"
|
||||
),
|
||||
"world_class_boundary": "Does not count as provider, human, or public world-class completion evidence.",
|
||||
}
|
||||
|
||||
|
||||
def world_class_boundary(skill_dir: Path) -> dict[str, Any] | None:
|
||||
ledger_summary = load_json(skill_dir / "reports" / "world_class_evidence_ledger.json").get("summary", {})
|
||||
if not ledger_summary:
|
||||
return None
|
||||
pending = int(ledger_summary.get("pending_count", 0) or 0)
|
||||
if pending == 0 and ledger_summary.get("ready_to_claim_world_class") is True:
|
||||
return None
|
||||
return {
|
||||
"gate_key": "world-class-evidence",
|
||||
"label": "World-Class Evidence",
|
||||
"status": "cannot-waive",
|
||||
"waiver_allowed": False,
|
||||
"risk_summary": (
|
||||
f"{pending} pending evidence entries; "
|
||||
f"{ledger_summary.get('human_pending_count', 0)} human pending; "
|
||||
f"{ledger_summary.get('external_pending_count', 0)} external pending"
|
||||
),
|
||||
"required_review": [
|
||||
"Do not use a waiver to claim public world-class readiness.",
|
||||
"Either submit accepted ledger evidence or state that this release does not claim world-class completion.",
|
||||
"Keep claim guard active until ledger summary.ready_to_claim_world_class is true.",
|
||||
],
|
||||
"suggested_evidence": "reports/world_class_evidence_ledger.md",
|
||||
"suggested_command": (
|
||||
"python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions "
|
||||
"&& python3 scripts/yao.py world-class-claim-guard ."
|
||||
),
|
||||
"world_class_boundary": "Non-waivable completion boundary.",
|
||||
}
|
||||
|
||||
|
||||
def build_waiver_candidates(skill_dir: Path, covered_gate_keys: list[str], today: date) -> list[dict[str, Any]]:
|
||||
covered = set(covered_gate_keys)
|
||||
candidates = []
|
||||
output_candidate = output_lab_candidate(skill_dir, covered, today)
|
||||
if output_candidate:
|
||||
candidates.append(output_candidate)
|
||||
world_boundary = world_class_boundary(skill_dir)
|
||||
if world_boundary:
|
||||
candidates.append(world_boundary)
|
||||
return candidates
|
||||
|
||||
|
||||
def render_report(
|
||||
skill_dir: Path,
|
||||
waivers_json: Path | None = None,
|
||||
@@ -163,6 +236,11 @@ def render_report(
|
||||
expired = [item for item in waivers if item["status"] == "expired"]
|
||||
invalid = [item for item in waivers if item["status"] == "invalid"]
|
||||
covered_gate_keys = sorted({item["gate_key"] for item in active})
|
||||
waiver_candidates = build_waiver_candidates(skill_dir, covered_gate_keys, today)
|
||||
waiverable_open = [
|
||||
item for item in waiver_candidates if item["waiver_allowed"] and item["status"] != "covered"
|
||||
]
|
||||
non_waivable = [item for item in waiver_candidates if not item["waiver_allowed"]]
|
||||
report = {
|
||||
"schema_version": "1.0",
|
||||
"ok": not failures,
|
||||
@@ -175,14 +253,21 @@ def render_report(
|
||||
"invalid_count": len(invalid),
|
||||
"covered_gate_count": len(covered_gate_keys),
|
||||
"covered_gate_keys": covered_gate_keys,
|
||||
"waiver_candidate_count": len(waiver_candidates),
|
||||
"waiverable_open_count": len(waiverable_open),
|
||||
"non_waivable_count": len(non_waivable),
|
||||
},
|
||||
"policy": {
|
||||
"blocker_waivers_allowed": False,
|
||||
"minimum_reason_chars": MIN_REASON_CHARS,
|
||||
"expires_required": True,
|
||||
"review_studio_gate_keys": sorted(REVIEW_STUDIO_GATE_KEYS),
|
||||
"known_gate_keys": sorted(KNOWN_GATE_KEYS),
|
||||
"waiverable_gate_keys": sorted(WAIVERABLE_GATE_KEYS),
|
||||
"non_waivable_gate_keys": sorted(NON_WAIVABLE_GATE_KEYS),
|
||||
},
|
||||
"waivers": waivers,
|
||||
"waiver_candidates": waiver_candidates,
|
||||
"failures": failures,
|
||||
"warnings": warnings,
|
||||
"artifacts": {
|
||||
@@ -207,12 +292,19 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
f"- Expired: `{summary['expired_count']}`",
|
||||
f"- Invalid: `{summary['invalid_count']}`",
|
||||
f"- Covered gates: `{', '.join(summary['covered_gate_keys']) or 'none'}`",
|
||||
f"- Waiver candidates: `{summary['waiver_candidate_count']}`",
|
||||
f"- Open waiverable candidates: `{summary['waiverable_open_count']}`",
|
||||
f"- Non-waivable boundaries: `{summary['non_waivable_count']}`",
|
||||
"",
|
||||
"## Policy",
|
||||
"",
|
||||
"- Blocker waivers allowed: `False`",
|
||||
f"- Minimum reason chars: `{report['policy']['minimum_reason_chars']}`",
|
||||
"- Expiry is required for every waiver.",
|
||||
"- World-class evidence completion cannot be waived; it can only be proven by accepted ledger evidence.",
|
||||
f"- Review Studio gates: `{', '.join(report['policy']['review_studio_gate_keys'])}`",
|
||||
f"- Waiverable gates: `{', '.join(report['policy']['waiverable_gate_keys'])}`",
|
||||
f"- Non-waivable gates: `{', '.join(report['policy']['non_waivable_gate_keys'])}`",
|
||||
"",
|
||||
"## Waivers",
|
||||
"",
|
||||
@@ -226,6 +318,28 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines.append(
|
||||
f"| `{item['id']}` | `{item['gate_key']}` | `{item['decision']}` | {item['reviewer']} | `{item['status']}` | `{item['expires_at']}` | {reason} |"
|
||||
)
|
||||
lines.extend(["", "## Candidate Actions", ""])
|
||||
candidates = report.get("waiver_candidates", [])
|
||||
if not candidates:
|
||||
lines.append("- None")
|
||||
else:
|
||||
lines.extend(["| Gate | Status | Waiver | Risk | Evidence |", "| --- | --- | --- | --- | --- |"])
|
||||
for item in candidates:
|
||||
risk = str(item.get("risk_summary", "")).replace("|", "\\|")
|
||||
lines.append(
|
||||
f"| `{item['gate_key']}` | `{item['status']}` | `{str(item['waiver_allowed']).lower()}` | {risk} | `{item.get('suggested_evidence', '')}` |"
|
||||
)
|
||||
for item in candidates:
|
||||
lines.extend(["", f"### {item['label']}", ""])
|
||||
lines.append(f"- gate: `{item['gate_key']}`")
|
||||
lines.append(f"- status: `{item['status']}`")
|
||||
lines.append(f"- waiver allowed: `{str(item['waiver_allowed']).lower()}`")
|
||||
lines.append(f"- risk: {item['risk_summary']}")
|
||||
lines.append(f"- evidence: `{item.get('suggested_evidence', '')}`")
|
||||
lines.append(f"- verification: `{item.get('suggested_command', '')}`")
|
||||
lines.append(f"- world-class boundary: {item.get('world_class_boundary', '')}")
|
||||
lines.extend(["", "#### Required Review", ""])
|
||||
lines.extend(f"- {review}" for review in item.get("required_review", []))
|
||||
lines.extend(["", "## Failures", ""])
|
||||
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
|
||||
lines.extend(["", "## Warnings", ""])
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from render_skill_overview import render_html
|
||||
from skill_report_model import build_report_model
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders the first-class skill interpretation report while reusing the Skill Overview v2 model and layout."
|
||||
|
||||
|
||||
def build_interpretation_model(skill_dir: Path) -> dict:
|
||||
summary = build_report_model(skill_dir)
|
||||
contract = dict(summary.get("report_contract", {}))
|
||||
contract.update(
|
||||
{
|
||||
"schema_version": "2.0",
|
||||
"report_kind": "skill-interpretation",
|
||||
"canonical_overview_report": "reports/skill-overview.html",
|
||||
"html_report": "reports/skill-interpretation.html",
|
||||
"json_report": "reports/skill-interpretation.json",
|
||||
"layout": "kami-white-audit-v2",
|
||||
"default_language": "zh-CN",
|
||||
"languages": ["zh-CN", "en"],
|
||||
"purpose": "Explain the generated skill's role, principles, usage scenarios, trigger contract, inputs, outputs, quality evidence, risks, assets, highlights, and next upgrade directions.",
|
||||
}
|
||||
)
|
||||
summary["report_contract"] = contract
|
||||
summary["interpretation_contract"] = {
|
||||
"schema_version": "2.0",
|
||||
"source_model": "skill-overview-v2",
|
||||
"source_model_reused": True,
|
||||
"overview_report": "reports/skill-overview.html",
|
||||
"html_report": "reports/skill-interpretation.html",
|
||||
"json_report": "reports/skill-interpretation.json",
|
||||
"default_language": "zh-CN",
|
||||
"languages": ["zh-CN", "en"],
|
||||
"includes": [
|
||||
"skill role",
|
||||
"principles",
|
||||
"usage scenarios",
|
||||
"trigger contract",
|
||||
"inputs and outputs",
|
||||
"quality evidence",
|
||||
"risk governance",
|
||||
"package assets",
|
||||
"highlights",
|
||||
"upgrade roadmap",
|
||||
],
|
||||
}
|
||||
deliverables = summary.get("skill_summary", {}).get("deliverables", [])
|
||||
for artifact in ["reports/skill-interpretation.html", "reports/skill-interpretation.json"]:
|
||||
if artifact not in deliverables:
|
||||
deliverables.append(artifact)
|
||||
if "skill_summary" in summary:
|
||||
summary["skill_summary"]["deliverables"] = deliverables
|
||||
return summary
|
||||
|
||||
|
||||
def render_skill_interpretation(skill_dir: Path, output_html: Path | None = None, output_json: Path | None = None) -> dict:
|
||||
skill_dir = skill_dir.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
reports_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
output_html = output_html or reports_dir / "skill-interpretation.html"
|
||||
output_json = output_json or reports_dir / "skill-interpretation.json"
|
||||
|
||||
summary = build_interpretation_model(skill_dir)
|
||||
output_html.write_text(render_html(summary), encoding="utf-8")
|
||||
output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"skill_dir": str(skill_dir),
|
||||
"artifacts": {
|
||||
"html": str(output_html),
|
||||
"json": str(output_json),
|
||||
"overview_html": str(skill_dir / "reports" / "skill-overview.html"),
|
||||
},
|
||||
"summary": {
|
||||
"name": summary.get("name"),
|
||||
"report_kind": summary.get("report_contract", {}).get("report_kind"),
|
||||
"default_language": summary.get("report_contract", {}).get("default_language"),
|
||||
"section_count": len(summary.get("report_contract", {}).get("nav_labels", [])),
|
||||
"source_model_reused": summary.get("interpretation_contract", {}).get("source_model_reused"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render the first-class HTML skill interpretation report for a skill package.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-html")
|
||||
parser.add_argument("--output-json")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = render_skill_interpretation(
|
||||
Path(args.skill_dir),
|
||||
Path(args.output_html).resolve() if args.output_html else None,
|
||||
Path(args.output_json).resolve() if args.output_json else None,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
STATUS_LABELS = {
|
||||
"pass": "pass",
|
||||
"warn": "warn",
|
||||
"human_required": "human-required",
|
||||
"external_required": "external-required",
|
||||
"missing": "missing",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def evidence(skill_dir: Path, *paths: str) -> list[dict[str, Any]]:
|
||||
items = []
|
||||
for rel in paths:
|
||||
path = skill_dir / rel
|
||||
items.append({"path": rel, "exists": path.exists()})
|
||||
return items
|
||||
|
||||
|
||||
def status_from(condition: bool, fallback: str = "missing") -> str:
|
||||
return "pass" if condition else fallback
|
||||
|
||||
|
||||
def audit_item(
|
||||
key: str,
|
||||
label: str,
|
||||
status: str,
|
||||
current: str,
|
||||
target: str,
|
||||
evidence_items: list[dict[str, Any]],
|
||||
next_action: str,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"key": key,
|
||||
"label": label,
|
||||
"status": status,
|
||||
"current": current,
|
||||
"target": target,
|
||||
"evidence": evidence_items,
|
||||
"next_action": next_action,
|
||||
}
|
||||
|
||||
|
||||
def count_sources(events_summary: dict[str, Any]) -> dict[str, int]:
|
||||
values = events_summary.get("source_types", {})
|
||||
return values if isinstance(values, dict) else {}
|
||||
|
||||
|
||||
def build_audit(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
reports = skill_dir / "reports"
|
||||
skill_ir = load_json(skill_dir / "skill-ir" / "examples" / "yao-meta-skill.json")
|
||||
compiled = load_json(reports / "compiled_targets.json")
|
||||
output_quality = load_json(reports / "output_quality_scorecard.json")
|
||||
output_execution = load_json(reports / "output_execution_runs.json")
|
||||
output_review = load_json(reports / "output_review_adjudication.json")
|
||||
benchmark_reproducibility = load_json(reports / "benchmark_reproducibility.json")
|
||||
conformance = load_json(reports / "conformance_matrix.json")
|
||||
trust = load_json(reports / "security_trust_report.json")
|
||||
runtime_permissions = load_json(reports / "runtime_permission_probes.json")
|
||||
atlas = load_json(reports / "skill_atlas.json")
|
||||
registry = load_json(reports / "registry_audit.json")
|
||||
package_verification = load_json(reports / "package_verification.json")
|
||||
install = load_json(reports / "install_simulation.json")
|
||||
review_studio = load_json(reports / "review-studio.json")
|
||||
adoption = load_json(reports / "adoption_drift_report.json")
|
||||
telemetry_hooks = load_json(reports / "telemetry_hook_recipes.json")
|
||||
|
||||
compiled_summary = compiled.get("summary", {})
|
||||
output_summary = output_quality.get("summary", {})
|
||||
execution_summary = output_execution.get("summary", {})
|
||||
review_summary = output_review.get("summary", {})
|
||||
benchmark_summary = benchmark_reproducibility.get("summary", {})
|
||||
conformance_summary = conformance.get("summary", {})
|
||||
trust_summary = trust.get("summary", {})
|
||||
permission_summary = runtime_permissions.get("summary", {})
|
||||
atlas_summary = atlas.get("summary", {})
|
||||
package_summary = package_verification.get("summary", {})
|
||||
install_summary = install.get("summary", {})
|
||||
studio_summary = review_studio.get("summary", {})
|
||||
adoption_summary = adoption.get("summary", {})
|
||||
telemetry_summary = telemetry_hooks.get("summary", {})
|
||||
sources = count_sources(adoption_summary)
|
||||
|
||||
items = [
|
||||
audit_item(
|
||||
"skill-ir",
|
||||
"Skill IR",
|
||||
status_from(
|
||||
skill_ir.get("schema_version") == "2.0.0"
|
||||
and bool((skill_dir / "skill-ir" / "schema.json").exists())
|
||||
and bool((skill_dir / "skill-ir" / "examples" / "yao-meta-skill.json").exists())
|
||||
),
|
||||
f"schema {skill_ir.get('schema_version', 'missing')}; targets {len(skill_ir.get('targets', []))}",
|
||||
"2.0 schema, root export, and target-neutral contract evidence",
|
||||
evidence(skill_dir, "skill-ir/schema.json", "skill-ir/examples/yao-meta-skill.json", "references/skill-ir-method.md"),
|
||||
"Keep IR as the source before target packaging.",
|
||||
),
|
||||
audit_item(
|
||||
"target-compiler",
|
||||
"Target Compiler",
|
||||
status_from(compiled_summary.get("target_count", 0) >= 5 and compiled_summary.get("pass_count") == compiled_summary.get("target_count")),
|
||||
f"{compiled_summary.get('pass_count', 0)}/{compiled_summary.get('target_count', 0)} targets pass",
|
||||
"OpenAI, Claude, generic, Agent Skills compatible, and VS Code contracts generated from IR",
|
||||
evidence(skill_dir, "scripts/compile_skill.py", "reports/compiled_targets.json", "tests/verify_compile_skill.py"),
|
||||
"Deepen target-native transforms when provider clients expose stronger runtime APIs.",
|
||||
),
|
||||
audit_item(
|
||||
"output-eval-lab",
|
||||
"Output Eval Lab",
|
||||
status_from(
|
||||
output_summary.get("case_count", 0) >= 5
|
||||
and output_summary.get("gate_pass") is True
|
||||
and execution_summary.get("variant_run_count", 0) >= 10
|
||||
and output_summary.get("blind_pair_count", 0) >= 5
|
||||
),
|
||||
(
|
||||
f"{output_summary.get('case_count', 0)} cases; "
|
||||
f"delta {output_summary.get('delta', 'n/a')}; "
|
||||
f"exec {execution_summary.get('variant_run_count', 0)}; "
|
||||
f"blind {output_summary.get('blind_pair_count', 0)}"
|
||||
),
|
||||
"with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy",
|
||||
evidence(
|
||||
skill_dir,
|
||||
"evals/output/cases.jsonl",
|
||||
"scripts/run_output_eval.py",
|
||||
"scripts/run_output_execution.py",
|
||||
"reports/output_quality_scorecard.json",
|
||||
"reports/output_execution_runs.json",
|
||||
"reports/output_blind_review_pack.json",
|
||||
),
|
||||
"Add more real-file and adversarial holdout cases as usage grows.",
|
||||
),
|
||||
audit_item(
|
||||
"provider-holdout",
|
||||
"Provider Holdout",
|
||||
"pass" if execution_summary.get("model_executed_count", 0) > 0 else "external_required",
|
||||
f"model-executed {execution_summary.get('model_executed_count', 0)}; token-observed {execution_summary.get('token_observed_count', 0)}",
|
||||
"At least one real provider-backed holdout run with observed model/timing/token metadata",
|
||||
evidence(skill_dir, "scripts/provider_output_eval_runner.py", "reports/output_execution_runs.json"),
|
||||
"Run provider-backed holdout cases with real credentials and commit only aggregate evidence.",
|
||||
),
|
||||
audit_item(
|
||||
"human-adjudication",
|
||||
"Human Adjudication",
|
||||
"pass" if review_summary.get("ready_for_human_evidence") is True else "human_required",
|
||||
f"{review_summary.get('judgment_count', 0)}/{review_summary.get('pair_count', 0)} decisions; pending {review_summary.get('pending_count', 0)}",
|
||||
"Real reviewer decisions, blind-review attestation, and integrity fingerprints recorded before claiming output review completion",
|
||||
evidence(skill_dir, "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "scripts/adjudicate_output_review.py"),
|
||||
"Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.",
|
||||
),
|
||||
audit_item(
|
||||
"benchmark-reproducibility",
|
||||
"Benchmark Reproducibility",
|
||||
status_from(
|
||||
benchmark_reproducibility.get("ok") is True
|
||||
and benchmark_summary.get("reproducibility_ready") is True
|
||||
and benchmark_summary.get("failure_disclosure_count", 0) > 0
|
||||
and benchmark_summary.get("missing_artifact_count", 1) == 0
|
||||
),
|
||||
(
|
||||
f"artifacts {benchmark_summary.get('required_artifact_count', 0)}; "
|
||||
f"missing {benchmark_summary.get('missing_artifact_count', 'n/a')}; "
|
||||
f"failures {benchmark_summary.get('failure_disclosure_count', 0)}"
|
||||
),
|
||||
"Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable",
|
||||
evidence(
|
||||
skill_dir,
|
||||
"reports/benchmark_methodology.md",
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/benchmark_reproducibility.md",
|
||||
"evals/failure-cases.md",
|
||||
"tests/verify_benchmark_reproducibility.py",
|
||||
),
|
||||
"Keep the manifest current with every benchmark, package, and release evidence change.",
|
||||
),
|
||||
audit_item(
|
||||
"runtime-conformance",
|
||||
"Runtime Conformance",
|
||||
status_from(conformance_summary.get("target_count", 0) >= 5 and conformance_summary.get("pass_count") == conformance_summary.get("target_count")),
|
||||
f"{conformance_summary.get('pass_count', 0)}/{conformance_summary.get('target_count', 0)} targets pass",
|
||||
"Target package structure, metadata, relative paths, and degradation notes pass",
|
||||
evidence(skill_dir, "runtime/conformance/schema.json", "scripts/run_conformance_suite.py", "reports/conformance_matrix.json"),
|
||||
"Keep target conformance fixtures updated as platform contracts change.",
|
||||
),
|
||||
audit_item(
|
||||
"trust-security",
|
||||
"Trust Security",
|
||||
status_from(
|
||||
trust.get("ok") is True
|
||||
and trust_summary.get("secret_findings", 1) == 0
|
||||
and trust_summary.get("help_smoke_failed_count", 1) == 0
|
||||
and trust_summary.get("permission_missing_count", 1) == 0
|
||||
),
|
||||
(
|
||||
f"secrets {trust_summary.get('secret_findings', 'n/a')}; "
|
||||
f"scripts {trust_summary.get('script_count', 'n/a')}; "
|
||||
f"help failures {trust_summary.get('help_smoke_failed_count', 'n/a')}"
|
||||
),
|
||||
"Secrets, scripts, dependencies, permissions, and package hash are reviewable",
|
||||
evidence(skill_dir, "scripts/trust_check.py", "reports/security_trust_report.json", "security/permission_policy.json"),
|
||||
"Keep high-permission approvals scoped, expiring, and target-mapped.",
|
||||
),
|
||||
audit_item(
|
||||
"runtime-permission-metadata",
|
||||
"Permission Metadata",
|
||||
status_from(permission_summary.get("target_count", 0) >= 4 and permission_summary.get("fail_count", 1) == 0),
|
||||
(
|
||||
f"{permission_summary.get('pass_count', 0)}/{permission_summary.get('target_count', 0)} target probes pass; "
|
||||
f"metadata fallback {permission_summary.get('metadata_fallback_count', 0)}; "
|
||||
f"installer enforcement {permission_summary.get('installer_enforcement_pass_count', 0)}"
|
||||
),
|
||||
"Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available",
|
||||
evidence(skill_dir, "scripts/probe_runtime_permissions.py", "reports/runtime_permission_probes.json"),
|
||||
"Preserve residual-risk notes until real native enforcement exists.",
|
||||
),
|
||||
audit_item(
|
||||
"native-permission-enforcement",
|
||||
"Native Permission Enforcement",
|
||||
"pass" if permission_summary.get("native_enforcement_count", 0) > 0 else "external_required",
|
||||
(
|
||||
f"native-enforced targets {permission_summary.get('native_enforcement_count', 0)}; "
|
||||
f"installer-enforced targets {permission_summary.get('installer_enforcement_pass_count', 0)}"
|
||||
),
|
||||
"At least one target/client enforces approved permissions at runtime",
|
||||
evidence(skill_dir, "reports/runtime_permission_probes.json", "reports/install_simulation.json", "security/permission_policy.json"),
|
||||
"Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.",
|
||||
),
|
||||
audit_item(
|
||||
"skill-atlas",
|
||||
"Skill Atlas",
|
||||
status_from(
|
||||
atlas_summary.get("actionable_skill_count", 0) >= 1
|
||||
and atlas_summary.get("actionable_route_collision_count", 1) == 0
|
||||
and atlas_summary.get("actionable_owner_gap_count", 1) == 0
|
||||
),
|
||||
f"{atlas_summary.get('skill_count', 0)} skills; actionable collisions {atlas_summary.get('actionable_route_collision_count', 0)}",
|
||||
"Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities",
|
||||
evidence(skill_dir, "scripts/build_skill_atlas.py", "skill_atlas/catalog.json", "reports/skill_atlas.json", "skill_atlas/policy.json"),
|
||||
"Feed real drift data into Atlas once client telemetry is installed.",
|
||||
),
|
||||
audit_item(
|
||||
"registry-distribution",
|
||||
"Registry Distribution",
|
||||
status_from(
|
||||
registry.get("ok") is True
|
||||
and package_verification.get("ok") is True
|
||||
and install.get("ok") is True
|
||||
and package_summary.get("archive_entry_count", 0) > 0
|
||||
and install_summary.get("installer_permission_failure_count", 1) == 0
|
||||
),
|
||||
(
|
||||
f"zip entries {package_summary.get('archive_entry_count', 0)}; "
|
||||
f"install failures {install_summary.get('failure_count', 0)}; "
|
||||
f"permission failures {install_summary.get('installer_permission_failure_count', 0)}"
|
||||
),
|
||||
"Package metadata, archive checksum, package verification, and install simulation pass",
|
||||
evidence(skill_dir, "registry/packages/yao-meta-skill.json", "reports/package_verification.json", "reports/install_simulation.json"),
|
||||
"Regenerate registry after package verification so checksums stay aligned.",
|
||||
),
|
||||
audit_item(
|
||||
"review-studio",
|
||||
"Review Studio",
|
||||
status_from(studio_summary.get("gate_count", 0) >= 13 and studio_summary.get("blocker_count", 1) == 0),
|
||||
f"decision {studio_summary.get('decision', 'missing')}; warnings {studio_summary.get('warning_count', 0)}; score {studio_summary.get('world_class_score', 'n/a')}",
|
||||
"One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations",
|
||||
evidence(skill_dir, "scripts/render_review_studio.py", "reports/review-studio.json", "reports/review-studio.html"),
|
||||
"Resolve human/external warning gates before claiming full release readiness.",
|
||||
),
|
||||
audit_item(
|
||||
"telemetry-drift",
|
||||
"Telemetry Drift",
|
||||
status_from(
|
||||
adoption.get("ok") is True
|
||||
and adoption.get("privacy_contract", {}).get("raw_content_allowed") is False
|
||||
and telemetry_summary.get("recipe_count", 0) >= 5
|
||||
),
|
||||
f"events {adoption_summary.get('event_count', 0)}; risk {adoption_summary.get('risk_band', 'missing')}; recipes {telemetry_summary.get('recipe_count', 0)}",
|
||||
"Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path",
|
||||
evidence(skill_dir, "reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json", "scripts/import_telemetry_events.py"),
|
||||
"Keep raw JSONL out of distributed packages and use aggregate reports for Atlas.",
|
||||
),
|
||||
audit_item(
|
||||
"native-client-telemetry",
|
||||
"Native Client Telemetry",
|
||||
"pass" if sources.get("external", 0) > 0 and adoption_summary.get("adoption_sample_count", 0) > 0 else "external_required",
|
||||
f"external source events {sources.get('external', 0)}; adoption samples {adoption_summary.get('adoption_sample_count', 0)}",
|
||||
"A real Browser/Chrome/provider client sends production metadata events",
|
||||
evidence(skill_dir, "scripts/telemetry_native_host.py", "reports/adoption_drift_report.json"),
|
||||
"Install a real client against the native host and import production metadata-only events.",
|
||||
),
|
||||
]
|
||||
|
||||
counts: dict[str, int] = {key: 0 for key in STATUS_LABELS}
|
||||
for item in items:
|
||||
counts[item["status"]] = counts.get(item["status"], 0) + 1
|
||||
open_gap_count = sum(counts.get(key, 0) for key in ("warn", "human_required", "external_required", "missing"))
|
||||
local_foundation_ready = counts.get("missing", 0) == 0 and counts.get("warn", 0) == 0
|
||||
world_class_ready = open_gap_count == 0
|
||||
summary = {
|
||||
"item_count": len(items),
|
||||
"pass_count": counts.get("pass", 0),
|
||||
"warn_count": counts.get("warn", 0),
|
||||
"human_required_count": counts.get("human_required", 0),
|
||||
"external_required_count": counts.get("external_required", 0),
|
||||
"missing_count": counts.get("missing", 0),
|
||||
"open_gap_count": open_gap_count,
|
||||
"local_foundation_ready": local_foundation_ready,
|
||||
"world_class_ready": world_class_ready,
|
||||
"decision": "world-class-ready" if world_class_ready else "continue-iteration",
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": counts.get("missing", 0) == 0,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": summary,
|
||||
"status_counts": counts,
|
||||
"items": items,
|
||||
"next_highest_leverage": [
|
||||
item for item in items if item["status"] in {"human_required", "external_required", "warn", "missing"}
|
||||
][:5],
|
||||
"artifacts": {
|
||||
"json": "reports/skill_os2_audit.json",
|
||||
"markdown": "reports/skill_os2_audit.md",
|
||||
"world_class_evidence_plan": "reports/world_class_evidence_plan.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Skill OS 2.0 Audit",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- pass: `{summary['pass_count']}` / `{summary['item_count']}`",
|
||||
f"- human required: `{summary['human_required_count']}`",
|
||||
f"- external required: `{summary['external_required_count']}`",
|
||||
f"- missing: `{summary['missing_count']}`",
|
||||
f"- world-class ready: `{str(summary['world_class_ready']).lower()}`",
|
||||
"- evidence plan: `reports/world_class_evidence_plan.md`",
|
||||
"",
|
||||
"## Audit Items",
|
||||
"",
|
||||
"| Area | Status | Current | Target | Next action |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for item in report["items"]:
|
||||
lines.append(
|
||||
"| "
|
||||
+ " | ".join(
|
||||
[
|
||||
item["label"],
|
||||
STATUS_LABELS.get(item["status"], item["status"]),
|
||||
item["current"].replace("|", "\\|"),
|
||||
item["target"].replace("|", "\\|"),
|
||||
item["next_action"].replace("|", "\\|"),
|
||||
]
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
lines.extend(["", "## Open Highest-Leverage Gaps", ""])
|
||||
for item in report["next_highest_leverage"]:
|
||||
lines.append(f"- `{item['key']}` ({STATUS_LABELS.get(item['status'], item['status'])}): {item['next_action']}")
|
||||
if not report["next_highest_leverage"]:
|
||||
lines.append("- None.")
|
||||
lines.extend(["", "## Evidence", ""])
|
||||
for item in report["items"]:
|
||||
paths = [entry["path"] for entry in item["evidence"] if entry.get("exists")]
|
||||
missing = [entry["path"] for entry in item["evidence"] if not entry.get("exists")]
|
||||
lines.append(f"### {item['label']}")
|
||||
lines.append("")
|
||||
lines.append(f"- existing evidence: {', '.join(f'`{path}`' for path in paths) if paths else '`none`'}")
|
||||
if missing:
|
||||
lines.append(f"- missing evidence: {', '.join(f'`{path}`' for path in missing)}")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a requirement-by-requirement Skill OS 2.0 audit.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json", default="reports/skill_os2_audit.json")
|
||||
parser.add_argument("--output-md", default="reports/skill_os2_audit.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_audit(skill_dir, args.generated_at)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,555 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from skill_os2_coverage_markdown import render_markdown
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a Skill OS 2.0 blueprint-to-evidence coverage audit."
|
||||
RECOMMENDED_PR_LABELS = {
|
||||
"benchmark-methodology": "Benchmark Methodology",
|
||||
"output-eval-schema": "Output Eval Schema",
|
||||
"output-eval-runner": "Output Eval Runner",
|
||||
"output-quality-scorecard": "Output Quality Scorecard",
|
||||
"skill-ir-v0": "Skill IR V0",
|
||||
"compiler-refactor": "Compiler Refactor",
|
||||
"agent-skills-conformance": "Agent Skills Conformance",
|
||||
"trust-check": "Trust Check",
|
||||
"skill-atlas-generator": "Skill Atlas Generator",
|
||||
"registry-package-format": "Registry Package Format",
|
||||
"review-studio-2": "Review Studio 2.0",
|
||||
"migration-v2-docs": "Migration V2 Docs",
|
||||
"evidence-consistency": "Evidence Consistency",
|
||||
}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def evidence(skill_dir: Path, paths: list[str]) -> list[dict[str, Any]]:
|
||||
return [{"path": item, "exists": (skill_dir / item).exists()} for item in paths]
|
||||
|
||||
|
||||
def all_exist(items: list[dict[str, Any]]) -> bool:
|
||||
return all(item["exists"] for item in items)
|
||||
|
||||
|
||||
def paths_exist(skill_dir: Path, paths: list[str]) -> bool:
|
||||
return all((skill_dir / item).exists() for item in paths)
|
||||
|
||||
|
||||
def latest_daily_skillops_paths(skill_dir: Path) -> list[str]:
|
||||
base_paths = [
|
||||
"scripts/render_daily_skillops_report.py",
|
||||
"tests/verify_daily_skillops.py",
|
||||
]
|
||||
daily_dir = skill_dir / "reports" / "skillops" / "daily"
|
||||
daily_json_reports = sorted(daily_dir.glob("*.json")) if daily_dir.exists() else []
|
||||
if not daily_json_reports:
|
||||
return [
|
||||
*base_paths,
|
||||
"reports/skillops/daily/YYYY-MM-DD.json",
|
||||
"reports/skillops/daily/YYYY-MM-DD.md",
|
||||
]
|
||||
latest_json = daily_json_reports[-1]
|
||||
latest_md = latest_json.with_suffix(".md")
|
||||
return [*base_paths, rel_path(latest_json, skill_dir), rel_path(latest_md, skill_dir)]
|
||||
|
||||
|
||||
def latest_weekly_curator_paths(skill_dir: Path) -> list[str]:
|
||||
base_paths = [
|
||||
"scripts/render_weekly_curator_report.py",
|
||||
"tests/verify_weekly_curator.py",
|
||||
]
|
||||
weekly_dir = skill_dir / "reports" / "skillops" / "weekly"
|
||||
weekly_json_reports = sorted(weekly_dir.glob("*.json")) if weekly_dir.exists() else []
|
||||
if not weekly_json_reports:
|
||||
return [
|
||||
*base_paths,
|
||||
"reports/skillops/weekly/YYYY-WNN.json",
|
||||
"reports/skillops/weekly/YYYY-WNN.md",
|
||||
]
|
||||
latest_json = weekly_json_reports[-1]
|
||||
latest_md = latest_json.with_suffix(".md")
|
||||
return [*base_paths, rel_path(latest_json, skill_dir), rel_path(latest_md, skill_dir)]
|
||||
|
||||
|
||||
def condition_status(condition: bool, evidence_items: list[dict[str, Any]]) -> str:
|
||||
if not all_exist(evidence_items):
|
||||
return "missing"
|
||||
return "pass" if condition else "warn"
|
||||
|
||||
|
||||
def build_item(
|
||||
*,
|
||||
key: str,
|
||||
category: str,
|
||||
label: str,
|
||||
objective: str,
|
||||
artifact_paths: list[str],
|
||||
command: str,
|
||||
test: str,
|
||||
current: str,
|
||||
condition: bool,
|
||||
next_action: str,
|
||||
skill_dir: Path,
|
||||
) -> dict[str, Any]:
|
||||
evidence_items = evidence(skill_dir, artifact_paths)
|
||||
status = condition_status(condition, evidence_items)
|
||||
return {
|
||||
"key": key,
|
||||
"category": category,
|
||||
"label": label,
|
||||
"status": status,
|
||||
"objective": objective,
|
||||
"current": current,
|
||||
"command": command,
|
||||
"test": test,
|
||||
"evidence": evidence_items,
|
||||
"next_action": next_action,
|
||||
}
|
||||
|
||||
|
||||
def build_extension_track(
|
||||
*,
|
||||
key: str,
|
||||
label: str,
|
||||
status: str,
|
||||
objective: str,
|
||||
current: str,
|
||||
target: str,
|
||||
artifact_paths: list[str],
|
||||
next_action: str,
|
||||
skill_dir: Path,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"key": key,
|
||||
"label": label,
|
||||
"status": status,
|
||||
"objective": objective,
|
||||
"current": current,
|
||||
"target": target,
|
||||
"evidence": evidence(skill_dir, artifact_paths),
|
||||
"next_action": next_action,
|
||||
}
|
||||
|
||||
|
||||
def summary_value(payload: dict[str, Any], key: str, default: Any = 0) -> Any:
|
||||
summary = payload.get("summary", {})
|
||||
return summary.get(key, default) if isinstance(summary, dict) else default
|
||||
|
||||
|
||||
def target_names(payload: dict[str, Any]) -> set[str]:
|
||||
names = set()
|
||||
for item in payload.get("targets", []) or payload.get("results", []):
|
||||
if isinstance(item, dict):
|
||||
value = item.get("target") or item.get("platform") or item.get("name")
|
||||
if value:
|
||||
names.add(str(value))
|
||||
return names
|
||||
|
||||
|
||||
def build_coverage(skill_dir: Path, generated_at: str) -> dict[str, Any]:
|
||||
reports = skill_dir / "reports"
|
||||
skill_ir = load_json(skill_dir / "skill-ir" / "examples" / "yao-meta-skill.json")
|
||||
output_quality = load_json(reports / "output_quality_scorecard.json")
|
||||
output_execution = load_json(reports / "output_execution_runs.json")
|
||||
compiled = load_json(reports / "compiled_targets.json")
|
||||
conformance = load_json(reports / "conformance_matrix.json")
|
||||
trust = load_json(reports / "security_trust_report.json")
|
||||
atlas = load_json(reports / "skill_atlas.json")
|
||||
registry = load_json(reports / "registry_audit.json")
|
||||
package_verification = load_json(reports / "package_verification.json")
|
||||
install = load_json(reports / "install_simulation.json")
|
||||
review_studio = load_json(reports / "review-studio.json")
|
||||
adoption = load_json(reports / "adoption_drift_report.json")
|
||||
telemetry_hooks = load_json(reports / "telemetry_hook_recipes.json")
|
||||
benchmark = load_json(reports / "benchmark_reproducibility.json")
|
||||
world_class_ledger = load_json(reports / "world_class_evidence_ledger.json")
|
||||
evidence_consistency = load_json(reports / "evidence_consistency.json")
|
||||
|
||||
output_case_count = summary_value(output_quality, "case_count")
|
||||
output_delta = summary_value(output_quality, "delta", "n/a")
|
||||
execution_count = summary_value(output_execution, "variant_run_count")
|
||||
compiled_count = summary_value(compiled, "target_count")
|
||||
compiled_pass = summary_value(compiled, "pass_count")
|
||||
conformance_count = summary_value(conformance, "target_count")
|
||||
conformance_pass = summary_value(conformance, "pass_count")
|
||||
script_count = summary_value(trust, "script_count")
|
||||
atlas_skill_count = summary_value(atlas, "skill_count")
|
||||
package_entries = summary_value(package_verification, "archive_entry_count")
|
||||
install_failures = summary_value(install, "failure_count")
|
||||
studio_gates = summary_value(review_studio, "gate_count")
|
||||
telemetry_recipe_count = summary_value(telemetry_hooks, "recipe_count")
|
||||
benchmark_artifacts = summary_value(benchmark, "required_artifact_count")
|
||||
pending_world_class = summary_value(world_class_ledger, "pending_count")
|
||||
|
||||
modules = [
|
||||
build_item(
|
||||
key="skill-ir",
|
||||
category="core-module",
|
||||
label="Skill IR",
|
||||
objective="Platform-neutral capability contract exists before platform-specific packaging.",
|
||||
artifact_paths=["skill-ir/schema.json", "skill-ir/examples/yao-meta-skill.json", "scripts/export_skill_ir.py", "tests/verify_skill_ir.py"],
|
||||
command="python3 scripts/yao.py skill-ir .",
|
||||
test="python3 tests/verify_skill_ir.py",
|
||||
current=f"schema {skill_ir.get('schema_version', 'missing')}; targets {len(skill_ir.get('targets', []))}",
|
||||
condition=skill_ir.get("schema_version") == "2.0.0" and len(skill_ir.get("targets", [])) >= 5,
|
||||
next_action="Keep all target packages compiled from IR rather than hand-maintained per target.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="output-eval-lab",
|
||||
category="core-module",
|
||||
label="Output Eval Lab",
|
||||
objective="with-skill/baseline output quality is measured with assertions, execution evidence, and blind review packs.",
|
||||
artifact_paths=["evals/output/schema.json", "evals/output/cases.jsonl", "scripts/run_output_eval.py", "scripts/run_output_execution.py", "reports/output_quality_scorecard.json", "tests/verify_output_eval_lab.py"],
|
||||
command="python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .",
|
||||
test="python3 tests/verify_output_eval_lab.py",
|
||||
current=f"{output_case_count} cases; delta {output_delta}; execution {execution_count}",
|
||||
condition=output_case_count >= 5 and execution_count >= 10 and summary_value(output_quality, "gate_pass") is True,
|
||||
next_action="Add more real-file and adversarial holdout cases as adoption data grows.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="runtime-conformance",
|
||||
category="core-module",
|
||||
label="Runtime Conformance",
|
||||
objective="Target packages can be consumed by OpenAI, Claude, Agent Skills, VS Code, and generic targets.",
|
||||
artifact_paths=["runtime/conformance/schema.json", "scripts/run_conformance_suite.py", "reports/conformance_matrix.json", "tests/verify_conformance_suite.py"],
|
||||
command="python3 scripts/yao.py conformance .",
|
||||
test="python3 tests/verify_conformance_suite.py",
|
||||
current=f"{conformance_pass}/{conformance_count} targets pass",
|
||||
condition=conformance_count >= 5 and conformance_pass == conformance_count and "agent-skills" in target_names(conformance),
|
||||
next_action="Keep conformance cases aligned with current platform metadata rules.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="trust-security",
|
||||
category="core-module",
|
||||
label="Trust Security",
|
||||
objective="Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.",
|
||||
artifact_paths=["scripts/trust_check.py", "security/trust_policy.md", "security/script_policy.md", "security/permission_policy.json", "reports/security_trust_report.json", "tests/verify_trust_check.py"],
|
||||
command="python3 scripts/yao.py trust .",
|
||||
test="python3 tests/verify_trust_check.py",
|
||||
current=f"{script_count} scripts; secrets {summary_value(trust, 'secret_findings', 'n/a')}; help failures {summary_value(trust, 'help_smoke_failed_count', 'n/a')}",
|
||||
condition=trust.get("ok") is True and summary_value(trust, "secret_findings", 1) == 0 and summary_value(trust, "help_smoke_failed_count", 1) == 0,
|
||||
next_action="Keep high-permission approvals scoped, expiring, and mapped to target enforcement.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="skill-atlas",
|
||||
category="core-module",
|
||||
label="Skill Atlas",
|
||||
objective="Team skill portfolio reveals route collisions, stale ownership, dependency graph, and no-route opportunities.",
|
||||
artifact_paths=["scripts/build_skill_atlas.py", "skill_atlas/catalog.json", "skill_atlas/route_overlap_matrix.csv", "skill_atlas/dependency_graph.json", "reports/skill_atlas.json", "tests/verify_skill_atlas.py"],
|
||||
command="python3 scripts/yao.py skill-atlas --workspace-root .",
|
||||
test="python3 tests/verify_skill_atlas.py",
|
||||
current=f"{atlas_skill_count} scanned skills; actionable collisions {summary_value(atlas, 'actionable_route_collision_count')}",
|
||||
condition=atlas_skill_count >= 1 and summary_value(atlas, "actionable_route_collision_count", 1) == 0,
|
||||
next_action="Use real telemetry to rank stale or drifting skills by impact, not only by static metadata.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="registry-distribution",
|
||||
category="core-module",
|
||||
label="Registry Distribution",
|
||||
objective="Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
|
||||
artifact_paths=["registry/index.schema.json", "registry/package.schema.json", "registry/packages/yao-meta-skill.json", "scripts/registry_audit.py", "reports/package_verification.json", "reports/install_simulation.json", "tests/verify_registry_audit.py"],
|
||||
command="python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
|
||||
test="python3 tests/verify_registry_audit.py",
|
||||
current=f"archive entries {package_entries}; install failures {install_failures}",
|
||||
condition=registry.get("ok") is True and package_verification.get("ok") is True and install.get("ok") is True and package_entries > 0 and install_failures == 0,
|
||||
next_action="Regenerate registry metadata after package verification so source and archive checksums stay aligned.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="review-studio",
|
||||
category="core-module",
|
||||
label="Review Studio",
|
||||
objective="One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.",
|
||||
artifact_paths=["scripts/render_review_studio.py", "reports/review-studio.html", "reports/review-studio.json", "tests/verify_review_studio.py"],
|
||||
command="python3 scripts/yao.py review-studio .",
|
||||
test="python3 tests/verify_review_studio.py",
|
||||
current=f"{studio_gates} gates; decision {summary_value(review_studio, 'decision', 'missing')}; warnings {summary_value(review_studio, 'warning_count')}",
|
||||
condition=review_studio.get("ok") is True and studio_gates >= 14 and summary_value(review_studio, "blocker_count", 1) == 0,
|
||||
next_action="Close pending human and external evidence before claiming full release readiness.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
build_item(
|
||||
key="telemetry-drift",
|
||||
category="core-module",
|
||||
label="Telemetry Drift",
|
||||
objective="Real usage feedback is captured as metadata-only local-first drift signals.",
|
||||
artifact_paths=["scripts/emit_telemetry_event.py", "scripts/import_telemetry_events.py", "scripts/telemetry_native_host.py", "reports/adoption_drift_report.json", "reports/telemetry_hook_recipes.json", "tests/verify_telemetry_hooks.py"],
|
||||
command="python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .",
|
||||
test="python3 tests/verify_telemetry_hooks.py",
|
||||
current=f"events {summary_value(adoption, 'event_count')}; recipes {telemetry_recipe_count}; risk {summary_value(adoption, 'risk_band', 'missing')}",
|
||||
condition=adoption.get("ok") is True and adoption.get("privacy_contract", {}).get("raw_content_allowed") is False and telemetry_recipe_count >= 5,
|
||||
next_action="Install a real client and import production metadata-only events into the local drift loop.",
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
]
|
||||
|
||||
recommended_prs = [
|
||||
("benchmark-methodology", "reports/benchmark_methodology.md", "reports/benchmark_reproducibility.json", "tests/verify_benchmark_reproducibility.py", benchmark.get("ok") is True and summary_value(benchmark, "methodology_complete") is True, f"{benchmark_artifacts} required artifacts checked"),
|
||||
("output-eval-schema", "evals/output/schema.json", "evals/output/cases.jsonl", "tests/verify_output_eval_lab.py", output_case_count >= 5, f"{output_case_count} output cases"),
|
||||
("output-eval-runner", "scripts/run_output_eval.py", "reports/output_quality_scorecard.json", "tests/verify_output_eval_lab.py", output_quality.get("ok") is True, f"delta {output_delta}"),
|
||||
("output-quality-scorecard", "reports/output_quality_scorecard.md", "reports/output_quality_scorecard.json", "tests/verify_output_eval_lab.py", summary_value(output_quality, "gate_pass") is True, f"gate pass {summary_value(output_quality, 'gate_pass', False)}"),
|
||||
("skill-ir-v0", "skill-ir/schema.json", "skill-ir/examples/yao-meta-skill.json", "tests/verify_skill_ir.py", skill_ir.get("schema_version") == "2.0.0", f"schema {skill_ir.get('schema_version', 'missing')}"),
|
||||
("compiler-refactor", "scripts/compile_skill.py", "reports/compiled_targets.json", "tests/verify_compile_skill.py", compiled_count >= 5 and compiled_pass == compiled_count, f"{compiled_pass}/{compiled_count} compiled targets"),
|
||||
("agent-skills-conformance", "runtime/conformance/schema.json", "reports/conformance_matrix.json", "tests/verify_conformance_suite.py", "agent-skills" in target_names(conformance), "agent-skills target present"),
|
||||
("trust-check", "scripts/trust_check.py", "reports/security_trust_report.json", "tests/verify_trust_check.py", trust.get("ok") is True, f"secret findings {summary_value(trust, 'secret_findings', 'n/a')}"),
|
||||
("skill-atlas-generator", "scripts/build_skill_atlas.py", "reports/skill_atlas.json", "tests/verify_skill_atlas.py", atlas.get("ok") is True, f"{atlas_skill_count} scanned skills"),
|
||||
("registry-package-format", "registry/package.schema.json", "reports/registry_audit.json", "tests/verify_registry_audit.py", registry.get("ok") is True, f"registry ok {registry.get('ok') is True}"),
|
||||
("review-studio-2", "scripts/render_review_studio.py", "reports/review-studio.html", "tests/verify_review_studio.py", review_studio.get("ok") is True and studio_gates >= 14, f"{studio_gates} review gates"),
|
||||
("migration-v2-docs", "docs/migration-v2.md", "reports/skill-os-2-review.md", "README.md", (skill_dir / "docs" / "migration-v2.md").exists(), "migration guide present"),
|
||||
("evidence-consistency", "scripts/render_evidence_consistency.py", "reports/evidence_consistency.json", "tests/verify_evidence_consistency.py", evidence_consistency.get("ok") is True and summary_value(evidence_consistency, "fail_count") == 0, f"{summary_value(evidence_consistency, 'check_count')} consistency checks"),
|
||||
]
|
||||
pr_items = [
|
||||
build_item(
|
||||
key=key,
|
||||
category="recommended-pr",
|
||||
label=RECOMMENDED_PR_LABELS[key],
|
||||
objective="Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
|
||||
artifact_paths=[path_a, path_b, path_c],
|
||||
command="make ci-test",
|
||||
test=path_c if path_c.startswith("tests/") else "docs review",
|
||||
current=current,
|
||||
condition=condition,
|
||||
next_action="Keep this item covered as the implementation evolves.",
|
||||
skill_dir=skill_dir,
|
||||
)
|
||||
for key, path_a, path_b, path_c, condition, current in recommended_prs
|
||||
]
|
||||
|
||||
interpretation_paths = [
|
||||
"reports/skill-overview.html",
|
||||
"reports/skill-overview.json",
|
||||
"scripts/render_skill_overview.py",
|
||||
"scripts/render_skill_interpretation.py",
|
||||
"schemas/skill-interpretation.schema.json",
|
||||
"tests/verify_skill_interpretation.py",
|
||||
]
|
||||
interpretation_ready = paths_exist(skill_dir, interpretation_paths)
|
||||
interpretation_status = "covered" if interpretation_ready else "partial"
|
||||
interpretation_current = (
|
||||
"Skill Overview v2 is canonical and mirrored as first-class skill-interpretation HTML/JSON with schema and tests."
|
||||
if interpretation_ready
|
||||
else "Skill Overview v2 already covers much of the explainer experience, but dedicated skill-interpretation JSON/HTML artifacts are not first-class yet."
|
||||
)
|
||||
interpretation_next_action = (
|
||||
"Keep overview and interpretation contracts in lockstep when report sections, metrics, or layout semantics change."
|
||||
if interpretation_ready
|
||||
else "Decide whether overview v2 is the canonical interpretation surface; if not, add a dedicated schema, renderer, and CJK/path-safety tests."
|
||||
)
|
||||
extension_tracks = [
|
||||
build_extension_track(
|
||||
key="skill-interpretation-report",
|
||||
label="Skill Interpretation Report",
|
||||
status=interpretation_status,
|
||||
objective="User-facing deep interpretation report explains use cases, triggers, inputs, outputs, workflow, principles, boundaries, quality gates, examples, and next iterations.",
|
||||
current=interpretation_current,
|
||||
target="Either keep skill-overview as the canonical interpretation report with an explicit contract, or split a dedicated reports/skill-interpretation.* renderer and tests.",
|
||||
artifact_paths=interpretation_paths,
|
||||
next_action=interpretation_next_action,
|
||||
skill_dir=skill_dir,
|
||||
),
|
||||
]
|
||||
adaptive_foundation_paths = [
|
||||
"references/autonomous-adaptation.md",
|
||||
"references/user-memory-policy.md",
|
||||
"schemas/adaptation-proposal.schema.json",
|
||||
"scripts/summarize_user_signals.py",
|
||||
"scripts/propose_adaptation.py",
|
||||
"tests/verify_adaptation_safety.py",
|
||||
]
|
||||
adaptive_apply_paths = [
|
||||
"scripts/apply_adaptation.py",
|
||||
"reports/adaptation_approval_ledger.json",
|
||||
"reports/adaptation_regression_report.json",
|
||||
]
|
||||
adaptive_foundation_ready = paths_exist(skill_dir, adaptive_foundation_paths)
|
||||
adaptive_apply_ready = paths_exist(skill_dir, adaptive_apply_paths)
|
||||
adaptive_status = "covered" if adaptive_foundation_ready and adaptive_apply_ready else ("partial" if adaptive_foundation_ready else "planned")
|
||||
adaptive_current = (
|
||||
"Proposal-only adapt-scan/adapt-propose foundation exists with policy, schema, and safety tests; approval-gated patch application is not implemented yet."
|
||||
if adaptive_status == "partial"
|
||||
else (
|
||||
"Full adaptive loop includes proposal, approval, patch application, regression evidence, and rollback metadata."
|
||||
if adaptive_status == "covered"
|
||||
else "The repo has feedback, iteration, telemetry, and review artifacts, but no adapt-scan/adapt-propose/adapt-apply approval loop and no user-memory policy."
|
||||
)
|
||||
)
|
||||
adaptive_next_action = (
|
||||
"Add adapt-apply only after approval ledger, allowlisted targets, dry-run diffs, regression reports, and rollback artifacts are designed."
|
||||
if adaptive_status == "partial"
|
||||
else "Start with policy and read-only scan tests; do not read shell history or private logs unless the user provides an explicit source path."
|
||||
)
|
||||
extension_tracks.append(
|
||||
build_extension_track(
|
||||
key="adaptive-self-iteration",
|
||||
label="Adaptive Self-Iteration",
|
||||
status=adaptive_status,
|
||||
objective="Local-first preference memory, repeated-signal extraction, adaptation proposals, approval, patch application, regression evidence, and rollback.",
|
||||
current=adaptive_current,
|
||||
target="Proposal-only adaptation with explicit input source, redaction, allowlisted write targets, approval ledger, regression report, and rollback plan.",
|
||||
artifact_paths=[
|
||||
*adaptive_foundation_paths,
|
||||
*adaptive_apply_paths,
|
||||
"reports/user_patterns.json",
|
||||
"reports/adaptation_proposals.json",
|
||||
"reports/iteration-directions.md",
|
||||
"reports/adoption_drift_report.md",
|
||||
],
|
||||
next_action=adaptive_next_action,
|
||||
skill_dir=skill_dir,
|
||||
)
|
||||
)
|
||||
daily_skillops_paths = latest_daily_skillops_paths(skill_dir)
|
||||
daily_skillops_ready = paths_exist(skill_dir, daily_skillops_paths)
|
||||
daily_skillops_status = "covered" if daily_skillops_ready else "partial"
|
||||
extension_tracks.append(
|
||||
build_extension_track(
|
||||
key="daily-skillops-report",
|
||||
label="Daily SkillOps Report",
|
||||
status=daily_skillops_status,
|
||||
objective="Daily operations layer summarizes explicit-source conversation patterns, proposal-only adaptation work, approval state, release locks, and world-class evidence gaps.",
|
||||
current=(
|
||||
"Daily SkillOps report is a CLI-backed, explicit-source operations cockpit with tests and committed dated evidence."
|
||||
if daily_skillops_ready
|
||||
else "Adaptive scan/propose exists, but the daily operations report is not fully wired into scripts, tests, and evidence artifacts."
|
||||
),
|
||||
target="Generate reports/skillops/daily/YYYY-MM-DD.* from explicit sources without private log scanning, source writes, auto-patching, or world-class overclaiming.",
|
||||
artifact_paths=daily_skillops_paths,
|
||||
next_action="Keep Daily SkillOps report aligned with proposal, approval, coverage, and world-class ledger contracts as the operations layer evolves.",
|
||||
skill_dir=skill_dir,
|
||||
)
|
||||
)
|
||||
weekly_curator_paths = latest_weekly_curator_paths(skill_dir)
|
||||
weekly_curator_ready = paths_exist(skill_dir, weekly_curator_paths)
|
||||
weekly_curator_status = "covered" if weekly_curator_ready else "partial"
|
||||
extension_tracks.append(
|
||||
build_extension_track(
|
||||
key="weekly-curator-report",
|
||||
label="Weekly Curator Report",
|
||||
status=weekly_curator_status,
|
||||
objective="Weekly curator layer aggregates Daily SkillOps opportunities, Skill Atlas portfolio signals, release locks, and world-class evidence gaps into a maintenance queue.",
|
||||
current=(
|
||||
"Weekly curator report is CLI-backed, proposal-only, and backed by committed weekly JSON/Markdown evidence."
|
||||
if weekly_curator_ready
|
||||
else "Daily SkillOps and Skill Atlas exist, but weekly curator aggregation is not fully wired into scripts, tests, and evidence artifacts."
|
||||
),
|
||||
target="Generate reports/skillops/weekly/YYYY-WNN.* without private log scanning, source writes, auto-patching, or world-class overclaiming.",
|
||||
artifact_paths=weekly_curator_paths,
|
||||
next_action="Use weekly curator output as the Skill Librarian maintenance queue before approving any durable skill-library changes.",
|
||||
skill_dir=skill_dir,
|
||||
)
|
||||
)
|
||||
extension_counts: dict[str, int] = {}
|
||||
for item in extension_tracks:
|
||||
extension_counts[item["status"]] = extension_counts.get(item["status"], 0) + 1
|
||||
|
||||
items = modules + pr_items
|
||||
counts: dict[str, int] = {"pass": 0, "warn": 0, "missing": 0}
|
||||
for item in items:
|
||||
counts[item["status"]] = counts.get(item["status"], 0) + 1
|
||||
local_ready = counts.get("missing", 0) == 0 and counts.get("warn", 0) == 0
|
||||
public_ready = local_ready and summary_value(world_class_ledger, "ready_to_claim_world_class", False) is True
|
||||
decision = "world-class-ready" if public_ready else ("local-blueprint-covered-evidence-pending" if local_ready else "continue-implementation")
|
||||
summary = {
|
||||
"item_count": len(items),
|
||||
"module_count": len(modules),
|
||||
"recommended_pr_count": len(pr_items),
|
||||
"pass_count": counts.get("pass", 0),
|
||||
"warn_count": counts.get("warn", 0),
|
||||
"missing_count": counts.get("missing", 0),
|
||||
"extension_track_count": len(extension_tracks),
|
||||
"extension_partial_count": extension_counts.get("partial", 0),
|
||||
"extension_planned_count": extension_counts.get("planned", 0),
|
||||
"extension_covered_count": extension_counts.get("covered", 0),
|
||||
"adaptive_extension_ready": adaptive_status == "covered",
|
||||
"local_blueprint_ready": local_ready,
|
||||
"public_world_class_ready": public_ready,
|
||||
"world_class_evidence_pending_count": pending_world_class,
|
||||
"decision": decision,
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": counts.get("missing", 0) == 0,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": summary,
|
||||
"status_counts": counts,
|
||||
"extension_status_counts": extension_counts,
|
||||
"modules": modules,
|
||||
"recommended_prs": pr_items,
|
||||
"reference_extension_tracks": extension_tracks,
|
||||
"next_highest_leverage": [
|
||||
"Close the four world-class evidence ledger entries with accepted human or external evidence.",
|
||||
"Keep the first-class skill interpretation report and Skill Overview v2 contract synchronized as the report model evolves.",
|
||||
"Use Daily SkillOps with explicit curated sources to review adaptation proposals without scanning private logs or auto-patching.",
|
||||
"Keep the blueprint coverage report in CI so 2.0 plan drift is visible.",
|
||||
],
|
||||
"source_blueprint": {
|
||||
"title": "Skill Overview / Skill OS 2.0 upgrade plan",
|
||||
"core_module_count": 8,
|
||||
"recommended_pr_count": 13,
|
||||
"reference_extension_count": len(extension_tracks),
|
||||
"reference_extensions": [
|
||||
"Skill interpretation report",
|
||||
"Adaptive self-iteration with local preference memory and approval gates",
|
||||
"Daily SkillOps operations report",
|
||||
],
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/skill_os2_coverage.json",
|
||||
"markdown": "reports/skill_os2_coverage.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render Skill OS 2.0 blueprint coverage evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json", default="reports/skill_os2_coverage.json")
|
||||
parser.add_argument("--output-md", default="reports/skill_os2_coverage.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_coverage(skill_dir, args.generated_at)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+151
-181
@@ -5,174 +5,20 @@ import json
|
||||
from pathlib import Path
|
||||
|
||||
from skill_report_charts import render_chart_set
|
||||
from skill_report_i18n import (
|
||||
KIND_ZH,
|
||||
LABEL_EN,
|
||||
PACKAGE_LABEL_ZH,
|
||||
bi_item,
|
||||
bi_span,
|
||||
en_for,
|
||||
mode_zh,
|
||||
readable_description_zh,
|
||||
)
|
||||
from skill_report_layout import render_language_switch, render_report_nav, skill_overview_css, skill_overview_script
|
||||
from skill_report_model import REPORT_NAV_V2, build_report_model
|
||||
|
||||
|
||||
TEXT_ZH = {
|
||||
"Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.": "从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill;适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。",
|
||||
"Understand the request.": "理解用户请求。",
|
||||
"Execute the main task.": "执行核心任务。",
|
||||
"Validate the result.": "校验交付结果。",
|
||||
"Understand the request": "理解用户请求。",
|
||||
"Execute the main task": "执行核心任务。",
|
||||
"Validate the result": "校验交付结果。",
|
||||
"Decide whether the request should become a skill and choose the lightest fit.": "判断请求是否应该沉淀为 Skill,并选择最轻量可靠的模式。",
|
||||
"Capture job, output, exclusions, constraints, and standards.": "捕捉任务、输出、排除项、约束和质量标准。",
|
||||
"Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.": "运行参考扫描:先看外部 benchmark,再看用户材料,最后校验本地适配;只暴露不确定性或冲突。",
|
||||
"Write the `description` early and test route quality before expanding the package.": "尽早写出 `description`,先测试路由质量,再扩展包体。",
|
||||
"Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.": "只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。",
|
||||
"Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.": "当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。",
|
||||
"Turn rough requests into a compact reusable demo skill.": "把粗糙请求整理成紧凑、可复用的演示 Skill。",
|
||||
"Tighten trigger and exclusions": "收紧触发与排除边界",
|
||||
"Add the first execution asset": "补上第一个执行资产",
|
||||
"Promote from scaffold to production-ready": "从脚手架推进到生产可用",
|
||||
"Borrow one proven pattern on purpose": "有选择地借鉴一个成熟模式",
|
||||
"Harden portability semantics": "加固跨环境语义",
|
||||
"Create an iteration evidence loop": "建立迭代证据回路",
|
||||
"The package needs clearer near-neighbor exclusions before it grows.": "在继续扩展前,需要先把相邻但不应触发的场景说清楚。",
|
||||
"The package is still mostly prose. Add one asset that removes repeated manual work.": "当前包体仍偏文本说明,应先增加一个能减少重复人工操作的资产。",
|
||||
"The first version exists; the next gain usually comes from adding the smallest useful gates.": "第一版已经存在,下一步收益通常来自补上最小但有效的质量门禁。",
|
||||
"You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.": "已经有公开 benchmark 对象,下一步应主动选择一个模式借鉴,而不是松散吸收所有做法。",
|
||||
"The skill already signals reuse across environments, so contract clarity matters early.": "这个 Skill 已经面向跨环境复用,因此早期就需要把契约语义说清楚。",
|
||||
"The package should show what changed and why after the first draft.": "第一版之后,包体应该能说明改了什么以及为什么改。",
|
||||
"Add 3 to 5 should-trigger and should-not-trigger examples.": "增加 3 到 5 个应触发和不应触发的例子。",
|
||||
"Refine the frontmatter description to name the recurring job and non-goals.": "精炼 frontmatter description,明确重复任务和非目标。",
|
||||
"Run a first trigger evaluation pass before expanding the package.": "扩展包体前先跑一轮触发评估。",
|
||||
"Move stable procedural guidance into references if users will need it repeatedly.": "如果用户会反复使用某段流程说明,把它沉淀到 references。",
|
||||
"Create one deterministic helper script if a repeated step can be executed instead of described.": "如果某个重复步骤可以执行而不是描述,就沉淀成一个确定性 helper script。",
|
||||
"Keep the main SKILL.md compact and route-oriented.": "保持主 SKILL.md 简洁,并围绕路由与入口组织。",
|
||||
"Decide whether this skill is personal, team-reused, or library-grade.": "判断这个 Skill 是个人使用、团队复用,还是库级基础能力。",
|
||||
"Add only the gates that match that risk level.": "只添加与风险等级匹配的质量门禁。",
|
||||
"Record lifecycle metadata and review cadence once reuse becomes real.": "一旦进入真实复用,就记录生命周期元数据和评审节奏。",
|
||||
"Decide whether to borrow method, structure, execution, or portability, but only one of them first.": "先判断要借鉴的是方法、结构、执行方式还是可迁移性,并且第一轮只借鉴其中一个。",
|
||||
"Record what you will not borrow so the package stays light.": "记录本轮不借鉴的内容,避免包体过重。",
|
||||
"Confirm activation mode, execution context, and trust assumptions.": "确认激活模式、执行上下文和信任假设。",
|
||||
"Add or review degradation strategy for non-native targets.": "补充或复核非原生目标端的降级策略。",
|
||||
"Package the skill once to verify adapter expectations.": "至少打包一次 Skill,用来验证 adapter 预期。",
|
||||
"Generate the HTML skill report and keep it aligned with the package.": "生成 HTML Skill 报告,并保持它与包体内容一致。",
|
||||
"Record reference scan choices and non-goals.": "记录参考扫描的取舍和非目标。",
|
||||
"Capture the next iteration choice explicitly before adding more files.": "在继续增加文件前,明确记录下一轮迭代选择。",
|
||||
"Cleaner routing and fewer accidental activations.": "路由更清晰,误触发更少。",
|
||||
"Stronger execution quality without bloating the entrypoint.": "在不膨胀入口文件的前提下提升执行质量。",
|
||||
"A clearer path from exploratory package to maintained asset.": "更清晰地从探索性包体走向可维护资产。",
|
||||
"A cleaner package shape with less accidental over-design.": "包体形态更清晰,也减少偶然过度设计。",
|
||||
"Safer cross-environment reuse with less target drift.": "跨环境复用更安全,目标漂移更少。",
|
||||
"A clearer path for the next author or reviewer.": "让下一位作者或评审者更容易接手。",
|
||||
}
|
||||
|
||||
TEXT_EN = {
|
||||
"触发面保持精简,并锚定在 frontmatter description。": "The trigger surface stays lean and anchored in the frontmatter description.",
|
||||
"已打包 agents/interface.yaml,便于后续做跨平台适配。": "Portable interface metadata is packaged for later adapter-based export.",
|
||||
"长指导被拆到 references 中,入口文件可以保持轻量。": "Extended guidance is separated into references so the entrypoint can stay compact.",
|
||||
"确定性辅助逻辑放在 scripts 中,而不是藏在提示词里。": "Deterministic helper logic lives in scripts instead of hidden prompt text.",
|
||||
"包内包含可随 Skill 迁移的质量门禁或触发检查。": "The package includes portable quality gates or trigger checks.",
|
||||
"这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。": "Use this report to quickly understand the generated skill's role, principles, trigger boundary, and deliverables.",
|
||||
"先确认重复任务、真实输入形态和可交付输出,再决定是否继续加 references、scripts 或 evals。": "Clarify the recurring job, real input shape, and deliverable output before adding references, scripts, or evals.",
|
||||
"如果需求仍然模糊,优先回到 intent dialogue 收紧边界,再扩展包体结构。": "If the request is still fuzzy, tighten the boundary through intent dialogue before expanding the package.",
|
||||
"已生成 Output Review Adjudication,可记录盲评决策、一致率和待评审项。": "Output Review Adjudication is generated to record blind-review decisions, agreement rate, and pending cases.",
|
||||
"已生成 Output Execution Runs,可区分记录样本、命令执行和模型执行证据。": "Output Execution Runs is generated to distinguish recorded fixtures, command runs, and model-run evidence.",
|
||||
"尚未生成盲评审定报告。": "The blind review adjudication report has not been generated yet.",
|
||||
"尚未生成输出执行证据报告。": "The output execution evidence report has not been generated yet.",
|
||||
"先记录 reviewer 对 A/B 的选择,再打开答案 key 计算一致率。": "Record the reviewer's A/B choice before opening the answer key and calculating agreement.",
|
||||
"缺少真实 reviewer 决策时只显示待评审,不伪造人工结论。": "When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.",
|
||||
"recorded fixture 只能证明可复现样本,不等同于模型执行。": "A recorded fixture proves reproducible samples only; it is not model execution.",
|
||||
"只有 provider runner 返回 model metadata 时才计入 model-executed。": "Only provider runners that return model metadata count as model-executed.",
|
||||
}
|
||||
|
||||
MODE_ZH = {
|
||||
"scaffold": "脚手架",
|
||||
"production": "生产",
|
||||
"library": "库级",
|
||||
"governed": "治理",
|
||||
"manual": "手动",
|
||||
"inline": "内联",
|
||||
"agent-skills": "Agent Skills",
|
||||
}
|
||||
|
||||
PACKAGE_LABEL_ZH = {
|
||||
"SKILL.md": "Skill 入口文件",
|
||||
"README.md": "人类可读使用说明",
|
||||
"agents/interface.yaml": "跨平台接口元数据",
|
||||
"manifest.json": "生命周期与打包元数据",
|
||||
"references": "扩展指导与复用资料",
|
||||
"scripts": "确定性脚本或本地工具",
|
||||
"evals": "触发与质量检查",
|
||||
"reports": "生成的证据与总结报告",
|
||||
}
|
||||
|
||||
KIND_ZH = {"file": "文件", "folder": "目录"}
|
||||
|
||||
LABEL_EN = {
|
||||
"强项": "Strength",
|
||||
"缺口": "Gap",
|
||||
"保留并复用": "Keep",
|
||||
"纳入下一轮修复": "Fix next",
|
||||
}
|
||||
|
||||
|
||||
def contains_cjk(text: str) -> bool:
|
||||
return any("\u4e00" <= char <= "\u9fff" for char in str(text))
|
||||
|
||||
|
||||
def zh_for(text: str) -> str:
|
||||
value = str(text).strip()
|
||||
if not value:
|
||||
return ""
|
||||
if value in TEXT_ZH:
|
||||
return TEXT_ZH[value]
|
||||
if value in TEXT_EN or contains_cjk(value):
|
||||
return value
|
||||
if value.startswith("Use this skill when the request matches:"):
|
||||
return "当用户请求与该 Skill 的触发描述匹配时使用。"
|
||||
if value.startswith("用户说出类似需求时:"):
|
||||
return "当用户提出与该 Skill 触发描述相近的请求时使用。"
|
||||
if value.startswith("Use $") and " when you need to " in value:
|
||||
skill, need = value.removeprefix("Use ").split(" when you need to ", 1)
|
||||
return f"当你需要{zh_for(need).rstrip('。')}时使用 `{skill}`。"
|
||||
if value.startswith("Read the strongest pattern from "):
|
||||
repo = value.removeprefix("Read the strongest pattern from ").rstrip(".")
|
||||
return f"阅读 `{repo}` 中最值得借鉴的模式。"
|
||||
if value.startswith("Primary prompt task family:"):
|
||||
return "主要提示任务类型已记录在 prompt quality profile 中。"
|
||||
if value.startswith("Complexity:"):
|
||||
return "复杂度判断已记录在 prompt quality profile 中。"
|
||||
if value.startswith("Stability:"):
|
||||
return "系统稳定性评分已记录在 system model 中。"
|
||||
if value.startswith("Owned job:"):
|
||||
return "负责的核心任务已在 system model 中说明。"
|
||||
if value.startswith("Leverage:"):
|
||||
return "关键杠杆点已在 system model 中说明。"
|
||||
return "原始说明可切换到英文查看;默认中文报告保留结论与结构说明。"
|
||||
|
||||
|
||||
def en_for(text: str) -> str:
|
||||
value = str(text).strip()
|
||||
return TEXT_EN.get(value, value)
|
||||
|
||||
|
||||
def bi_span(zh: str, en: str | None = None) -> str:
|
||||
english = en if en is not None else en_for(zh)
|
||||
return (
|
||||
f'<span data-lang="zh-CN">{html.escape(str(zh))}</span>'
|
||||
f'<span data-lang="en">{html.escape(str(english))}</span>'
|
||||
)
|
||||
|
||||
|
||||
def bi_item(text: str) -> str:
|
||||
return bi_span(zh_for(text), en_for(text))
|
||||
|
||||
|
||||
def mode_zh(value: str) -> str:
|
||||
return MODE_ZH.get(str(value), str(value))
|
||||
|
||||
|
||||
def readable_description_zh(description: str) -> str:
|
||||
if contains_cjk(description):
|
||||
return description
|
||||
return "该 Skill 的触发描述来自 SKILL.md frontmatter;默认中文报告先呈现能力边界,原始英文描述可切换到英文查看。"
|
||||
|
||||
|
||||
def render_list(items: list[str], class_name: str = "list") -> str:
|
||||
if not items:
|
||||
return f'<ul class="{class_name}"><li>{bi_span("暂无记录。", "No records yet.")}</li></ul>'
|
||||
@@ -192,7 +38,7 @@ def render_metric_cards(scorecard: dict) -> str:
|
||||
cards.append(
|
||||
"<article class='metric-card'>"
|
||||
"<div class='metric-card-head'>"
|
||||
f"<span class='metric-label'>{html.escape(str(item.get('label', key)))}</span>"
|
||||
f"<span class='metric-label'>{bi_item(str(item.get('label', key)))}</span>"
|
||||
f"<strong>{html.escape(str(item.get('score', 'n/a')))}</strong>"
|
||||
"</div>"
|
||||
f"<div class='metric-card-body'>{reasons}</div>"
|
||||
@@ -219,7 +65,7 @@ def render_metric_summary(scorecard: dict) -> str:
|
||||
items.append(
|
||||
"<li>"
|
||||
f"<span class='metric-status'>{bi_span(verdict, verdict_en)}</span>"
|
||||
f"<b>{html.escape(label)}</b>"
|
||||
f"<b>{bi_item(label)}</b>"
|
||||
f"<em>{score}</em>"
|
||||
f"<small>{bi_item(reason)}</small>"
|
||||
"</li>"
|
||||
@@ -242,6 +88,118 @@ def render_audit_rows(items: list[dict]) -> str:
|
||||
return "".join(rows)
|
||||
|
||||
|
||||
WORLD_CLASS_CHECK_COPY = {
|
||||
"Provider model run": ("提供商实跑", "Provider model run"),
|
||||
"Token usage observed": ("Token 用量", "Token usage observed"),
|
||||
"No pending decisions": ("无待判定", "No pending decisions"),
|
||||
"Judgments complete": ("盲评完成", "Judgments complete"),
|
||||
"Native enforcement": ("原生执行", "Native enforcement"),
|
||||
"External events": ("外部事件", "External events"),
|
||||
"Adoption sample": ("采用样本", "Adoption sample"),
|
||||
}
|
||||
|
||||
|
||||
def render_world_class_check(check: object) -> str:
|
||||
if isinstance(check, dict):
|
||||
label_en = str(check.get("label_en") or check.get("label") or "")
|
||||
label_zh = str(check.get("label_zh") or "")
|
||||
else:
|
||||
label_en = str(check)
|
||||
label_zh = ""
|
||||
label_zh, label_en = WORLD_CLASS_CHECK_COPY.get(label_en, (label_zh or label_en, label_en))
|
||||
return f"<li>{bi_span(label_zh, label_en)}</li>"
|
||||
|
||||
|
||||
def render_world_class_readiness(readiness: dict) -> str:
|
||||
if not readiness:
|
||||
return ""
|
||||
kpis = [
|
||||
(
|
||||
"待补证据",
|
||||
"Pending",
|
||||
readiness.get("pending_count", 0),
|
||||
"仍需外部或人工证据接受。",
|
||||
"External or human evidence still needs acceptance.",
|
||||
),
|
||||
(
|
||||
"已接受",
|
||||
"Accepted",
|
||||
readiness.get("accepted_count", 0),
|
||||
"已通过 source check 与提交契约。",
|
||||
"Passed source checks and submission contract.",
|
||||
),
|
||||
(
|
||||
"源检查",
|
||||
"Source Checks",
|
||||
f"{readiness.get('source_pass_count', 0)} / {readiness.get('source_check_count', 0)}",
|
||||
"通过数 / 总检查数。",
|
||||
"Passed checks / total checks.",
|
||||
),
|
||||
]
|
||||
kpi_html = "".join(
|
||||
"<article class='evidence-kpi'>"
|
||||
f"<span>{bi_span(zh, en)}</span>"
|
||||
f"<strong>{html.escape(str(value))}</strong>"
|
||||
f"<small>{bi_span(note_zh, note_en)}</small>"
|
||||
"</article>"
|
||||
for zh, en, value, note_zh, note_en in kpis
|
||||
)
|
||||
entries = readiness.get("entries", [])
|
||||
if entries:
|
||||
blocks = []
|
||||
for item in entries:
|
||||
blocked_checks = [
|
||||
str(check).strip()
|
||||
for check in item.get("blocked_checks", [])
|
||||
if str(check).strip()
|
||||
]
|
||||
if blocked_checks:
|
||||
checks_html = (
|
||||
"<ul class='blocked-checks'>"
|
||||
+ "".join(render_world_class_check(check) for check in blocked_checks)
|
||||
+ "</ul>"
|
||||
)
|
||||
else:
|
||||
checks_html = (
|
||||
"<p class='blocked-checks-empty'>"
|
||||
f"{bi_span('当前没有阻塞检查。', 'No blocked checks right now.')}"
|
||||
"</p>"
|
||||
)
|
||||
blocks.append(
|
||||
"<article class='evidence-item'>"
|
||||
"<div>"
|
||||
f"<span>{bi_span(str(item.get('category_zh', '外部证据')), str(item.get('category_en', 'External evidence')))}</span>"
|
||||
f"<h4>{bi_span(str(item.get('label_zh', item.get('key', '证据项'))), str(item.get('label_en', item.get('key', 'Evidence item'))))}</h4>"
|
||||
"</div>"
|
||||
f"<p>{bi_span(str(item.get('summary_zh', '仍待补充证据。')), str(item.get('summary_en', 'Evidence is still pending.')))}</p>"
|
||||
f"<h5>{bi_span('阻塞检查', 'Blocked Checks')}</h5>"
|
||||
f"{checks_html}"
|
||||
"</article>"
|
||||
)
|
||||
entry_html = "".join(blocks)
|
||||
else:
|
||||
entry_html = (
|
||||
"<article class='evidence-item empty'>"
|
||||
f"<p>{bi_span('尚未生成 world-class ledger;这里只保留反过度承诺提示。', 'No world-class ledger has been generated; this panel keeps the anti-overclaim guard visible.')}</p>"
|
||||
"</article>"
|
||||
)
|
||||
status = "可宣称" if readiness.get("ready") else "证据待补"
|
||||
status_en = "Claim-ready" if readiness.get("ready") else "Evidence pending"
|
||||
return (
|
||||
"<article class='panel world-readiness'>"
|
||||
"<div class='world-readiness-head'>"
|
||||
"<div>"
|
||||
f"<h3>{bi_span('世界证据', 'World Evidence')}</h3>"
|
||||
f"<p>{bi_span(str(readiness.get('conclusion_zh', '世界级证据状态未知。')), str(readiness.get('conclusion_en', 'World-class evidence status is unknown.')))}</p>"
|
||||
"</div>"
|
||||
f"<span class='world-status'>{bi_span(status, status_en)}</span>"
|
||||
"</div>"
|
||||
f"<div class='evidence-kpis'>{kpi_html}</div>"
|
||||
f"<div class='evidence-list'>{entry_html}</div>"
|
||||
"</article>"
|
||||
)
|
||||
|
||||
|
||||
def render_score_strip(scorecard: dict) -> str:
|
||||
keys = ["completeness_score", "trigger_score", "evidence_score", "context_cost"]
|
||||
cards = []
|
||||
@@ -253,7 +211,7 @@ def render_score_strip(scorecard: dict) -> str:
|
||||
reason = item.get("reasons", [""])[0]
|
||||
cards.append(
|
||||
"<article class='score-chip'>"
|
||||
f"<span>{html.escape(str(item.get('label', key)))}</span>"
|
||||
f"<span>{bi_item(str(item.get('label', key)))}</span>"
|
||||
f"<strong>{score}</strong>"
|
||||
f"<i style='--score:{score}%'></i>"
|
||||
f"<small>{bi_item(str(reason))}</small>"
|
||||
@@ -289,12 +247,16 @@ def render_html(summary: dict) -> str:
|
||||
contract = summary.get("contract_boundary", {})
|
||||
quality = summary.get("quality_review", {})
|
||||
risk = summary.get("risk_governance", {})
|
||||
world_readiness = summary.get("world_class_readiness", {})
|
||||
assets = summary.get("package_assets", {})
|
||||
roadmap = summary.get("iteration_roadmap", {})
|
||||
output_execution = summary.get("output_execution", {})
|
||||
output_execution_summary = output_execution.get("summary", {})
|
||||
output_review = summary.get("output_review_adjudication", {})
|
||||
output_review_summary = output_review.get("summary", {})
|
||||
report_contract = summary.get("report_contract", {})
|
||||
html_report_path = str(report_contract.get("html_report") or "reports/skill-overview.html")
|
||||
open_report_message = f"创建完成后建议先打开 {html_report_path},再继续扩展包体。"
|
||||
hero_meta = [
|
||||
(f"技能名称:{summary['name']}", f"Skill name: {summary['name']}"),
|
||||
(f"成熟度:{mode_zh(metadata.get('maturity_tier', 'scaffold'))}", f"Maturity: {metadata.get('maturity_tier', 'scaffold')}"),
|
||||
@@ -375,13 +337,13 @@ def render_html(summary: dict) -> str:
|
||||
<p class="eyebrow">{bi_span("YAO Skill 生成审计报告", "YAO Skill Generation Audit")}</p>
|
||||
<p class="slug">{html.escape(summary['name'])}</p>
|
||||
<h1 id="report-title">{bi_span("技能审计报告", f"{summary['display_name']} Audit Report")}</h1>
|
||||
<p class="lead">{bi_span("这份报告默认使用中文简体,把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。", summary["description"])}</p>
|
||||
<p class="lead">{bi_span("这份报告默认使用中文简体,把新 Skill 的定位、指标、原理、契约、质量、风险、资产和迭代路线整理为一份可审计的 HTML 报告。", en_for(str(summary["description"])))}</p>
|
||||
<div class="hero-meta">{hero_meta_html}</div>
|
||||
<div class="badges">{target_badges}</div>
|
||||
</div>
|
||||
<aside class="hero-card">
|
||||
<h3>{bi_span("核心判断", "Core reading")}</h3>
|
||||
{render_list([skill.get("core_value", ""), skill.get("audience", ""), "创建完成后建议先打开 reports/skill-overview.html,再继续扩展包体。"])}
|
||||
{render_list([skill.get("core_value", ""), skill.get("audience", ""), open_report_message])}
|
||||
</aside>
|
||||
</div>
|
||||
<div class="score-strip" aria-label="报告关键指标">{score_strip}</div>
|
||||
@@ -409,17 +371,24 @@ def render_html(summary: dict) -> str:
|
||||
<h2>{bi_span("总览指标", "Metrics")}</h2>
|
||||
<p>{bi_span("分数来自本地文件和 reports 证据,缺失时明确标为证据不足。", "Scores are derived from package files and reports; missing inputs are shown as evidence gaps.")}</p>
|
||||
</div>
|
||||
<div>
|
||||
<div class="metrics-stack">
|
||||
<div class="metrics-lead">
|
||||
{charts["radar"]}
|
||||
<article class="panel metrics-note">
|
||||
<h3>{bi_span("指标判读", "Reading")}</h3>
|
||||
<p>{bi_span("先看雷达图判断能力短板,再看下方每项分数的证据原因。分数不是装饰数字,必须和本地文件、reports 证据或证据不足提示对应。", "Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.")}</p>
|
||||
{render_metric_summary(scorecard)}
|
||||
</article>
|
||||
</div>
|
||||
<div class="metric-grid">{render_metric_cards(scorecard)}</div>
|
||||
<article class="panel metrics-note">
|
||||
<h3>{bi_span("指标判读", "Reading")}</h3>
|
||||
<p>{bi_span("先看雷达图判断能力短板,再看每项分数的证据原因。分数不是装饰数字,必须和本地文件、reports 证据或证据不足提示对应。", "Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.")}</p>
|
||||
</article>
|
||||
</div>
|
||||
<div class="section-body metrics-report">
|
||||
<div class="metrics-flow">
|
||||
<div class="metrics-primary" aria-label="指标图表与成熟度摘要">
|
||||
{charts["radar"]}
|
||||
<article class="panel metrics-note metrics-summary-panel">
|
||||
<h3>{bi_span("成熟度条", "Maturity Bar")}</h3>
|
||||
<p>{bi_span("这里把每个指标压缩为状态、分数和第一条证据,先给出整体判断,再进入下方明细。", "This compresses each metric into status, score, and the first evidence item before the detailed cards below.")}</p>
|
||||
{render_metric_summary(scorecard)}
|
||||
</article>
|
||||
</div>
|
||||
<div class="metric-detail-section">
|
||||
<div class="detail-section-kicker">{bi_span("指标明细", "Metric Details")}</div>
|
||||
<div class="metric-grid metric-detail-grid">{render_metric_cards(scorecard)}</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -472,7 +441,7 @@ def render_html(summary: dict) -> str:
|
||||
<div class="two-col">
|
||||
<article class="panel">
|
||||
<h3>{bi_span("触发描述", "Trigger")}</h3>
|
||||
<p>{bi_span(readable_description_zh(str(trigger.get("description", ""))), str(trigger.get("description", "")))}</p>
|
||||
<p>{bi_span(readable_description_zh(str(trigger.get("description", ""))), en_for(str(trigger.get("description", ""))))}</p>
|
||||
<h3>{bi_span("输入材料", "Inputs")}</h3>
|
||||
{render_list(contract.get("inputs", []))}
|
||||
</article>
|
||||
@@ -536,6 +505,7 @@ def render_html(summary: dict) -> str:
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
{render_world_class_readiness(world_readiness)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -4,8 +4,6 @@ from __future__ import annotations
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
|
||||
|
||||
WIDTH = 1280
|
||||
HEIGHT = 640
|
||||
@@ -20,15 +18,19 @@ PNG_PATH = OUT_DIR / "social-preview.png"
|
||||
SVG_PATH = OUT_DIR / "social-preview.svg"
|
||||
|
||||
|
||||
def font(path: str, size: int) -> ImageFont.FreeTypeFont:
|
||||
return ImageFont.truetype(path, size)
|
||||
|
||||
|
||||
SERIF = "/System/Library/Fonts/Supplemental/Georgia.ttf"
|
||||
SANS = "/System/Library/Fonts/SFNS.ttf"
|
||||
|
||||
|
||||
def draw_png() -> None:
|
||||
try:
|
||||
from PIL import Image, ImageDraw, ImageFont
|
||||
except ImportError as exc:
|
||||
raise SystemExit("Pillow is required to render PNG social previews.") from exc
|
||||
|
||||
def font(path: str, size: int):
|
||||
return ImageFont.truetype(path, size)
|
||||
|
||||
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
image = Image.new("RGB", (WIDTH, HEIGHT), BACKGROUND)
|
||||
draw = ImageDraw.Draw(image)
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_adoption_drift_report import SENSITIVE_FIELDS, display_path
|
||||
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
DEFAULT_RECIPES = [
|
||||
{
|
||||
"id": "browser-extension",
|
||||
"client": "Browser extension",
|
||||
"command": "browser-extension",
|
||||
"event": "skill_activation",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"trigger_points": [
|
||||
"Skill is explicitly selected by the user.",
|
||||
"Skill activation is accepted by the host client.",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chrome-extension",
|
||||
"client": "Chrome extension",
|
||||
"command": "chrome-extension",
|
||||
"event": "skill_output",
|
||||
"activation_type": "manual",
|
||||
"outcome": "edited",
|
||||
"failure_type": "none",
|
||||
"trigger_points": [
|
||||
"User keeps or edits a skill-generated artifact.",
|
||||
"Extension records only the final outcome class, not page content.",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "vscode-extension",
|
||||
"client": "VS Code extension",
|
||||
"command": "vscode-extension",
|
||||
"event": "skill_activation",
|
||||
"activation_type": "implicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"trigger_points": [
|
||||
"Workspace skill route chooses this skill.",
|
||||
"Extension records route outcome without file paths or code snippets.",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "cli-wrapper",
|
||||
"client": "CLI wrapper",
|
||||
"command": "cli-wrapper",
|
||||
"event": "script_run",
|
||||
"activation_type": "manual",
|
||||
"outcome": "unknown",
|
||||
"failure_type": "none",
|
||||
"trigger_points": [
|
||||
"Wrapper starts or finishes a known skill workflow command.",
|
||||
"Wrapper records command family only, never command arguments.",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "provider-adapter",
|
||||
"client": "Provider adapter",
|
||||
"command": "provider-adapter",
|
||||
"event": "skill_output",
|
||||
"activation_type": "manual",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"trigger_points": [
|
||||
"Provider adapter receives an accepted or rejected skill output signal.",
|
||||
"Adapter records only normalized outcome metadata.",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def default_spool(skill_dir: Path) -> Path:
|
||||
return skill_dir / ".yao" / "telemetry_spool" / "external_events.jsonl"
|
||||
|
||||
|
||||
def command_text(argv: list[str]) -> str:
|
||||
return shlex.join(argv)
|
||||
|
||||
|
||||
def emit_argv(skill_dir: Path, output_jsonl: Path, recipe: dict[str, Any], dry_run: bool = False) -> list[str]:
|
||||
argv = [
|
||||
"python3",
|
||||
"scripts/yao.py",
|
||||
"telemetry-emit",
|
||||
display_path(skill_dir),
|
||||
"--output-jsonl",
|
||||
display_path(output_jsonl),
|
||||
"--event",
|
||||
recipe["event"],
|
||||
"--activation-type",
|
||||
recipe["activation_type"],
|
||||
"--outcome",
|
||||
recipe["outcome"],
|
||||
"--failure-type",
|
||||
recipe["failure_type"],
|
||||
"--command",
|
||||
recipe["command"],
|
||||
]
|
||||
if dry_run:
|
||||
argv.append("--dry-run")
|
||||
return argv
|
||||
|
||||
|
||||
def import_argv(skill_dir: Path, output_jsonl: Path) -> list[str]:
|
||||
return [
|
||||
"python3",
|
||||
"scripts/yao.py",
|
||||
"telemetry-import",
|
||||
display_path(skill_dir),
|
||||
"--input-jsonl",
|
||||
display_path(output_jsonl),
|
||||
]
|
||||
|
||||
|
||||
def build_recipe(skill_dir: Path, output_jsonl: Path, recipe: dict[str, Any]) -> dict[str, Any]:
|
||||
emit = emit_argv(skill_dir, output_jsonl, recipe)
|
||||
dry_run = emit_argv(skill_dir, output_jsonl, recipe, dry_run=True)
|
||||
return {
|
||||
**recipe,
|
||||
"source": "external",
|
||||
"native_auto_capture": False,
|
||||
"integration_status": "client-hook-recipe",
|
||||
"metadata_only": True,
|
||||
"emit_argv": emit,
|
||||
"emit_command": command_text(emit),
|
||||
"dry_run_argv": dry_run,
|
||||
"dry_run_command": command_text(dry_run),
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# Telemetry Hook Recipes",
|
||||
"",
|
||||
"These recipes show how Browser, Chrome, IDE, wrapper, or provider-side integrations can call `telemetry-emit` without collecting raw prompts, outputs, transcripts, messages, notes, arguments, or file content.",
|
||||
"",
|
||||
"They are client hook recipes, not proof that a host client is already natively integrated.",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- Recipe count: `{report['summary']['recipe_count']}`",
|
||||
f"- Native auto-capture integrations claimed: `{report['summary']['native_auto_capture_count']}`",
|
||||
f"- Default spool: `{report['artifacts']['spool_jsonl']}`",
|
||||
"",
|
||||
"## Recipes",
|
||||
"",
|
||||
"| Client | Command | Event | Outcome | Dry run |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for recipe in report["recipes"]:
|
||||
lines.append(
|
||||
f"| {recipe['client']} | `{recipe['command']}` | `{recipe['event']}` | `{recipe['outcome']}` | `{recipe['dry_run_command']}` |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Import",
|
||||
"",
|
||||
"After a client finishes a batch, import the local spool:",
|
||||
"",
|
||||
"```bash",
|
||||
report["artifacts"]["import_command"],
|
||||
"```",
|
||||
"",
|
||||
"## Privacy Contract",
|
||||
"",
|
||||
"- Allowed fields: `event`, `skill`, `version`, `source`, `command`, `activation_type`, `outcome`, `failure_type`, `timestamp`.",
|
||||
f"- Blocked raw-content fields: `{', '.join(report['privacy_contract']['blocked_fields'])}`.",
|
||||
"- Client integrations must map local state to normalized outcome classes before calling the hook.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def render_recipes(
|
||||
skill_dir: Path,
|
||||
output_json: Path | None = None,
|
||||
output_md: Path | None = None,
|
||||
output_jsonl: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
output_json = (output_json or reports_dir / "telemetry_hook_recipes.json").resolve()
|
||||
output_md = (output_md or reports_dir / "telemetry_hook_recipes.md").resolve()
|
||||
output_jsonl = (output_jsonl or default_spool(skill_dir)).resolve()
|
||||
recipes = [build_recipe(skill_dir, output_jsonl, recipe) for recipe in DEFAULT_RECIPES]
|
||||
import_cmd = import_argv(skill_dir, output_jsonl)
|
||||
report = {
|
||||
"ok": True,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"skill_dir": display_path(skill_dir),
|
||||
"summary": {
|
||||
"recipe_count": len(recipes),
|
||||
"native_auto_capture_count": sum(1 for recipe in recipes if recipe["native_auto_capture"]),
|
||||
"metadata_only_recipe_count": sum(1 for recipe in recipes if recipe["metadata_only"]),
|
||||
},
|
||||
"privacy_contract": {
|
||||
"raw_content_allowed": False,
|
||||
"blocked_fields": sorted(SENSITIVE_FIELDS),
|
||||
"allowed_fields": [
|
||||
"event",
|
||||
"skill",
|
||||
"version",
|
||||
"source",
|
||||
"command",
|
||||
"activation_type",
|
||||
"outcome",
|
||||
"failure_type",
|
||||
"timestamp",
|
||||
],
|
||||
},
|
||||
"recipes": recipes,
|
||||
"artifacts": {
|
||||
"json": display_path(output_json),
|
||||
"markdown": display_path(output_md),
|
||||
"spool_jsonl": display_path(output_jsonl),
|
||||
"import_command": command_text(import_cmd),
|
||||
"import_argv": import_cmd,
|
||||
},
|
||||
"warnings": [
|
||||
"These recipes do not prove native host-client integration; they define the local metadata-only hook contract for those integrations."
|
||||
],
|
||||
"failures": [],
|
||||
}
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
return report
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render metadata-only telemetry client hook recipes.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--output-jsonl")
|
||||
args = parser.parse_args()
|
||||
report = render_recipes(
|
||||
Path(args.skill_dir),
|
||||
output_json=Path(args.output_json) if args.output_json else None,
|
||||
output_md=Path(args.output_md) if args.output_md else None,
|
||||
output_jsonl=Path(args.output_jsonl) if args.output_jsonl else None,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,433 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a weekly SkillOps curator report from generated reports without scanning private logs or writing source files."
|
||||
|
||||
SUMMARY_FIELDS = [
|
||||
"decision",
|
||||
"week_id",
|
||||
"daily_report_count",
|
||||
"opportunity_count",
|
||||
"unique_opportunity_count",
|
||||
"ready_for_approval_review_count",
|
||||
"proposal_review_count",
|
||||
"observe_more_evidence_count",
|
||||
"report_only_count",
|
||||
"top_score",
|
||||
"skill_count",
|
||||
"actionable_portfolio_issue_count",
|
||||
"release_lock_ready",
|
||||
"evidence_consistency_ok",
|
||||
"public_world_class_ready",
|
||||
"world_class_pending_count",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
"failure_count",
|
||||
]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def parse_date(value: str) -> date:
|
||||
candidate = value[:10] if len(value) >= 10 else value
|
||||
try:
|
||||
return date.fromisoformat(candidate)
|
||||
except ValueError:
|
||||
return date.today()
|
||||
|
||||
|
||||
def week_id(generated_at: str) -> str:
|
||||
iso = parse_date(generated_at).isocalendar()
|
||||
return f"{iso.year}-W{iso.week:02d}"
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return f"[external-explicit-source]/{path.name}"
|
||||
|
||||
|
||||
def resolve_output(skill_dir: Path, value: str) -> Path:
|
||||
path = Path(value)
|
||||
return path if path.is_absolute() else skill_dir / path
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def write_text(path: Path, text: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def default_weekly_path(skill_dir: Path, generated_at: str, suffix: str) -> Path:
|
||||
return skill_dir / "reports" / "skillops" / "weekly" / f"{week_id(generated_at)}.{suffix}"
|
||||
|
||||
|
||||
def daily_report_paths(skill_dir: Path, explicit_paths: list[Path]) -> list[Path]:
|
||||
if explicit_paths:
|
||||
return [path if path.is_absolute() else skill_dir / path for path in explicit_paths]
|
||||
daily_dir = skill_dir / "reports" / "skillops" / "daily"
|
||||
if not daily_dir.exists():
|
||||
return []
|
||||
return sorted(daily_dir.glob("*.json"))
|
||||
|
||||
|
||||
def _as_int(value: Any, default: int = 0) -> int:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _dedupe_opportunities(daily_reports: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
by_id: dict[str, dict[str, Any]] = {}
|
||||
for report in daily_reports:
|
||||
report_date = str(report.get("generated_at") or "")[:10]
|
||||
for item in report.get("opportunities", []):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
opportunity_id = str(item.get("opportunity_id") or "")
|
||||
if not opportunity_id:
|
||||
continue
|
||||
current = {**item, "daily_report_date": report_date}
|
||||
existing = by_id.get(opportunity_id)
|
||||
if existing is None or _as_int(current.get("score")) > _as_int(existing.get("score")):
|
||||
by_id[opportunity_id] = current
|
||||
return sorted(by_id.values(), key=lambda item: (-_as_int(item.get("score")), str(item.get("opportunity_id"))))
|
||||
|
||||
|
||||
def _counter(items: list[dict[str, Any]], key: str) -> dict[str, int]:
|
||||
values = Counter(str(item.get(key) or "") for item in items)
|
||||
values.pop("", None)
|
||||
return dict(sorted(values.items()))
|
||||
|
||||
|
||||
def _portfolio_issue_count(atlas_summary: dict[str, Any]) -> int:
|
||||
keys = [
|
||||
"actionable_route_collision_count",
|
||||
"actionable_owner_gap_count",
|
||||
"actionable_stale_count",
|
||||
"actionable_drift_signal_count",
|
||||
"no_route_opportunity_count",
|
||||
]
|
||||
return sum(_as_int(atlas_summary.get(key)) for key in keys)
|
||||
|
||||
|
||||
def build_actions(summary: dict[str, Any], opportunities: list[dict[str, Any]], atlas_summary: dict[str, Any]) -> list[dict[str, str]]:
|
||||
actions: list[dict[str, str]] = []
|
||||
if summary["ready_for_approval_review_count"]:
|
||||
actions.append(
|
||||
{
|
||||
"key": "review-ready-opportunities",
|
||||
"priority": "high",
|
||||
"action": "Review the top ready-for-approval SkillOps opportunities before preparing any approval ledger entry.",
|
||||
}
|
||||
)
|
||||
elif opportunities:
|
||||
actions.append(
|
||||
{
|
||||
"key": "review-proposal-queue",
|
||||
"priority": "medium",
|
||||
"action": "Review proposal-level SkillOps opportunities and keep low-evidence items in observation.",
|
||||
}
|
||||
)
|
||||
if summary["actionable_portfolio_issue_count"]:
|
||||
actions.append(
|
||||
{
|
||||
"key": "triage-skill-library",
|
||||
"priority": "high",
|
||||
"action": "Triage actionable route collisions, owner gaps, stale skills, drift signals, or no-route opportunities from Skill Atlas.",
|
||||
}
|
||||
)
|
||||
if summary["world_class_pending_count"]:
|
||||
actions.append(
|
||||
{
|
||||
"key": "close-world-class-evidence",
|
||||
"priority": "high",
|
||||
"action": "Collect accepted external or human evidence for pending world-class ledger entries before public claims.",
|
||||
}
|
||||
)
|
||||
if summary["evidence_consistency_ok"] is not True:
|
||||
actions.append(
|
||||
{
|
||||
"key": "refresh-evidence-consistency",
|
||||
"priority": "high",
|
||||
"action": "Regenerate evidence consistency before using this curator report for release decisions.",
|
||||
}
|
||||
)
|
||||
if not atlas_summary:
|
||||
actions.append(
|
||||
{
|
||||
"key": "refresh-skill-atlas",
|
||||
"priority": "medium",
|
||||
"action": "Generate Skill Atlas so weekly curation can see portfolio-level drift and stale-skill signals.",
|
||||
}
|
||||
)
|
||||
if not actions:
|
||||
actions.append({"key": "monitor", "priority": "low", "action": "No curation action required beyond routine monitoring."})
|
||||
return actions
|
||||
|
||||
|
||||
def build_report(skill_dir: Path, generated_at: str, daily_json: list[Path] | None = None) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
failures: list[str] = []
|
||||
|
||||
daily_paths = daily_report_paths(skill_dir, daily_json or [])
|
||||
daily_reports = []
|
||||
for path in daily_paths:
|
||||
payload = load_json(path)
|
||||
if payload.get("ok") is True:
|
||||
daily_reports.append(payload)
|
||||
else:
|
||||
failures.append(f"Daily SkillOps report is missing or invalid: {display_path(path, skill_dir)}")
|
||||
|
||||
opportunities = _dedupe_opportunities(daily_reports)
|
||||
decision_counts = _counter(opportunities, "decision")
|
||||
action_counts = _counter(opportunities, "action_type")
|
||||
risk_counts = _counter(opportunities, "risk_level")
|
||||
|
||||
atlas = load_json(reports_dir / "skill_atlas.json")
|
||||
atlas_summary = atlas.get("summary", {}) if isinstance(atlas.get("summary"), dict) else {}
|
||||
benchmark = load_json(reports_dir / "benchmark_reproducibility.json")
|
||||
benchmark_summary = benchmark.get("summary", {}) if isinstance(benchmark.get("summary"), dict) else {}
|
||||
consistency = load_json(reports_dir / "evidence_consistency.json")
|
||||
ledger = load_json(reports_dir / "world_class_evidence_ledger.json")
|
||||
ledger_summary = ledger.get("summary", {}) if isinstance(ledger.get("summary"), dict) else {}
|
||||
|
||||
portfolio_issue_count = _portfolio_issue_count(atlas_summary)
|
||||
failure_count = len(failures)
|
||||
summary = {
|
||||
"decision": "blocked"
|
||||
if failure_count
|
||||
else "curator-review"
|
||||
if opportunities or portfolio_issue_count or _as_int(ledger_summary.get("pending_count"))
|
||||
else "monitor",
|
||||
"week_id": week_id(generated_at),
|
||||
"daily_report_count": len(daily_reports),
|
||||
"opportunity_count": sum(_as_int(report.get("opportunity_summary", {}).get("opportunity_count")) for report in daily_reports),
|
||||
"unique_opportunity_count": len(opportunities),
|
||||
"ready_for_approval_review_count": decision_counts.get("ready_for_approval_review", 0),
|
||||
"proposal_review_count": decision_counts.get("proposal_review", 0),
|
||||
"observe_more_evidence_count": decision_counts.get("observe_more_evidence", 0),
|
||||
"report_only_count": decision_counts.get("report_only", 0),
|
||||
"top_score": _as_int(opportunities[0].get("score")) if opportunities else 0,
|
||||
"skill_count": _as_int(atlas_summary.get("skill_count")),
|
||||
"actionable_portfolio_issue_count": portfolio_issue_count,
|
||||
"release_lock_ready": benchmark_summary.get("release_lock_ready") is True,
|
||||
"evidence_consistency_ok": consistency.get("ok") is True,
|
||||
"public_world_class_ready": ledger_summary.get("ready_to_claim_world_class") is True,
|
||||
"world_class_pending_count": _as_int(ledger_summary.get("pending_count")),
|
||||
"writes_source_files": False,
|
||||
"auto_patch_enabled": False,
|
||||
"failure_count": failure_count,
|
||||
}
|
||||
actions = build_actions(summary, opportunities, atlas_summary)
|
||||
curator_contract = {
|
||||
"schema_version": "1.0",
|
||||
"contract": "weekly-skillops-curator-report",
|
||||
"cadence": "weekly",
|
||||
"source_of_truth": [
|
||||
"reports/skillops/daily/*.json",
|
||||
"reports/skill_atlas.json",
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/evidence_consistency.json",
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
],
|
||||
"raw_content_stored": False,
|
||||
"redacted_or_generated_evidence_only": True,
|
||||
"proposal_only": True,
|
||||
"writes_source_files": False,
|
||||
"auto_patch_enabled": False,
|
||||
"approval_required_for_writes": True,
|
||||
"counts_as_world_class_evidence": False,
|
||||
}
|
||||
report = {
|
||||
"schema_version": "1.0",
|
||||
"ok": failure_count == 0,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
**{key: summary[key] for key in SUMMARY_FIELDS},
|
||||
"summary": summary,
|
||||
"curator_contract": curator_contract,
|
||||
"report_contract": {
|
||||
"schema_version": "1.0",
|
||||
"contract": "weekly-skillops-curator-report",
|
||||
"top_level_mirrors_summary": True,
|
||||
"summary_fields": SUMMARY_FIELDS,
|
||||
},
|
||||
"daily_reports": [display_path(path, skill_dir) for path in daily_paths],
|
||||
"opportunity_summary": {
|
||||
"action_type_counts": action_counts,
|
||||
"decision_counts": decision_counts,
|
||||
"risk_level_counts": risk_counts,
|
||||
},
|
||||
"curator_queue": opportunities[:20],
|
||||
"portfolio": {
|
||||
"skill_count": summary["skill_count"],
|
||||
"actionable_issue_count": portfolio_issue_count,
|
||||
"route_collision_count": _as_int(atlas_summary.get("route_collision_count")),
|
||||
"actionable_route_collision_count": _as_int(atlas_summary.get("actionable_route_collision_count")),
|
||||
"owner_gap_count": _as_int(atlas_summary.get("owner_gap_count")),
|
||||
"actionable_owner_gap_count": _as_int(atlas_summary.get("actionable_owner_gap_count")),
|
||||
"stale_count": _as_int(atlas_summary.get("stale_count")),
|
||||
"actionable_stale_count": _as_int(atlas_summary.get("actionable_stale_count")),
|
||||
"drift_signal_count": _as_int(atlas_summary.get("drift_signal_count")),
|
||||
"actionable_drift_signal_count": _as_int(atlas_summary.get("actionable_drift_signal_count")),
|
||||
"no_route_opportunity_count": _as_int(atlas_summary.get("no_route_opportunity_count")),
|
||||
},
|
||||
"release_state": {
|
||||
"release_lock_ready": summary["release_lock_ready"],
|
||||
"evidence_consistency_ok": summary["evidence_consistency_ok"],
|
||||
"public_world_class_ready": summary["public_world_class_ready"],
|
||||
"world_class_pending_count": summary["world_class_pending_count"],
|
||||
},
|
||||
"actions": actions,
|
||||
"failures": failures,
|
||||
"source_reports": {
|
||||
"daily_dir": "reports/skillops/daily",
|
||||
"skill_atlas": "reports/skill_atlas.json",
|
||||
"benchmark_reproducibility": "reports/benchmark_reproducibility.json",
|
||||
"evidence_consistency": "reports/evidence_consistency.json",
|
||||
"world_class_ledger": "reports/world_class_evidence_ledger.json",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": str(default_weekly_path(skill_dir, generated_at, "json").relative_to(skill_dir)),
|
||||
"markdown": str(default_weekly_path(skill_dir, generated_at, "md").relative_to(skill_dir)),
|
||||
},
|
||||
}
|
||||
return report
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Weekly SkillOps Curator Report",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
f"Week: `{summary['week_id']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
]
|
||||
for key in [
|
||||
"decision",
|
||||
"daily_report_count",
|
||||
"unique_opportunity_count",
|
||||
"ready_for_approval_review_count",
|
||||
"proposal_review_count",
|
||||
"top_score",
|
||||
"skill_count",
|
||||
"actionable_portfolio_issue_count",
|
||||
"release_lock_ready",
|
||||
"evidence_consistency_ok",
|
||||
"public_world_class_ready",
|
||||
"world_class_pending_count",
|
||||
]:
|
||||
lines.append(f"- {key}: `{summary[key]}`")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"This report is a weekly curator cockpit for generated SkillOps evidence. It does not scan private logs, write source files, apply patches, or count as world-class evidence.",
|
||||
"",
|
||||
"## Curator Boundary",
|
||||
"",
|
||||
]
|
||||
)
|
||||
contract = report["curator_contract"]
|
||||
for key in [
|
||||
"raw_content_stored",
|
||||
"redacted_or_generated_evidence_only",
|
||||
"proposal_only",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
"approval_required_for_writes",
|
||||
"counts_as_world_class_evidence",
|
||||
]:
|
||||
lines.append(f"- {key}: `{str(contract[key]).lower()}`")
|
||||
lines.extend(["", "## Actions", ""])
|
||||
for action in report["actions"]:
|
||||
lines.append(f"- `{action['priority']}` {action['action']}")
|
||||
lines.extend(["", "## Curator Queue", ""])
|
||||
if not report["curator_queue"]:
|
||||
lines.append("- No weekly curator opportunities.")
|
||||
for item in report["curator_queue"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {item.get('title', '')}",
|
||||
"",
|
||||
f"- ID: `{item.get('opportunity_id', '')}`",
|
||||
f"- Action: `{item.get('action_type', '')}`",
|
||||
f"- Decision: `{item.get('decision', '')}`",
|
||||
f"- Score: `{item.get('score', 0)}`",
|
||||
f"- Risk: `{item.get('risk_level', '')}`",
|
||||
f"- Source day: `{item.get('daily_report_date', '')}`",
|
||||
f"- Policy: {item.get('policy_reason', '')}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(["## Portfolio Signals", ""])
|
||||
for key, value in report["portfolio"].items():
|
||||
lines.append(f"- {key}: `{value}`")
|
||||
lines.extend(["", "## Evidence", ""])
|
||||
for label, path in report["source_reports"].items():
|
||||
lines.append(f"- {label}: `{path}`")
|
||||
if report["failures"]:
|
||||
lines.extend(["", "## Failures", ""])
|
||||
lines.extend(f"- {failure}" for failure in report["failures"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a weekly SkillOps curator report from generated SkillOps evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--daily-json", action="append", default=[], help="Explicit Daily SkillOps JSON report to include.")
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--generated-at", default=utc_now())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(skill_dir, args.generated_at, [Path(item) for item in args.daily_json])
|
||||
output_json = resolve_output(skill_dir, args.output_json) if args.output_json else default_weekly_path(skill_dir, args.generated_at, "json")
|
||||
output_md = resolve_output(skill_dir, args.output_md) if args.output_md else default_weekly_path(skill_dir, args.generated_at, "md")
|
||||
report["artifacts"] = {
|
||||
"json": display_path(output_json, skill_dir),
|
||||
"markdown": display_path(output_md, skill_dir),
|
||||
}
|
||||
write_json(output_json, report)
|
||||
write_text(output_md, render_markdown(report))
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Scans public claim surfaces so world-class completion is not claimed before accepted evidence exists."
|
||||
|
||||
FORBIDDEN_WHEN_PENDING = [
|
||||
{
|
||||
"key": "ready-to-claim-true",
|
||||
"pattern": re.compile(r"ready[\s_-]*to[\s_-]*claim[\s_-]*world[\s_-]*class\s*[:=]\s*`?true`?", re.IGNORECASE),
|
||||
"reason": "ready_to_claim_world_class cannot be true while ledger evidence is pending.",
|
||||
},
|
||||
{
|
||||
"key": "world-class-ready-true",
|
||||
"pattern": re.compile(r"(public\s+)?world[\s_-]*class[\s_-]*ready\s*[:=]\s*`?true`?", re.IGNORECASE),
|
||||
"reason": "world-class readiness cannot be claimed before accepted external and human evidence exists.",
|
||||
},
|
||||
{
|
||||
"key": "json-world-class-ready-true",
|
||||
"pattern": re.compile(r'"(?:ready_to_claim_world_class|world_class_ready|public_world_class_ready)"\s*:\s*true', re.IGNORECASE),
|
||||
"reason": "machine-readable claim fields must remain false until the ledger is ready.",
|
||||
},
|
||||
{
|
||||
"key": "completion-phrase",
|
||||
"pattern": re.compile(r"(public\s+)?world[\s_-]*class\s+(?:complete|completed|achieved|certified)", re.IGNORECASE),
|
||||
"reason": "completion language is blocked until the world-class ledger is accepted.",
|
||||
},
|
||||
{
|
||||
"key": "zh-completion-phrase",
|
||||
"pattern": re.compile(r"世界级(?:已完成|完成|已达成|达成|准备就绪|可声明|认证通过)"),
|
||||
"reason": "中文完成态表述必须等到 ledger ready 后才能出现。",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def resolve_claim_surface(raw_path: str, skill_dir: Path) -> Path:
|
||||
path = Path(raw_path)
|
||||
if not path.is_absolute():
|
||||
path = skill_dir / path
|
||||
return path.resolve()
|
||||
|
||||
|
||||
def should_scan_claim_surface(path: Path) -> bool:
|
||||
parts = path.parts
|
||||
if "release_snapshots" in parts:
|
||||
return False
|
||||
if path.name in {"world_class_claim_guard.md", "world_class_claim_guard.html"}:
|
||||
return False
|
||||
if "dist" in parts and "install-simulation" in parts:
|
||||
return False
|
||||
if "evidence" in parts and "world_class" in parts and "submissions" in parts:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def default_claim_surfaces(skill_dir: Path) -> list[Path]:
|
||||
paths: list[Path] = []
|
||||
for rel in ["README.md", "SKILL.md", "manifest.json"]:
|
||||
path = skill_dir / rel
|
||||
if path.exists():
|
||||
paths.append(path)
|
||||
for directory, patterns in [
|
||||
(skill_dir / "agents", ["*.yaml", "*.yml", "*.json"]),
|
||||
(skill_dir / "docs", ["*.md"]),
|
||||
(skill_dir / "evidence" / "world_class", ["*.md", "*.json"]),
|
||||
(skill_dir / "dist", ["*.json", "*.md"]),
|
||||
(skill_dir / "reports", ["*.md", "*.html", "*.json"]),
|
||||
(skill_dir / "registry", ["*.json"]),
|
||||
(skill_dir / "security", ["*.md", "*.json"]),
|
||||
(skill_dir / "skill-ir", ["*.json"]),
|
||||
]:
|
||||
if not directory.exists():
|
||||
continue
|
||||
for pattern in patterns:
|
||||
paths.extend(
|
||||
path
|
||||
for path in directory.rglob(pattern)
|
||||
if path.is_file() and should_scan_claim_surface(path)
|
||||
)
|
||||
return sorted(set(paths), key=lambda item: rel_path(item, skill_dir))
|
||||
|
||||
|
||||
def line_matches(path: Path, ledger_ready: bool) -> list[dict[str, Any]]:
|
||||
if ledger_ready:
|
||||
return []
|
||||
violations: list[dict[str, Any]] = []
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
return violations
|
||||
for lineno, line in enumerate(lines, start=1):
|
||||
for rule in FORBIDDEN_WHEN_PENDING:
|
||||
if rule["pattern"].search(line):
|
||||
violations.append(
|
||||
{
|
||||
"line": lineno,
|
||||
"rule": rule["key"],
|
||||
"reason": rule["reason"],
|
||||
"excerpt": line.strip()[:240],
|
||||
}
|
||||
)
|
||||
return violations
|
||||
|
||||
|
||||
def escape_markdown_table_cell(value: Any) -> str:
|
||||
return str(value).replace("|", "\\|")
|
||||
|
||||
|
||||
def build_guard(skill_dir: Path, generated_at: str, claim_surfaces: list[Path] | None = None) -> dict[str, Any]:
|
||||
ledger = build_ledger(skill_dir, generated_at)
|
||||
ledger_ready = ledger.get("summary", {}).get("ready_to_claim_world_class") is True
|
||||
surfaces = claim_surfaces or default_claim_surfaces(skill_dir)
|
||||
scanned = []
|
||||
violations = []
|
||||
for path in surfaces:
|
||||
matches = line_matches(path, ledger_ready)
|
||||
rel = rel_path(path, skill_dir)
|
||||
scanned.append({"path": rel, "violation_count": len(matches)})
|
||||
for match in matches:
|
||||
violations.append({"path": rel, **match})
|
||||
ok = len(violations) == 0
|
||||
json_surface_count = sum(1 for item in scanned if item["path"].endswith(".json"))
|
||||
metadata_surface_count = sum(
|
||||
1 for item in scanned if item["path"].endswith((".json", ".yaml", ".yml"))
|
||||
)
|
||||
package_surface_count = sum(
|
||||
1
|
||||
for item in scanned
|
||||
if item["path"] == "manifest.json"
|
||||
or item["path"].startswith(("agents/", "dist/", "security/"))
|
||||
)
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": ok,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": ledger_ready,
|
||||
"ledger_pending_count": ledger.get("summary", {}).get("pending_count", 0),
|
||||
"claim_surface_count": len(scanned),
|
||||
"json_claim_surface_count": json_surface_count,
|
||||
"metadata_claim_surface_count": metadata_surface_count,
|
||||
"package_claim_surface_count": package_surface_count,
|
||||
"violation_count": len(violations),
|
||||
"overclaim_guard_active": True,
|
||||
"decision": (
|
||||
"claim-ready"
|
||||
if ledger_ready and ok
|
||||
else ("claim-blocked-overclaim" if violations else "claim-guard-pass-evidence-pending")
|
||||
),
|
||||
},
|
||||
"rules": [
|
||||
{"key": item["key"], "reason": item["reason"], "pattern": item["pattern"].pattern}
|
||||
for item in FORBIDDEN_WHEN_PENDING
|
||||
],
|
||||
"scanned_surfaces": scanned,
|
||||
"violations": violations,
|
||||
"source_ledger": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
"markdown": "reports/world_class_evidence_ledger.md",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_claim_guard.json",
|
||||
"markdown": "reports/world_class_claim_guard.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# World-Class Claim Guard",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- ledger ready to claim world-class: `{str(summary['ledger_ready_to_claim_world_class']).lower()}`",
|
||||
f"- ledger pending evidence: `{summary['ledger_pending_count']}`",
|
||||
f"- claim surfaces scanned: `{summary['claim_surface_count']}`",
|
||||
f"- JSON claim surfaces scanned: `{summary['json_claim_surface_count']}`",
|
||||
f"- metadata claim surfaces scanned: `{summary['metadata_claim_surface_count']}`",
|
||||
f"- package/runtime claim surfaces scanned: `{summary['package_claim_surface_count']}`",
|
||||
f"- violations: `{summary['violation_count']}`",
|
||||
f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
|
||||
"",
|
||||
"This guard scans public claim surfaces, machine-readable reports, and package/runtime metadata for completion language that would contradict the world-class evidence ledger. It allows evidence planning and pending-state language, but blocks completion claims until the ledger is ready.",
|
||||
"",
|
||||
"## Violations",
|
||||
"",
|
||||
"| Path | Line | Rule | Excerpt |",
|
||||
"| --- | ---: | --- | --- |",
|
||||
]
|
||||
if report["violations"]:
|
||||
for item in report["violations"]:
|
||||
excerpt = str(item["excerpt"]).replace("|", "\\|")
|
||||
lines.append(f"| `{item['path']}` | {item['line']} | `{item['rule']}` | {excerpt} |")
|
||||
else:
|
||||
lines.append("| `none` | 0 | `none` | none |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Rules",
|
||||
"",
|
||||
"| Rule | Reason |",
|
||||
"| --- | --- |",
|
||||
]
|
||||
)
|
||||
for rule in report["rules"]:
|
||||
reason = escape_markdown_table_cell(rule["reason"])
|
||||
lines.append(f"| `{rule['key']}` | {reason} |")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Guard public world-class completion claims against the evidence ledger.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--claim-surface", action="append", default=[])
|
||||
parser.add_argument("--output-json", default="reports/world_class_claim_guard.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_claim_guard.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
claim_surfaces = [resolve_claim_surface(item, skill_dir) for item in args.claim_surface] if args.claim_surface else None
|
||||
report = build_guard(skill_dir, args.generated_at, claim_surfaces)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(0 if report["ok"] else 2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
from world_class_evidence_contract import load_json, rel_path, validate_payload
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Validates world-class human and external evidence intake packets before ledger review."
|
||||
|
||||
def template_paths(skill_dir: Path, keys: list[str]) -> dict[str, Path]:
|
||||
template_dir = skill_dir / "evidence" / "world_class" / "templates"
|
||||
return {key: template_dir / f"{key}.intake.json" for key in keys}
|
||||
|
||||
|
||||
def submission_paths(submissions_dir: Path) -> list[Path]:
|
||||
if not submissions_dir.exists():
|
||||
return []
|
||||
ignored_names = {"submission_manifest.json"}
|
||||
return sorted(path for path in submissions_dir.glob("*.json") if path.is_file() and path.name not in ignored_names)
|
||||
|
||||
|
||||
def find_entry(entries: list[dict[str, Any]], key: str) -> dict[str, Any] | None:
|
||||
for entry in entries:
|
||||
if entry.get("key") == key:
|
||||
return entry
|
||||
return None
|
||||
|
||||
|
||||
def first_by_key(items: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
by_key: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
key = str(item.get("evidence_key", ""))
|
||||
if key and key not in by_key:
|
||||
by_key[key] = item
|
||||
return by_key
|
||||
|
||||
|
||||
def shell_path(path: Path, root: Path) -> str:
|
||||
return shlex.quote(rel_path(path, root))
|
||||
|
||||
|
||||
def checklist_readiness(
|
||||
entry: dict[str, Any],
|
||||
template_result: dict[str, Any] | None,
|
||||
submission_result: dict[str, Any] | None,
|
||||
) -> tuple[str, str]:
|
||||
if entry.get("status") == "accepted":
|
||||
return "accepted", "Ledger already accepts this evidence item."
|
||||
if template_result is None or template_result.get("status") != "pass":
|
||||
return "fix-template", "The intake template is missing or invalid."
|
||||
if submission_result is None:
|
||||
return "awaiting-submission", "No real evidence submission has been provided yet."
|
||||
if submission_result.get("status") == "pass":
|
||||
observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {}
|
||||
if entry.get("source_accepted") is True or observed.get("accepted") is True:
|
||||
return "ready-for-ledger-review", "Submission passes intake validation and is ready for ledger review."
|
||||
return "source-evidence-incomplete", "Submission passes intake validation, but the source evidence checks still do not pass."
|
||||
return "fix-submission", "Submission exists but failed intake validation."
|
||||
|
||||
|
||||
def build_operator_checklist(
|
||||
skill_dir: Path,
|
||||
ledger: dict[str, Any],
|
||||
template_results: list[dict[str, Any]],
|
||||
submission_results: list[dict[str, Any]],
|
||||
submissions_dir: Path,
|
||||
) -> list[dict[str, Any]]:
|
||||
templates_by_key = first_by_key(template_results)
|
||||
submissions_by_key = first_by_key(submission_results)
|
||||
checklist = []
|
||||
for entry in ledger.get("entries", []):
|
||||
key = str(entry.get("key", ""))
|
||||
template_path = skill_dir / "evidence" / "world_class" / "templates" / f"{key}.intake.json"
|
||||
submission_path = submissions_dir / f"{key}.json"
|
||||
template_result = templates_by_key.get(key)
|
||||
submission_result = submissions_by_key.get(key)
|
||||
readiness, blocking_reason = checklist_readiness(entry, template_result, submission_result)
|
||||
submissions_dir_arg = shell_path(submissions_dir, skill_dir)
|
||||
checklist.append(
|
||||
{
|
||||
"evidence_key": key,
|
||||
"label": entry.get("label", key),
|
||||
"category": entry.get("category", "external"),
|
||||
"owner": entry.get("owner", "release reviewer"),
|
||||
"readiness": readiness,
|
||||
"blocking_reason": blocking_reason,
|
||||
"template_status": template_result.get("status", "missing") if template_result else "missing",
|
||||
"submission_status": submission_result.get("status", "missing") if submission_result else "missing",
|
||||
"source_accepted": entry.get("source_accepted") is True,
|
||||
"observed_state": entry.get("observed_state", {}),
|
||||
"template_path": rel_path(template_path, skill_dir),
|
||||
"submission_path": rel_path(submission_path, skill_dir),
|
||||
"commands": {
|
||||
"prepare_submission": (
|
||||
"python3 scripts/yao.py world-class-submission-kit . "
|
||||
f"--evidence-key {shlex.quote(key)} --output-dir {shell_path(submission_path.parent, skill_dir)}"
|
||||
),
|
||||
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {submissions_dir_arg}",
|
||||
"submission_review": (
|
||||
f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {submissions_dir_arg}"
|
||||
),
|
||||
"refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {submissions_dir_arg}",
|
||||
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
},
|
||||
"must_collect": {
|
||||
"provenance_requirements": entry.get("provenance_requirements", []),
|
||||
"runbook": entry.get("runbook", []),
|
||||
"success_checks": entry.get("success_checks", []),
|
||||
"evidence_artifacts": entry.get("evidence_artifacts", []),
|
||||
"privacy_contract": entry.get("privacy_contract", []),
|
||||
},
|
||||
"anti_overclaim": entry.get("anti_overclaim", {}),
|
||||
"next_action": entry.get("next_action", ""),
|
||||
}
|
||||
)
|
||||
return checklist
|
||||
|
||||
|
||||
def build_intake(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
ledger = build_ledger(skill_dir, generated_at)
|
||||
entries = ledger.get("entries", [])
|
||||
keys = [str(entry.get("key", "")) for entry in entries]
|
||||
schema_path = skill_dir / "evidence" / "world_class" / "intake.schema.json"
|
||||
template_results = []
|
||||
for key, path in template_paths(skill_dir, keys).items():
|
||||
entry = find_entry(entries, key) or {"key": key, "category": "external"}
|
||||
payload = load_json(path)
|
||||
if not payload:
|
||||
template_results.append(
|
||||
{
|
||||
"path": rel_path(path, skill_dir),
|
||||
"evidence_key": key,
|
||||
"status": "fail",
|
||||
"template_only": True,
|
||||
"errors": ["template missing or invalid JSON"],
|
||||
}
|
||||
)
|
||||
continue
|
||||
template_results.append(validate_payload(payload, entry, path=path, root=skill_dir, template_expected=True))
|
||||
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
submission_results = []
|
||||
for path in submission_paths(submissions_dir):
|
||||
payload = load_json(path)
|
||||
key = str(payload.get("evidence_key", ""))
|
||||
entry = find_entry(entries, key)
|
||||
if not payload or entry is None:
|
||||
submission_results.append(
|
||||
{
|
||||
"path": rel_path(path, skill_dir),
|
||||
"evidence_key": key or "unknown",
|
||||
"status": "fail",
|
||||
"template_only": False,
|
||||
"errors": ["submission missing, invalid JSON, or unknown evidence_key"],
|
||||
}
|
||||
)
|
||||
continue
|
||||
submission_results.append(validate_payload(payload, entry, path=path, root=skill_dir, template_expected=False))
|
||||
|
||||
template_pass_count = sum(1 for item in template_results if item["status"] == "pass")
|
||||
valid_submission_count = sum(1 for item in submission_results if item["status"] == "pass")
|
||||
invalid_submission_count = sum(1 for item in submission_results if item["status"] == "fail")
|
||||
schema_exists = schema_path.exists()
|
||||
intake_ready = schema_exists and template_pass_count == len(keys) and invalid_submission_count == 0
|
||||
operator_checklist = build_operator_checklist(skill_dir, ledger, template_results, submission_results, submissions_dir)
|
||||
ready_checklist_count = sum(1 for item in operator_checklist if item["readiness"] in {"accepted", "ready-for-ledger-review"})
|
||||
source_incomplete_count = sum(1 for item in operator_checklist if item["readiness"] == "source-evidence-incomplete")
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": intake_ready,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": {
|
||||
"schema_present": schema_exists,
|
||||
"ledger_entry_count": len(keys),
|
||||
"template_count": len(template_results),
|
||||
"template_pass_count": template_pass_count,
|
||||
"submission_count": len(submission_results),
|
||||
"valid_submission_count": valid_submission_count,
|
||||
"invalid_submission_count": invalid_submission_count,
|
||||
"operator_checklist_count": len(operator_checklist),
|
||||
"operator_checklist_ready_count": ready_checklist_count,
|
||||
"valid_packet_source_incomplete_count": source_incomplete_count,
|
||||
"ready_for_external_collection": intake_ready,
|
||||
"ready_for_ledger_review": ready_checklist_count > 0,
|
||||
"ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True,
|
||||
"overclaim_guard_active": True,
|
||||
"decision": (
|
||||
"fix-intake"
|
||||
if not intake_ready
|
||||
else (
|
||||
"intake-ready-for-ledger-review"
|
||||
if ready_checklist_count
|
||||
else ("source-evidence-incomplete" if source_incomplete_count else "awaiting-submissions")
|
||||
)
|
||||
),
|
||||
},
|
||||
"schema": rel_path(schema_path, skill_dir),
|
||||
"templates": template_results,
|
||||
"submissions": submission_results,
|
||||
"operator_checklist": operator_checklist,
|
||||
"source_ledger": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
"markdown": "reports/world_class_evidence_ledger.md",
|
||||
"pending_count": ledger.get("summary", {}).get("pending_count", 0),
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_intake.json",
|
||||
"markdown": "reports/world_class_evidence_intake.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_table(items: list[dict[str, Any]]) -> list[str]:
|
||||
lines = ["| Evidence | Status | Path | Artifacts | Errors |", "| --- | --- | --- | --- | --- |"]
|
||||
if not items:
|
||||
lines.append("| `none` | `n/a` | none | none | none |")
|
||||
return lines
|
||||
for item in items:
|
||||
errors = "; ".join(item.get("errors", [])) or "none"
|
||||
safe_errors = errors.replace("|", "\\|")
|
||||
integrity = item.get("artifact_integrity", {}) if isinstance(item.get("artifact_integrity", {}), dict) else {}
|
||||
artifact_summary = (
|
||||
f"{integrity.get('artifact_existing_count', 0)} existing / "
|
||||
f"{integrity.get('artifact_sha256_verified_count', 0)} sha256 verified / "
|
||||
f"{integrity.get('required_artifact_verified_count', 0)} required verified / "
|
||||
f"{integrity.get('artifact_ref_count', 0)} refs"
|
||||
)
|
||||
lines.append(
|
||||
f"| `{item['evidence_key']}` | `{item['status']}` | `{item['path']}` | {artifact_summary} | {safe_errors} |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def render_operator_checklist(items: list[dict[str, Any]]) -> list[str]:
|
||||
lines = [
|
||||
"| Evidence | Readiness | Submission | Next action |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
if not items:
|
||||
lines.append("| `none` | `accepted` | none | none |")
|
||||
return lines
|
||||
for item in items:
|
||||
action = str(item.get("next_action", "")).replace("|", "\\|")
|
||||
lines.append(
|
||||
f"| `{item['evidence_key']}` | `{item['readiness']}` | `{item['submission_status']}` | {action} |"
|
||||
)
|
||||
for item in items:
|
||||
lines.extend(["", f"### {item['label']}", ""])
|
||||
lines.append(f"- readiness: `{item['readiness']}`")
|
||||
lines.append(f"- blocking reason: {item['blocking_reason']}")
|
||||
lines.append(f"- owner: {item['owner']}")
|
||||
lines.append(f"- template: `{item['template_path']}`")
|
||||
lines.append(f"- submission: `{item['submission_path']}`")
|
||||
lines.extend(["", "#### Commands", ""])
|
||||
commands = item.get("commands", {})
|
||||
for label in ["prepare_submission", "validate_intake", "submission_review", "refresh_ledger", "guard_claim"]:
|
||||
if commands.get(label):
|
||||
lines.append(f"- {label}: `{commands[label]}`")
|
||||
must_collect = item.get("must_collect", {})
|
||||
lines.extend(["", "#### Must Collect", ""])
|
||||
for label in ["provenance_requirements", "success_checks", "evidence_artifacts", "privacy_contract"]:
|
||||
values = must_collect.get(label, [])
|
||||
if values:
|
||||
lines.append(f"- {label}:")
|
||||
lines.extend(f" - {value}" for value in values)
|
||||
runbook = must_collect.get("runbook", [])
|
||||
if runbook:
|
||||
lines.extend(["", "#### Source Runbook", ""])
|
||||
lines.extend(f"- `{value}`" if str(value).startswith("python3 ") or "=" in str(value) else f"- {value}" for value in runbook)
|
||||
return lines
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# World-Class Evidence Intake",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- schema present: `{str(summary['schema_present']).lower()}`",
|
||||
f"- templates: `{summary['template_pass_count']}` / `{summary['template_count']}`",
|
||||
f"- submissions: `{summary['valid_submission_count']}` valid / `{summary['submission_count']}` total",
|
||||
f"- invalid submissions: `{summary['invalid_submission_count']}`",
|
||||
f"- valid packet but source incomplete: `{summary['valid_packet_source_incomplete_count']}`",
|
||||
f"- operator checklist: `{summary['operator_checklist_ready_count']}` ready / `{summary['operator_checklist_count']}` total",
|
||||
f"- ready for external collection: `{str(summary['ready_for_external_collection']).lower()}`",
|
||||
f"- ready for ledger review: `{str(summary['ready_for_ledger_review']).lower()}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
|
||||
"",
|
||||
"This report validates the intake contract for human and external evidence. A valid intake packet means the evidence is ready for ledger review; it does not by itself make a world-class claim true.",
|
||||
"",
|
||||
"## Templates",
|
||||
"",
|
||||
*render_table(report["templates"]),
|
||||
"",
|
||||
"## Submissions",
|
||||
"",
|
||||
*render_table(report["submissions"]),
|
||||
"",
|
||||
"## Operator Checklist",
|
||||
"",
|
||||
*render_operator_checklist(report["operator_checklist"]),
|
||||
"",
|
||||
"## Boundary",
|
||||
"",
|
||||
"- Templates and planned work do not count as accepted evidence.",
|
||||
"- Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests.",
|
||||
"- Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata.",
|
||||
"- Local command-runner output does not count as provider-backed model evidence.",
|
||||
"- Metadata fallback does not count as native permission enforcement.",
|
||||
"- Pending reviewer work does not count as human adjudication.",
|
||||
]
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Validate world-class evidence intake packets.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--submissions-dir")
|
||||
parser.add_argument("--output-json", default="reports/world_class_evidence_intake.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_evidence_intake.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
|
||||
report = build_intake(skill_dir, args.generated_at, submissions_dir)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,389 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from output_review_privacy import forbidden_raw_content_field_paths
|
||||
from render_world_class_evidence_plan import build_plan
|
||||
from world_class_evidence_contract import load_json, load_json_with_status, rel_path, validate_payload
|
||||
from world_class_source_checks import build_source_checklist, summarize_source_checklist
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a machine-checkable ledger for world-class external and human evidence gaps."
|
||||
|
||||
|
||||
def provider_state(skill_dir: Path) -> dict[str, Any]:
|
||||
summary = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {})
|
||||
return {
|
||||
"model_executed_count": summary.get("model_executed_count", 0),
|
||||
"timing_observed_count": summary.get("timing_observed_count", 0),
|
||||
"token_observed_count": summary.get("token_observed_count", 0),
|
||||
"accepted": (
|
||||
summary.get("model_executed_count", 0) > 0
|
||||
and summary.get("timing_observed_count", 0) > 0
|
||||
and summary.get("token_observed_count", 0) > 0
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def human_state(skill_dir: Path) -> dict[str, Any]:
|
||||
adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json")
|
||||
summary = adjudication.get("summary", {})
|
||||
raw_content_paths = forbidden_raw_content_field_paths(adjudication, "output_review_adjudication")
|
||||
return {
|
||||
"pair_count": summary.get("pair_count", 0),
|
||||
"judgment_count": summary.get("judgment_count", 0),
|
||||
"pending_count": summary.get("pending_count", 0),
|
||||
"invalid_decision_count": summary.get("invalid_decision_count", 0),
|
||||
"answer_revealed_count": summary.get("answer_revealed_count", 0),
|
||||
"reviewer_metadata_present": summary.get("reviewer_metadata_present", False),
|
||||
"reason_required": summary.get("reason_required", False),
|
||||
"blind_review_attested": summary.get("blind_review_attested", False),
|
||||
"raw_content_excluded_attested": summary.get("raw_content_excluded_attested", False),
|
||||
"reviewer_reason_required_attested": summary.get("reviewer_reason_required_attested", False),
|
||||
"ready_for_human_evidence": summary.get("ready_for_human_evidence", False),
|
||||
"raw_content_allowed": bool(raw_content_paths),
|
||||
"raw_content_path_count": len(raw_content_paths),
|
||||
"accepted": (
|
||||
summary.get("ready_for_human_evidence") is True
|
||||
and not raw_content_paths
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def permission_state(skill_dir: Path) -> dict[str, Any]:
|
||||
summary = load_json(skill_dir / "reports" / "runtime_permission_probes.json").get("summary", {})
|
||||
return {
|
||||
"native_enforcement_count": summary.get("native_enforcement_count", 0),
|
||||
"metadata_fallback_count": summary.get("metadata_fallback_count", 0),
|
||||
"installer_enforcement_pass_count": summary.get("installer_enforcement_pass_count", 0),
|
||||
"installer_permission_failure_count": summary.get("installer_permission_failure_count", 0),
|
||||
"installer_enforcement_ready": summary.get("installer_enforcement_ready", False),
|
||||
"residual_risk_count": summary.get("residual_risk_count", 0),
|
||||
"failure_count": summary.get("failure_count", 0),
|
||||
"accepted": summary.get("native_enforcement_count", 0) > 0 and summary.get("failure_count", 0) == 0,
|
||||
}
|
||||
|
||||
|
||||
def telemetry_state(skill_dir: Path) -> dict[str, Any]:
|
||||
payload = load_json(skill_dir / "reports" / "adoption_drift_report.json")
|
||||
summary = payload.get("summary", {})
|
||||
source_types = summary.get("source_types", {}) if isinstance(summary.get("source_types", {}), dict) else {}
|
||||
privacy = payload.get("privacy_contract", {})
|
||||
return {
|
||||
"external_source_events": source_types.get("external", 0),
|
||||
"adoption_sample_count": summary.get("adoption_sample_count", 0),
|
||||
"raw_content_allowed": privacy.get("raw_content_allowed"),
|
||||
"risk_band": summary.get("risk_band", "missing"),
|
||||
"accepted": (
|
||||
source_types.get("external", 0) > 0
|
||||
and summary.get("adoption_sample_count", 0) > 0
|
||||
and privacy.get("raw_content_allowed") is False
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
STATE_LOADERS = {
|
||||
"provider-holdout": provider_state,
|
||||
"human-adjudication": human_state,
|
||||
"native-permission-enforcement": permission_state,
|
||||
"native-client-telemetry": telemetry_state,
|
||||
}
|
||||
|
||||
PROVENANCE_REQUIREMENTS = {
|
||||
"provider-holdout": ["provider-backed model run", "observed timing", "observed token metadata"],
|
||||
"human-adjudication": ["real reviewer identity", "blind A/B decisions", "answer key unopened until decisions exist"],
|
||||
"native-permission-enforcement": ["real target client or external installer runtime guard", "native enforcement flag or externally accepted guard proof", "residual risk retained for fallback targets"],
|
||||
"native-client-telemetry": ["real external client source", "metadata-only event", "local-first import path"],
|
||||
}
|
||||
|
||||
TOP_LEVEL_SUMMARY_FIELDS = [
|
||||
"decision",
|
||||
"ready_to_claim_world_class",
|
||||
"ledger_entry_count",
|
||||
"source_accepted_count",
|
||||
"accepted_count",
|
||||
"pending_count",
|
||||
"human_pending_count",
|
||||
"external_pending_count",
|
||||
"submitted_entry_count",
|
||||
"reviewer_approved_submission_count",
|
||||
"missing_submission_count",
|
||||
"invalid_submission_count",
|
||||
"source_check_count",
|
||||
"source_pass_count",
|
||||
"source_blocked_count",
|
||||
"submitted_but_pending_count",
|
||||
"source_accepted_without_valid_submission_count",
|
||||
"overclaim_guard_active",
|
||||
]
|
||||
|
||||
|
||||
def top_level_summary_mirrors(summary: dict[str, Any]) -> dict[str, Any]:
|
||||
return {key: summary[key] for key in TOP_LEVEL_SUMMARY_FIELDS if key in summary}
|
||||
|
||||
|
||||
def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]:
|
||||
key = str(task.get("key", ""))
|
||||
path = submissions_dir / f"{key}.json"
|
||||
payload, load_status = load_json_with_status(path)
|
||||
if load_status != "present":
|
||||
return {
|
||||
"status": load_status,
|
||||
"path": rel_path(path, skill_dir),
|
||||
"artifact_ref_count": 0,
|
||||
"attested_real_evidence": False,
|
||||
"privacy_contract_satisfied": False,
|
||||
"ledger_reviewer_approved": False,
|
||||
"ledger_reviewer": "",
|
||||
"ledger_reviewed_at": "",
|
||||
"ledger_counts_as_completion": False,
|
||||
}
|
||||
validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False)
|
||||
refs = payload.get("artifact_refs", [])
|
||||
attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
|
||||
status = "submitted" if validation["status"] == "pass" else "invalid-contract"
|
||||
artifact_integrity = validation.get("artifact_integrity", {})
|
||||
return {
|
||||
"status": status,
|
||||
"path": rel_path(path, skill_dir),
|
||||
"submitted_by": payload.get("submitted_by", ""),
|
||||
"submitted_at": payload.get("submitted_at", ""),
|
||||
"artifact_ref_count": len(refs) if isinstance(refs, list) else 0,
|
||||
"artifact_existing_count": artifact_integrity.get("artifact_existing_count", 0),
|
||||
"artifact_sha256_verified_count": artifact_integrity.get("artifact_sha256_verified_count", 0),
|
||||
"attested_real_evidence": attestation.get("real_external_or_human_evidence") is True,
|
||||
"reviewer_or_operator_identity_present": attestation.get("reviewer_or_operator_identity_present") is True,
|
||||
"privacy_contract_satisfied": attestation.get("privacy_contract_satisfied") is True,
|
||||
"ledger_reviewer_approved": attestation.get("ledger_reviewer_approved") is True,
|
||||
"ledger_reviewer": str(attestation.get("ledger_reviewer", "")).strip(),
|
||||
"ledger_reviewed_at": str(attestation.get("ledger_reviewed_at", "")).strip(),
|
||||
"errors": validation.get("errors", []),
|
||||
"ledger_counts_as_completion": False,
|
||||
}
|
||||
|
||||
|
||||
def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]:
|
||||
state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir)
|
||||
submission = submission_state(skill_dir, task, submissions_dir)
|
||||
source_checklist = build_source_checklist([{"key": task["key"], "observed_state": state}])
|
||||
source_summary = summarize_source_checklist(source_checklist)
|
||||
source_accepted = bool(source_checklist) and source_summary["source_blocked_count"] == 0
|
||||
accepted = source_accepted and submission.get("status") == "submitted" and submission.get("ledger_reviewer_approved") is True
|
||||
return {
|
||||
"key": task["key"],
|
||||
"label": task["label"],
|
||||
"category": task["category"],
|
||||
"owner": task["owner"],
|
||||
"status": "accepted" if accepted else "pending",
|
||||
"source_status": task["status"],
|
||||
"source_accepted": source_accepted,
|
||||
"current": task["current"],
|
||||
"objective": task["objective"],
|
||||
"runbook": task.get("runbook", []),
|
||||
"provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]),
|
||||
"success_checks": task["success_checks"],
|
||||
"evidence_artifacts": task["evidence_artifacts"],
|
||||
"privacy_contract": task["privacy_contract"],
|
||||
"observed_state": state,
|
||||
"source_checklist": source_checklist,
|
||||
**source_summary,
|
||||
"submission_state": submission,
|
||||
"anti_overclaim": {
|
||||
"planned_work_counts_as_evidence": False,
|
||||
"metadata_fallback_counts_as_native_enforcement": False,
|
||||
"pending_review_counts_as_human_decision": False,
|
||||
"local_command_runner_counts_as_provider_model": False,
|
||||
},
|
||||
"next_action": task["audit_next_action"],
|
||||
}
|
||||
|
||||
|
||||
def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
plan = build_plan(skill_dir, generated_at)
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
evidence_requirements = plan.get("evidence_requirements") or plan.get("tasks", [])
|
||||
entries = [build_entry(skill_dir, task, submissions_dir) for task in evidence_requirements]
|
||||
source_rows = [row for entry in entries for row in entry.get("source_checklist", [])]
|
||||
source_summary = summarize_source_checklist(source_rows)
|
||||
source_accepted_count = sum(1 for entry in entries if entry.get("source_accepted") is True)
|
||||
accepted_count = sum(1 for entry in entries if entry["status"] == "accepted")
|
||||
pending_count = len(entries) - accepted_count
|
||||
human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending")
|
||||
external_pending_count = sum(1 for entry in entries if entry["category"] == "external" and entry["status"] == "pending")
|
||||
submitted_entry_count = sum(1 for entry in entries if entry["submission_state"]["status"] == "submitted")
|
||||
reviewer_approved_submission_count = sum(
|
||||
1 for entry in entries if entry["submission_state"].get("ledger_reviewer_approved") is True
|
||||
)
|
||||
missing_submission_count = sum(1 for entry in entries if entry["submission_state"]["status"] == "missing")
|
||||
invalid_submission_count = sum(1 for entry in entries if entry["submission_state"]["status"] in {"invalid-json", "invalid-contract"})
|
||||
submitted_but_pending_count = sum(
|
||||
1 for entry in entries if entry["submission_state"]["status"] == "submitted" and entry["status"] == "pending"
|
||||
)
|
||||
source_accepted_without_valid_submission_count = sum(
|
||||
1
|
||||
for entry in entries
|
||||
if entry.get("source_accepted") is True and entry["submission_state"]["status"] != "submitted"
|
||||
)
|
||||
ready = bool(entries) and pending_count == 0 and plan["summary"].get("world_class_ready") is True
|
||||
summary = {
|
||||
"ledger_entry_count": len(entries),
|
||||
"source_accepted_count": source_accepted_count,
|
||||
"accepted_count": accepted_count,
|
||||
"pending_count": pending_count,
|
||||
"human_pending_count": human_pending_count,
|
||||
"external_pending_count": external_pending_count,
|
||||
"submitted_entry_count": submitted_entry_count,
|
||||
"reviewer_approved_submission_count": reviewer_approved_submission_count,
|
||||
"missing_submission_count": missing_submission_count,
|
||||
"invalid_submission_count": invalid_submission_count,
|
||||
**source_summary,
|
||||
"submitted_but_pending_count": submitted_but_pending_count,
|
||||
"source_accepted_without_valid_submission_count": source_accepted_without_valid_submission_count,
|
||||
"overclaim_guard_active": True,
|
||||
"ready_to_claim_world_class": ready,
|
||||
"decision": "ready-for-completion-audit" if ready else "evidence-pending",
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
**top_level_summary_mirrors(summary),
|
||||
"summary": summary,
|
||||
"report_contract": {
|
||||
"schema_version": "1.0",
|
||||
"contract": "world-class-evidence-ledger",
|
||||
"top_level_mirrors_summary": True,
|
||||
"summary_fields": TOP_LEVEL_SUMMARY_FIELDS,
|
||||
"source_of_truth": "summary",
|
||||
},
|
||||
"entries": entries,
|
||||
"source_plan": {
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"task_count": plan["summary"].get("task_count", 0),
|
||||
"evidence_requirement_count": len(evidence_requirements),
|
||||
},
|
||||
"submissions": {
|
||||
"directory": rel_path(submissions_dir, skill_dir),
|
||||
"ledger_counts_submission_as_completion": False,
|
||||
"source_pass_requires_valid_submission": True,
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_ledger.json",
|
||||
"markdown": "reports/world_class_evidence_ledger.md",
|
||||
"intake": "reports/world_class_evidence_intake.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(ledger: dict[str, Any]) -> str:
|
||||
summary = ledger["summary"]
|
||||
lines = [
|
||||
"# World-Class Evidence Ledger",
|
||||
"",
|
||||
f"Generated at: `{ledger['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- entries: `{summary['ledger_entry_count']}`",
|
||||
f"- source accepted: `{summary.get('source_accepted_count', 0)}`",
|
||||
f"- source checks: `{summary.get('source_pass_count', 0)}` pass / `{summary.get('source_check_count', 0)}` total",
|
||||
f"- source blocked: `{summary.get('source_blocked_count', 0)}`",
|
||||
f"- accepted: `{summary['accepted_count']}`",
|
||||
f"- pending: `{summary['pending_count']}`",
|
||||
f"- human pending: `{summary['human_pending_count']}`",
|
||||
f"- external pending: `{summary['external_pending_count']}`",
|
||||
f"- submitted entries: `{summary['submitted_entry_count']}`",
|
||||
f"- reviewer approved submissions: `{summary.get('reviewer_approved_submission_count', 0)}`",
|
||||
f"- submitted but pending: `{summary['submitted_but_pending_count']}`",
|
||||
f"- source accepted without valid submission: `{summary.get('source_accepted_without_valid_submission_count', 0)}`",
|
||||
f"- invalid submissions: `{summary['invalid_submission_count']}`",
|
||||
f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
|
||||
"",
|
||||
"This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.",
|
||||
"",
|
||||
"## Ledger",
|
||||
"",
|
||||
"| Evidence | Status | Submission | Category | Current | Next action |",
|
||||
"| --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for entry in ledger["entries"]:
|
||||
current = str(entry["current"]).replace("|", "\\|")
|
||||
action = str(entry["next_action"]).replace("|", "\\|")
|
||||
submission = entry.get("submission_state", {}).get("status", "missing")
|
||||
lines.append(f"| `{entry['key']}` | `{entry['status']}` | `{submission}` | `{entry['category']}` | {current} | {action} |")
|
||||
if not ledger["entries"]:
|
||||
lines.append("| `none` | `accepted` | `none` | all evidence collected | none |")
|
||||
for entry in ledger["entries"]:
|
||||
lines.extend(["", f"## {entry['label']}", ""])
|
||||
lines.append(f"- objective: {entry['objective']}")
|
||||
lines.append(f"- source status: `{entry['source_status']}`")
|
||||
lines.append(f"- observed state: `{json.dumps(entry['observed_state'], ensure_ascii=False)}`")
|
||||
lines.append(
|
||||
f"- source checks: `{entry.get('source_pass_count', 0)}` pass / "
|
||||
f"`{entry.get('source_check_count', 0)}` total"
|
||||
)
|
||||
lines.append(f"- submission state: `{json.dumps(entry.get('submission_state', {}), ensure_ascii=False)}`")
|
||||
lines.extend(["", "### Provenance Requirements", ""])
|
||||
lines.extend(f"- {item}" for item in entry["provenance_requirements"])
|
||||
lines.extend(["", "### Source Runbook", ""])
|
||||
lines.extend(f"- `{item}`" if str(item).startswith("python3 ") or "=" in str(item) else f"- {item}" for item in entry.get("runbook", []))
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"### Source Evidence Checks",
|
||||
"",
|
||||
"| Check | Current | Expected | Status |",
|
||||
"| --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
source_checks = entry.get("source_checklist", [])
|
||||
if source_checks:
|
||||
for row in source_checks:
|
||||
lines.append(
|
||||
f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` |"
|
||||
)
|
||||
else:
|
||||
lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` |")
|
||||
lines.extend(["", "### Completion Assertions", ""])
|
||||
lines.extend(f"- {item}" for item in entry["success_checks"])
|
||||
lines.extend(["", "### Privacy Contract", ""])
|
||||
lines.extend(f"- {item}" for item in entry["privacy_contract"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a world-class evidence ledger.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json", default="reports/world_class_evidence_ledger.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_evidence_ledger.md")
|
||||
parser.add_argument("--submissions-dir")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
|
||||
ledger = build_ledger(skill_dir, args.generated_at, submissions_dir=submissions_dir)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(ledger), encoding="utf-8")
|
||||
print(json.dumps(ledger, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,327 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_skill_os2_audit import build_audit
|
||||
from world_class_evidence_contract import load_json_with_status, validate_payload
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
TASK_TEMPLATES: dict[str, dict[str, Any]] = {
|
||||
"provider-holdout": {
|
||||
"category": "external",
|
||||
"owner": "operator with provider credentials",
|
||||
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
|
||||
"runbook": [
|
||||
"Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.",
|
||||
"For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60",
|
||||
"For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/output_execution_runs.json summary.model_executed_count > 0",
|
||||
"reports/output_execution_runs.json summary.timing_observed_count > 0",
|
||||
"reports/output_execution_runs.json summary.token_observed_count > 0",
|
||||
"reports/skill_os2_audit.json item provider-holdout status becomes pass",
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"reports/output_execution_runs.json",
|
||||
"reports/output_execution_runs.md",
|
||||
"reports/skill_os2_audit.json",
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Do not commit provider credentials or environment dumps.",
|
||||
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts.",
|
||||
],
|
||||
},
|
||||
"human-adjudication": {
|
||||
"category": "human",
|
||||
"owner": "human reviewer",
|
||||
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
|
||||
"runbook": [
|
||||
"python3 scripts/yao.py output-review-kit --write-template",
|
||||
"Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.",
|
||||
"python3 scripts/adjudicate_output_review.py --write-template",
|
||||
"Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.",
|
||||
"python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication",
|
||||
"python3 scripts/yao.py output-review",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/output_review_adjudication.json summary.pending_count == 0",
|
||||
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
|
||||
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
|
||||
"reports/output_review_adjudication.json summary.reviewer_metadata_present is true",
|
||||
"reports/output_review_adjudication.json summary.blind_review_attested is true",
|
||||
"reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json",
|
||||
"reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text",
|
||||
"reports/output_review_adjudication.json summary.ready_for_human_evidence is true",
|
||||
"reports/skill_os2_audit.json item human-adjudication status becomes pass",
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"reports/output_blind_review_pack.md",
|
||||
"reports/output_review_kit.md",
|
||||
"reports/output_review_decisions.json",
|
||||
"reports/output_review_adjudication.json",
|
||||
"reports/output_review_adjudication.md",
|
||||
"scripts/import_output_review_decisions.py",
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Reviewer decisions should not include raw user data or private customer detail.",
|
||||
"Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.",
|
||||
"The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.",
|
||||
"The adjudication evidence stores prompt_sha256 instead of raw prompt text.",
|
||||
"The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.",
|
||||
"Keep the answer key separate until after decisions are recorded.",
|
||||
],
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"category": "external",
|
||||
"owner": "target client or installer integrator",
|
||||
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
|
||||
"runbook": [
|
||||
"Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.",
|
||||
"Update the generated target adapter only when the guard is actually enforced by that target.",
|
||||
"python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip",
|
||||
"python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation",
|
||||
"python3 scripts/yao.py runtime-permissions . --package-dir dist",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
|
||||
"reports/runtime_permission_probes.json summary.failure_count == 0",
|
||||
"reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence",
|
||||
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass",
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"dist/targets/*/adapter.json",
|
||||
"reports/runtime_permission_probes.json",
|
||||
"reports/runtime_permission_probes.md",
|
||||
"reports/install_simulation.json",
|
||||
"reports/install_simulation.md",
|
||||
"security/permission_policy.json",
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Do not mark native_enforcement true for metadata-only fallbacks.",
|
||||
"Keep residual risks visible for targets that still rely on operator enforcement.",
|
||||
],
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
"Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.",
|
||||
"python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl",
|
||||
"python3 scripts/yao.py skill-atlas --workspace-root .",
|
||||
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
|
||||
],
|
||||
"success_checks": [
|
||||
"reports/adoption_drift_report.json summary.source_types.external > 0",
|
||||
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
|
||||
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass",
|
||||
],
|
||||
"evidence_artifacts": [
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/adoption_drift_report.md",
|
||||
"reports/telemetry_hook_recipes.json",
|
||||
"scripts/telemetry_native_host.py",
|
||||
],
|
||||
"privacy_contract": [
|
||||
"Telemetry must remain metadata-only and local-first.",
|
||||
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def build_task(item: dict[str, Any]) -> dict[str, Any]:
|
||||
template = TASK_TEMPLATES.get(
|
||||
item["key"],
|
||||
{
|
||||
"category": "review",
|
||||
"owner": "release reviewer",
|
||||
"objective": item.get("target", "Collect stronger evidence for this non-pass audit item."),
|
||||
"runbook": ["Open reports/skill_os2_audit.md and resolve the listed next action."],
|
||||
"success_checks": [f"reports/skill_os2_audit.json item {item['key']} status becomes pass"],
|
||||
"evidence_artifacts": [entry["path"] for entry in item.get("evidence", []) if entry.get("exists")],
|
||||
"privacy_contract": ["Do not add raw private user content to release evidence."],
|
||||
},
|
||||
)
|
||||
intake_runbook = [
|
||||
f"Copy evidence/world_class/templates/{item['key']}.intake.json to evidence/world_class/submissions/{item['key']}.json and fill only real evidence fields.",
|
||||
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
|
||||
]
|
||||
intake_artifacts = [
|
||||
"evidence/world_class/intake.schema.json",
|
||||
f"evidence/world_class/templates/{item['key']}.intake.json",
|
||||
"reports/world_class_evidence_intake.json",
|
||||
"reports/world_class_evidence_intake.md",
|
||||
]
|
||||
return {
|
||||
"key": item["key"],
|
||||
"label": item["label"],
|
||||
"status": item["status"],
|
||||
"category": template["category"],
|
||||
"owner": template["owner"],
|
||||
"current": item["current"],
|
||||
"objective": template["objective"],
|
||||
"runbook": [*template["runbook"], *intake_runbook],
|
||||
"success_checks": template["success_checks"],
|
||||
"evidence_artifacts": [*template["evidence_artifacts"], *intake_artifacts],
|
||||
"privacy_contract": template["privacy_contract"],
|
||||
"audit_next_action": item["next_action"],
|
||||
}
|
||||
|
||||
|
||||
def has_accepted_ledger_submission(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> bool:
|
||||
path = submissions_dir / f"{task['key']}.json"
|
||||
payload, load_status = load_json_with_status(path)
|
||||
if load_status != "present":
|
||||
return False
|
||||
validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False)
|
||||
return validation.get("status") == "pass"
|
||||
|
||||
|
||||
def build_plan(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
audit = build_audit(skill_dir, generated_at)
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
evidence_keys = set(TASK_TEMPLATES)
|
||||
evidence_requirements = [build_task(item) for item in audit["items"] if item["key"] in evidence_keys]
|
||||
tasks = [
|
||||
task
|
||||
for task in evidence_requirements
|
||||
if task["status"] != "pass" or not has_accepted_ledger_submission(skill_dir, task, submissions_dir)
|
||||
]
|
||||
category_counts: dict[str, int] = {}
|
||||
for task in tasks:
|
||||
category_counts[task["category"]] = category_counts.get(task["category"], 0) + 1
|
||||
audit_world_class_ready = audit["summary"].get("world_class_ready") is True
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": audit["ok"],
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": {
|
||||
"audit_decision": audit["summary"]["decision"],
|
||||
"world_class_ready": bool(audit["summary"]["world_class_ready"]),
|
||||
"audit_world_class_ready": audit_world_class_ready,
|
||||
"ready_to_claim_world_class": False,
|
||||
"ledger_completion_required": True,
|
||||
"evidence_requirement_count": len(evidence_requirements),
|
||||
"task_count": len(tasks),
|
||||
"human_task_count": category_counts.get("human", 0),
|
||||
"external_task_count": category_counts.get("external", 0),
|
||||
"review_task_count": category_counts.get("review", 0),
|
||||
"decision": "audit-ready-ledger-required" if audit_world_class_ready and not tasks else "collect-external-evidence",
|
||||
},
|
||||
"tasks": tasks,
|
||||
"evidence_requirements": evidence_requirements,
|
||||
"source_audit": {
|
||||
"json": "reports/skill_os2_audit.json",
|
||||
"markdown": "reports/skill_os2_audit.md",
|
||||
"open_gap_count": audit["summary"]["open_gap_count"],
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_plan.json",
|
||||
"markdown": "reports/world_class_evidence_plan.md",
|
||||
"ledger": "reports/world_class_evidence_ledger.md",
|
||||
"intake": "reports/world_class_evidence_intake.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(plan: dict[str, Any]) -> str:
|
||||
summary = plan["summary"]
|
||||
lines = [
|
||||
"# World-Class Evidence Plan",
|
||||
"",
|
||||
f"Generated at: `{plan['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- audit decision: `{summary['audit_decision']}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- ledger completion required: `{str(summary.get('ledger_completion_required', True)).lower()}`",
|
||||
f"- evidence requirements: `{summary.get('evidence_requirement_count', 0)}`",
|
||||
f"- tasks: `{summary['task_count']}`",
|
||||
f"- human tasks: `{summary['human_task_count']}`",
|
||||
f"- external tasks: `{summary['external_task_count']}`",
|
||||
"",
|
||||
"This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions.",
|
||||
"",
|
||||
"## Task Table",
|
||||
"",
|
||||
"| Task | Status | Category | Owner | Current |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for task in plan["tasks"]:
|
||||
current = str(task["current"]).replace("|", "\\|")
|
||||
lines.append(
|
||||
f"| `{task['key']}` | `{task['status']}` | `{task['category']}` | {task['owner']} | {current} |"
|
||||
)
|
||||
if not plan["tasks"]:
|
||||
lines.append("| `none` | `pass` | `none` | none | audit gaps closed; ledger validation still required |")
|
||||
for task in plan["tasks"]:
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
f"## {task['label']}",
|
||||
"",
|
||||
f"- objective: {task['objective']}",
|
||||
f"- audit next action: {task['audit_next_action']}",
|
||||
"",
|
||||
"### Runbook",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for command in task["runbook"]:
|
||||
lines.append(f"- `{command}`" if command.startswith("python3 ") or "=" in command else f"- {command}")
|
||||
lines.extend(["", "### Success Checks", ""])
|
||||
lines.extend(f"- {check}" for check in task["success_checks"])
|
||||
lines.extend(["", "### Evidence Artifacts", ""])
|
||||
lines.extend(f"- `{artifact}`" for artifact in task["evidence_artifacts"])
|
||||
lines.extend(["", "### Privacy Contract", ""])
|
||||
lines.extend(f"- {item}" for item in task["privacy_contract"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a world-class evidence collection plan.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-json", default="reports/world_class_evidence_plan.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_evidence_plan.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
plan = build_plan(skill_dir, args.generated_at)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(plan, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(plan), encoding="utf-8")
|
||||
print(json.dumps(plan, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,651 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from html_rendering import html_text
|
||||
from render_world_class_evidence_intake import build_intake
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
from render_world_class_preflight import build_preflight
|
||||
from render_world_class_submission_review import build_submission_review
|
||||
from world_class_operator_runbook_coordination import build_coordination_plan, build_release_gate
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders an operator runbook for collecting pending world-class evidence without accepting evidence."
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def by_key(items: list[dict[str, Any]], key_name: str) -> dict[str, dict[str, Any]]:
|
||||
return {str(item.get(key_name, "")): item for item in items if item.get(key_name)}
|
||||
|
||||
|
||||
def output_path(path: str, skill_dir: Path) -> str:
|
||||
candidate = Path(path)
|
||||
if candidate.is_absolute():
|
||||
return rel_path(candidate, skill_dir)
|
||||
return path
|
||||
|
||||
|
||||
def build_runbook_item(
|
||||
entry: dict[str, Any],
|
||||
checklist: dict[str, Any],
|
||||
review_item: dict[str, Any],
|
||||
preflight_item: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
accepted = entry.get("status") == "accepted" or review_item.get("review_state") == "accepted"
|
||||
commands = checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {}
|
||||
must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {}
|
||||
source_checklist = review_item.get("source_checklist", [])
|
||||
blocked_source_checks = [
|
||||
row for row in source_checklist if isinstance(row, dict) and row.get("status") != "pass"
|
||||
]
|
||||
next_source_actions = []
|
||||
for row in blocked_source_checks:
|
||||
action = str(row.get("next_action", "")).strip()
|
||||
if action and action not in next_source_actions:
|
||||
next_source_actions.append(action)
|
||||
repair_checklist = (
|
||||
preflight_item.get("repair_checklist", []) if isinstance(preflight_item.get("repair_checklist", []), list) else []
|
||||
)
|
||||
phase_queue = preflight_item.get("phase_queue", []) if isinstance(preflight_item.get("phase_queue", []), list) else []
|
||||
if accepted:
|
||||
repair_checklist = []
|
||||
phase_queue = []
|
||||
return {
|
||||
"evidence_key": entry.get("key", ""),
|
||||
"label": entry.get("label", entry.get("key", "")),
|
||||
"category": entry.get("category", "external"),
|
||||
"owner": entry.get("owner", "release reviewer"),
|
||||
"ledger_status": entry.get("status", "pending"),
|
||||
"intake_readiness": checklist.get("readiness", "missing"),
|
||||
"review_state": review_item.get("review_state", "missing"),
|
||||
"source_accepted": review_item.get("source_accepted") is True,
|
||||
"objective": entry.get("objective", ""),
|
||||
"current": entry.get("current", ""),
|
||||
"execution_runbook": entry.get("runbook", []),
|
||||
"blocking_reason": review_item.get("blocking_reason") or checklist.get("blocking_reason") or entry.get("next_action", ""),
|
||||
"submission_path": checklist.get("submission_path") or entry.get("submission_state", {}).get("path", ""),
|
||||
"template_path": checklist.get("template_path", ""),
|
||||
"commands": {
|
||||
"prepare_submission": commands.get("prepare_submission", ""),
|
||||
"validate_intake": commands.get("validate_intake", ""),
|
||||
"review_queue": commands.get("submission_review", ""),
|
||||
"refresh_ledger": commands.get("refresh_ledger", "python3 scripts/yao.py world-class-ledger ."),
|
||||
"guard_claim": commands.get("guard_claim", "python3 scripts/yao.py world-class-claim-guard ."),
|
||||
},
|
||||
"must_collect": {
|
||||
"provenance_requirements": must_collect.get("provenance_requirements", entry.get("provenance_requirements", [])),
|
||||
"success_checks": must_collect.get("success_checks", entry.get("success_checks", [])),
|
||||
"privacy_contract": must_collect.get("privacy_contract", entry.get("privacy_contract", [])),
|
||||
},
|
||||
"evidence_artifacts": entry.get("evidence_artifacts", []),
|
||||
"observed_state": entry.get("observed_state", {}),
|
||||
"source_checklist": source_checklist,
|
||||
"blocked_source_check_count": len(blocked_source_checks),
|
||||
"next_source_actions": next_source_actions,
|
||||
"repair_checklist": repair_checklist,
|
||||
"repair_blocked_count": sum(
|
||||
1
|
||||
for row in repair_checklist
|
||||
if isinstance(row, dict) and row.get("status") != "ready"
|
||||
),
|
||||
"repair_counts_as_completion": False,
|
||||
"phase_queue": phase_queue,
|
||||
"phase_queue_blocked_count": sum(
|
||||
1
|
||||
for row in phase_queue
|
||||
if isinstance(row, dict) and row.get("status") != "ready"
|
||||
),
|
||||
"phase_queue_counts_as_completion": False,
|
||||
"submission_state": entry.get("submission_state", {}),
|
||||
"anti_overclaim": entry.get("anti_overclaim", {}),
|
||||
}
|
||||
|
||||
|
||||
def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
preflight = build_preflight(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
checklist_by_key = by_key(intake.get("operator_checklist", []), "evidence_key")
|
||||
review_by_key = by_key(review.get("items", []), "evidence_key")
|
||||
preflight_by_key = by_key(preflight.get("items", []), "evidence_key")
|
||||
items = [
|
||||
build_runbook_item(
|
||||
entry,
|
||||
checklist_by_key.get(str(entry.get("key", "")), {}),
|
||||
review_by_key.get(str(entry.get("key", "")), {}),
|
||||
preflight_by_key.get(str(entry.get("key", "")), {}),
|
||||
)
|
||||
for entry in ledger.get("entries", [])
|
||||
]
|
||||
coordination_plan = build_coordination_plan(items)
|
||||
release_gate = build_release_gate(skill_dir, ledger)
|
||||
summary = ledger.get("summary", {})
|
||||
review_summary = review.get("summary", {})
|
||||
preflight_summary = preflight.get("summary", {}) if isinstance(preflight.get("summary", {}), dict) else {}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": {
|
||||
"evidence_item_count": len(items),
|
||||
"pending_count": summary.get("pending_count", 0),
|
||||
"accepted_count": summary.get("accepted_count", 0),
|
||||
"awaiting_submission_count": review_summary.get("awaiting_submission_count", 0),
|
||||
"ready_for_ledger_review_count": review_summary.get("ready_for_ledger_review_count", 0),
|
||||
"valid_packet_source_incomplete_count": review_summary.get("valid_packet_source_incomplete_count", 0),
|
||||
"invalid_submission_count": review_summary.get("invalid_submission_count", 0),
|
||||
"source_check_count": review_summary.get("source_check_count", 0),
|
||||
"source_pass_count": review_summary.get("source_pass_count", 0),
|
||||
"source_blocked_count": review_summary.get("source_blocked_count", 0),
|
||||
"repair_checklist_count": preflight_summary.get("repair_checklist_count", 0),
|
||||
"repair_blocked_count": preflight_summary.get("repair_blocked_count", 0),
|
||||
"phase_queue_count": preflight_summary.get("phase_queue_count", 0),
|
||||
"phase_queue_blocked_count": preflight_summary.get("phase_queue_blocked_count", 0),
|
||||
"phase_queue_row_count": preflight_summary.get("phase_queue_row_count", 0),
|
||||
"phase_queue_next_phase": preflight_summary.get("phase_queue_next_phase", ""),
|
||||
"phase_queue_next_action_id": preflight_summary.get("phase_queue_next_action_id", ""),
|
||||
"phase_queue_next_command": preflight_summary.get("phase_queue_next_command", ""),
|
||||
"phase_queue_counts_as_completion": False,
|
||||
"coordination_step_count": len(coordination_plan),
|
||||
"coordination_user_required_step_count": sum(1 for step in coordination_plan if step.get("requires_user_input")),
|
||||
"coordination_pending_evidence_keys": sorted(
|
||||
key
|
||||
for key in {str(step.get("evidence_key", "")) for step in coordination_plan}
|
||||
if key
|
||||
),
|
||||
"coordination_counts_as_completion": False,
|
||||
"release_gate_ready": release_gate["ready"],
|
||||
"release_gate_blocked_count": release_gate["blocked_count"],
|
||||
"release_gate_check_count": release_gate["check_count"],
|
||||
"release_gate_counts_as_completion": False,
|
||||
"ready_to_claim_world_class": summary.get("ready_to_claim_world_class") is True,
|
||||
"runbook_counts_as_completion": False,
|
||||
"decision": "ready-for-completion-audit" if summary.get("ready_to_claim_world_class") is True else "collect-evidence",
|
||||
},
|
||||
"submissions": {
|
||||
"directory": rel_path(submissions_dir, skill_dir),
|
||||
"runbook_counts_submission_as_completion": False,
|
||||
},
|
||||
"items": items,
|
||||
"coordination_plan": coordination_plan,
|
||||
"release_gate": release_gate,
|
||||
"repair_checklist": preflight.get("repair_checklist", [])
|
||||
if isinstance(preflight.get("repair_checklist", []), list)
|
||||
else [],
|
||||
"phase_queue": preflight.get("phase_queue", []) if isinstance(preflight.get("phase_queue", []), list) else [],
|
||||
"source_reports": {
|
||||
"ledger": "reports/world_class_evidence_ledger.json",
|
||||
"intake": "reports/world_class_evidence_intake.json",
|
||||
"preflight": "reports/world_class_evidence_preflight.json",
|
||||
"submission_review": "reports/world_class_submission_review.json",
|
||||
"claim_guard": "reports/world_class_claim_guard.json",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_operator_runbook.json",
|
||||
"markdown": "reports/world_class_operator_runbook.md",
|
||||
"html": "reports/world_class_operator_runbook.html",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def list_lines(values: list[Any], empty: str) -> list[str]:
|
||||
if not values:
|
||||
return [f"- {empty}"]
|
||||
return [f"- {value}" for value in values]
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# World-Class Operator Runbook",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- runbook counts as completion: `{str(summary['runbook_counts_as_completion']).lower()}`",
|
||||
f"- evidence items: `{summary['evidence_item_count']}`",
|
||||
f"- pending: `{summary['pending_count']}`",
|
||||
f"- awaiting submission: `{summary['awaiting_submission_count']}`",
|
||||
f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`",
|
||||
f"- phase queue: `{summary['phase_queue_blocked_count']}` blocked / `{summary['phase_queue_count']}` phases",
|
||||
f"- phase queue rows: `{summary['phase_queue_row_count']}`",
|
||||
f"- phase queue counts as completion: `{str(summary['phase_queue_counts_as_completion']).lower()}`",
|
||||
f"- coordination steps: `{summary['coordination_user_required_step_count']}` user-required / `{summary['coordination_step_count']}` total",
|
||||
f"- coordination pending keys: `{', '.join(summary['coordination_pending_evidence_keys'])}`",
|
||||
f"- coordination counts as completion: `{str(summary['coordination_counts_as_completion']).lower()}`",
|
||||
f"- release gate ready: `{str(summary['release_gate_ready']).lower()}`",
|
||||
f"- release gate blocked checks: `{summary['release_gate_blocked_count']}` / `{summary['release_gate_check_count']}`",
|
||||
f"- release gate counts as completion: `{str(summary['release_gate_counts_as_completion']).lower()}`",
|
||||
"",
|
||||
"This runbook coordinates evidence collection only. It does not accept submissions or make world-class completion true.",
|
||||
"",
|
||||
"## Fast Path",
|
||||
"",
|
||||
"1. Run the real external or human work for one evidence item.",
|
||||
"2. Generate the matching submission draft.",
|
||||
"3. Replace template-only fields with aggregate evidence and provenance.",
|
||||
"4. Validate intake, review the queue, refresh the ledger, then run the claim guard.",
|
||||
"",
|
||||
]
|
||||
lines.extend(
|
||||
[
|
||||
"## Coordination Plan",
|
||||
"",
|
||||
"| Step | Evidence | Owner | Needs user | User action | Assistant action | Command | Pass condition |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for step in report.get("coordination_plan", []):
|
||||
lines.append(
|
||||
f"| `{step.get('step_id', '')}` | `{step.get('evidence_key', '') or 'all'}` | {step.get('owner', '')} | "
|
||||
f"`{str(step.get('requires_user_input') is True).lower()}` | {step.get('user_action', '')} | "
|
||||
f"{step.get('assistant_action', '')} | `{step.get('command', '')}` | {step.get('pass_condition', '')} |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.extend(
|
||||
[
|
||||
"## Phase Queue",
|
||||
"",
|
||||
"| Phase | Status | Rows | Blocked | Owners | Next action | Verify |",
|
||||
"| --- | --- | ---: | ---: | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for row in report.get("phase_queue", []):
|
||||
owners = ", ".join(str(owner) for owner in row.get("owners", []))
|
||||
lines.append(
|
||||
f"| `{row.get('phase', '')}` | `{row.get('status', '')}` | `{row.get('row_count', 0)}` | "
|
||||
f"`{row.get('blocked_count', 0)}` | {owners} | {row.get('next_action', '')} | `{row.get('verification_command', '')}` |"
|
||||
)
|
||||
lines.append("")
|
||||
lines.extend(
|
||||
[
|
||||
"## Evidence Items",
|
||||
"",
|
||||
"| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |",
|
||||
"| --- | --- | --- | --- | ---: | --- | --- |",
|
||||
]
|
||||
)
|
||||
for item in report["items"]:
|
||||
next_action = item.get("next_source_actions", ["none"])[0] if item.get("next_source_actions") else "none"
|
||||
lines.append(
|
||||
f"| `{item['evidence_key']}` | `{item['ledger_status']}` | `{item['intake_readiness']}` | "
|
||||
f"`{item['review_state']}` | `{item.get('blocked_source_check_count', 0)}` | {next_action} | {item['owner']} |"
|
||||
)
|
||||
for item in report["items"]:
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
f"## {item['label']}",
|
||||
"",
|
||||
f"- objective: {item['objective']}",
|
||||
f"- blocking reason: {item['blocking_reason']}",
|
||||
f"- blocked source checks: `{item.get('blocked_source_check_count', 0)}`",
|
||||
f"- repair rows: `{item.get('repair_blocked_count', 0)}` blocked",
|
||||
f"- phase queue: `{item.get('phase_queue_blocked_count', 0)}` blocked phases",
|
||||
f"- submission: `{item['submission_path'] or 'missing'}`",
|
||||
f"- template: `{item['template_path'] or 'missing'}`",
|
||||
"",
|
||||
"### Phase Queue",
|
||||
"",
|
||||
"| Phase | Status | Rows | Blocked | Next action |",
|
||||
"| --- | --- | ---: | ---: | --- |",
|
||||
]
|
||||
)
|
||||
item_phase_queue = item.get("phase_queue", [])
|
||||
if item_phase_queue:
|
||||
for row in item_phase_queue:
|
||||
lines.append(
|
||||
f"| `{row.get('phase', '')}` | `{row.get('status', '')}` | `{row.get('row_count', 0)}` | "
|
||||
f"`{row.get('blocked_count', 0)}` | {row.get('next_action', '')} |"
|
||||
)
|
||||
else:
|
||||
lines.append("| No phase queue listed. | `n/a` | `0` | `0` | n/a |")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"### Source Runbook",
|
||||
"",
|
||||
*list_lines(item.get("execution_runbook", []), "No source runbook listed."),
|
||||
"",
|
||||
"### Commands",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for label, command in item["commands"].items():
|
||||
lines.append(f"- {label}: `{command}`")
|
||||
lines.extend(["", "### Must Collect", ""])
|
||||
lines.extend(list_lines(item["must_collect"].get("provenance_requirements", []), "No provenance requirements listed."))
|
||||
lines.extend(["", "### Success Checks", ""])
|
||||
lines.extend(list_lines(item["must_collect"].get("success_checks", []), "No success checks listed."))
|
||||
lines.extend(["", "### Privacy Contract", ""])
|
||||
lines.extend(list_lines(item["must_collect"].get("privacy_contract", []), "No privacy contract listed."))
|
||||
lines.extend(["", "### Evidence Artifacts", ""])
|
||||
lines.extend(list_lines(item.get("evidence_artifacts", []), "No evidence artifacts listed."))
|
||||
lines.extend(["", "### Next Source Actions", ""])
|
||||
lines.extend(list_lines(item.get("next_source_actions", []), "No blocked source checks."))
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"### Source Evidence Snapshot",
|
||||
"",
|
||||
"| Check | Current | Expected | Status | Next action |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
source_rows = item.get("source_checklist", [])
|
||||
if source_rows:
|
||||
for row in source_rows:
|
||||
lines.append(
|
||||
f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` | {row.get('next_action', '')} |"
|
||||
)
|
||||
else:
|
||||
lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` | n/a |")
|
||||
release_gate = report.get("release_gate", {})
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Release Gate",
|
||||
"",
|
||||
f"- decision: `{release_gate.get('decision', 'missing')}`",
|
||||
f"- ready: `{str(release_gate.get('ready') is True).lower()}`",
|
||||
f"- blocked checks: `{release_gate.get('blocked_count', 0)}` / `{release_gate.get('check_count', 0)}`",
|
||||
f"- counts as completion: `{str(release_gate.get('counts_as_completion') is True).lower()}`",
|
||||
f"- final manual check: {release_gate.get('final_manual_check', '')}",
|
||||
"",
|
||||
"| Check | Current | Expected | Status | Artifact |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
)
|
||||
for check in release_gate.get("checks", []):
|
||||
status = "pass" if check.get("passed") is True else "blocked"
|
||||
lines.append(
|
||||
f"| {check.get('label', '')} | `{check.get('current', '')}` | `{check.get('expected', '')}` | "
|
||||
f"`{status}` | `{check.get('artifact', '')}` |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Boundary",
|
||||
"",
|
||||
"- Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.",
|
||||
"- Valid intake means ready for submission review; ledger review still requires passing source evidence.",
|
||||
"- The world-class ledger and claim guard remain the source of truth.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def html_list(values: list[Any], empty: str) -> str:
|
||||
if not values:
|
||||
return f"<li>{html_text(empty)}</li>"
|
||||
return "".join(f"<li>{html_text(value)}</li>" for value in values)
|
||||
|
||||
|
||||
def html_source_checks(rows: list[dict[str, Any]]) -> str:
|
||||
if not rows:
|
||||
return "<p class='muted'>No source checks listed.</p>"
|
||||
items = []
|
||||
for row in rows:
|
||||
items.append(
|
||||
"<li class='source-check "
|
||||
+ html_text(row.get("status", ""))
|
||||
+ "'>"
|
||||
f"<span>{html_text(row.get('label', ''))}</span>"
|
||||
f"<code>{html_text(row.get('field', ''))}: {html_text(row.get('actual', ''))} / {html_text(row.get('expected', ''))}</code>"
|
||||
f"<small>{html_text(row.get('next_action', ''))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='source-checks'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def html_phase_queue(rows: list[dict[str, Any]]) -> str:
|
||||
if not rows:
|
||||
return "<p class='muted'>No phase queue listed.</p>"
|
||||
items = []
|
||||
for row in rows:
|
||||
items.append(
|
||||
"<li class='source-check'>"
|
||||
f"<span>{html_text(row.get('label', row.get('phase', '')))}</span>"
|
||||
f"<code>{html_text(row.get('phase', ''))}: {html_text(row.get('blocked_count', 0))} / {html_text(row.get('row_count', 0))}</code>"
|
||||
f"<small>{html_text(row.get('next_action', ''))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='source-checks'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def render_html_coordination_plan(steps: list[dict[str, Any]]) -> str:
|
||||
if not steps:
|
||||
return "<p class='muted'>No coordination plan listed.</p>"
|
||||
rows = []
|
||||
for step in steps:
|
||||
rows.append(
|
||||
"<tr>"
|
||||
f"<td><code>{html_text(step.get('step_id', ''))}</code></td>"
|
||||
f"<td><code>{html_text(step.get('evidence_key', '') or 'all')}</code></td>"
|
||||
f"<td>{html_text(step.get('owner', ''))}</td>"
|
||||
f"<td><code>{html_text(str(step.get('requires_user_input') is True).lower())}</code></td>"
|
||||
f"<td>{html_text(step.get('user_action', ''))}</td>"
|
||||
f"<td>{html_text(step.get('assistant_action', ''))}</td>"
|
||||
f"<td><code>{html_text(step.get('command', ''))}</code></td>"
|
||||
f"<td>{html_text(step.get('pass_condition', ''))}</td>"
|
||||
"</tr>"
|
||||
)
|
||||
return "<div class='table-wrap'><table><thead><tr><th>Step</th><th>Evidence</th><th>Owner</th><th>User</th><th>User action</th><th>Assistant action</th><th>Command</th><th>Pass condition</th></tr></thead><tbody>" + "".join(rows) + "</tbody></table></div>"
|
||||
|
||||
|
||||
def render_html_release_gate(release_gate: dict[str, Any]) -> str:
|
||||
checks = release_gate.get("checks", []) if isinstance(release_gate.get("checks", []), list) else []
|
||||
rows = []
|
||||
for check in checks:
|
||||
status = "pass" if check.get("passed") is True else "blocked"
|
||||
rows.append(
|
||||
f"<tr class='{html_text(status)}'>"
|
||||
f"<td>{html_text(check.get('label', ''))}</td>"
|
||||
f"<td><code>{html_text(check.get('current', ''))}</code></td>"
|
||||
f"<td><code>{html_text(check.get('expected', ''))}</code></td>"
|
||||
f"<td><code>{html_text(status)}</code></td>"
|
||||
f"<td><code>{html_text(check.get('artifact', ''))}</code></td>"
|
||||
"</tr>"
|
||||
)
|
||||
check_table = (
|
||||
"<div class='table-wrap'><table><thead><tr><th>Check</th><th>Current</th><th>Expected</th><th>Status</th><th>Artifact</th></tr></thead><tbody>"
|
||||
+ "".join(rows)
|
||||
+ "</tbody></table></div>"
|
||||
if rows
|
||||
else "<p class='muted'>No release checks listed.</p>"
|
||||
)
|
||||
return f"""
|
||||
<div class="gate-summary">
|
||||
<article><span>Decision</span><strong>{html_text(release_gate.get('decision', 'missing'))}</strong></article>
|
||||
<article><span>Ready</span><strong>{html_text(str(release_gate.get('ready') is True).lower())}</strong></article>
|
||||
<article><span>Blocked</span><strong>{html_text(release_gate.get('blocked_count', 0))}/{html_text(release_gate.get('check_count', 0))}</strong></article>
|
||||
<article><span>Counts</span><strong>{html_text(str(release_gate.get('counts_as_completion') is True).lower())}</strong></article>
|
||||
</div>
|
||||
<p class="muted">{html_text(release_gate.get('final_manual_check', ''))}</p>
|
||||
{check_table}
|
||||
""".strip()
|
||||
|
||||
|
||||
def render_html_item(item: dict[str, Any]) -> str:
|
||||
commands = "".join(
|
||||
f"<li><span>{html_text(label.replace('_', ' '))}</span><code>{html_text(command)}</code></li>"
|
||||
for label, command in item["commands"].items()
|
||||
)
|
||||
must_collect = item["must_collect"]
|
||||
return f"""
|
||||
<article class="item-card {html_text(item['review_state'])}">
|
||||
<header><span>{html_text(item['category'])} · {html_text(item['review_state'])}</span><h3>{html_text(item['label'])}</h3></header>
|
||||
<p>{html_text(item['blocking_reason'])}</p>
|
||||
<dl>
|
||||
<dt>Owner</dt><dd>{html_text(item['owner'])}</dd>
|
||||
<dt>Ledger</dt><dd><code>{html_text(item['ledger_status'])}</code></dd>
|
||||
<dt>Blocked</dt><dd><code>{html_text(item.get('blocked_source_check_count', 0))}</code></dd>
|
||||
<dt>Queue</dt><dd><code>{html_text(item.get('phase_queue_blocked_count', 0))}</code></dd>
|
||||
<dt>Submission</dt><dd><code>{html_text(item['submission_path'])}</code></dd>
|
||||
</dl>
|
||||
<section class="source-panel"><h4>Phase Queue</h4>{html_phase_queue(item.get('phase_queue', []))}</section>
|
||||
<section class="source-panel"><h4>Source Runbook</h4><ul>{html_list(item.get('execution_runbook', []), 'No source runbook listed.')}</ul></section>
|
||||
<section><h4>Commands</h4><ul class="commands">{commands}</ul></section>
|
||||
<div class="mini-grid">
|
||||
<section><h4>Must Collect</h4><ul>{html_list(must_collect.get('provenance_requirements', []), 'No provenance requirements listed.')}</ul></section>
|
||||
<section><h4>Success Checks</h4><ul>{html_list(must_collect.get('success_checks', []), 'No success checks listed.')}</ul></section>
|
||||
<section><h4>Privacy</h4><ul>{html_list(must_collect.get('privacy_contract', []), 'No privacy contract listed.')}</ul></section>
|
||||
</div>
|
||||
<section class="source-panel"><h4>Next Source Actions</h4><ul>{html_list(item.get('next_source_actions', []), 'No blocked source checks.')}</ul></section>
|
||||
<section class="source-panel"><h4>Source Evidence Snapshot</h4>{html_source_checks(item.get('source_checklist', []))}</section>
|
||||
</article>
|
||||
""".strip()
|
||||
|
||||
|
||||
def render_html(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
stats = [
|
||||
("Pending", summary["pending_count"]),
|
||||
("Awaiting", summary["awaiting_submission_count"]),
|
||||
("Ready", summary["ready_for_ledger_review_count"]),
|
||||
("Source", f"{summary.get('source_pass_count', 0)}/{summary.get('source_check_count', 0)}"),
|
||||
("Queue", f"{summary.get('phase_queue_blocked_count', 0)}/{summary.get('phase_queue_count', 0)}"),
|
||||
("Steps", f"{summary.get('coordination_user_required_step_count', 0)}/{summary.get('coordination_step_count', 0)}"),
|
||||
("Gate", f"{summary.get('release_gate_blocked_count', 0)}/{summary.get('release_gate_check_count', 0)}"),
|
||||
("Blocked", summary.get("source_blocked_count", 0)),
|
||||
("Invalid", summary["invalid_submission_count"]),
|
||||
]
|
||||
stat_html = "".join(f"<article><span>{html_text(label)}</span><strong>{html_text(value)}</strong></article>" for label, value in stats)
|
||||
item_html = "".join(render_html_item(item) for item in report["items"])
|
||||
coordination_html = render_html_coordination_plan(report.get("coordination_plan", []))
|
||||
release_gate_html = render_html_release_gate(report.get("release_gate", {}))
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>World-Class Operator Runbook</title>
|
||||
<style>
|
||||
:root {{ --ink:#1B365D; --text:#202124; --muted:#6f6a63; --line:#e8e1d8; --soft:#f8f6f2; --warn:#9b4d0f; --pass:#1f6f43; }}
|
||||
* {{ box-sizing:border-box; }}
|
||||
body {{ margin:0; background:#fff; color:var(--text); font:16px/1.55 Georgia, "Times New Roman", serif; }}
|
||||
.topbar {{ position:sticky; top:0; z-index:10; background:rgba(255,255,255,.96); border-bottom:1px solid var(--line); }}
|
||||
.topbar-inner {{ max-width:1180px; margin:0 auto; padding:12px 24px; display:flex; justify-content:space-between; gap:16px; align-items:center; }}
|
||||
.brand, a {{ color:var(--ink); }}
|
||||
.links {{ display:flex; gap:14px; flex-wrap:wrap; }}
|
||||
.links a {{ text-decoration:none; }}
|
||||
.shell {{ max-width:1180px; margin:0 auto; padding:36px 24px 72px; }}
|
||||
.hero {{ border-bottom:1px solid var(--line); padding:32px 0 28px; }}
|
||||
.eyebrow {{ color:var(--ink); font-size:12px; text-transform:uppercase; font-weight:700; letter-spacing:0; }}
|
||||
h1 {{ margin:8px 0 12px; color:var(--ink); font-size:56px; line-height:1.04; letter-spacing:0; }}
|
||||
h2, h3, h4 {{ color:var(--ink); letter-spacing:0; }}
|
||||
h2 {{ margin:0 0 14px; font-size:30px; }}
|
||||
h3 {{ margin:4px 0 10px; font-size:22px; }}
|
||||
h4 {{ margin:0 0 8px; font-size:16px; }}
|
||||
.lede {{ max-width:800px; color:var(--muted); font-size:20px; }}
|
||||
.stats {{ display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:26px; }}
|
||||
.stats article, .panel, .item-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }}
|
||||
.stats article {{ padding:16px; }}
|
||||
.stats span, .item-card span, .muted {{ color:var(--muted); }}
|
||||
.stats strong {{ display:block; color:var(--ink); font-size:34px; line-height:1; }}
|
||||
.section {{ padding:32px 0; border-bottom:1px solid var(--line); }}
|
||||
.item-grid {{ display:grid; grid-template-columns:1fr; gap:18px; }}
|
||||
.item-card {{ padding:20px; border-left:4px solid var(--warn); }}
|
||||
.item-card.ready-for-ledger-review, .item-card.accepted {{ border-left-color:var(--pass); }}
|
||||
dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }}
|
||||
dt {{ color:var(--ink); }}
|
||||
dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }}
|
||||
code {{ font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:13px; overflow-wrap:anywhere; }}
|
||||
ul, ol {{ padding-left:20px; }}
|
||||
.commands {{ list-style:none; padding:0; margin:0; display:grid; gap:10px; }}
|
||||
.commands li {{ padding:12px; background:var(--soft); border-radius:8px; }}
|
||||
.commands span {{ display:block; color:var(--ink); font-weight:700; margin-bottom:4px; }}
|
||||
.mini-grid {{ display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:14px; }}
|
||||
.mini-grid section, .panel {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }}
|
||||
.mini-grid li, .panel li {{ overflow-wrap:anywhere; }}
|
||||
.source-panel {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }}
|
||||
.table-wrap {{ overflow-x:auto; border:1px solid var(--line); border-radius:8px; background:#fff; }}
|
||||
table {{ width:100%; min-width:980px; border-collapse:collapse; }}
|
||||
th, td {{ padding:12px; border-bottom:1px solid var(--line); text-align:left; vertical-align:top; overflow-wrap:anywhere; }}
|
||||
th {{ color:var(--ink); background:var(--soft); font-size:13px; }}
|
||||
tr:last-child td {{ border-bottom:0; }}
|
||||
.gate-summary {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin:12px 0 16px; }}
|
||||
.gate-summary article {{ border:1px solid var(--line); border-radius:8px; background:#fff; padding:14px; min-width:0; }}
|
||||
.gate-summary span {{ display:block; color:var(--muted); }}
|
||||
.gate-summary strong {{ display:block; color:var(--ink); font-size:20px; line-height:1.15; overflow-wrap:anywhere; }}
|
||||
tr.blocked td:first-child {{ border-left:3px solid var(--warn); }}
|
||||
tr.pass td:first-child {{ border-left:3px solid var(--pass); }}
|
||||
.source-checks {{ list-style:none; padding:0; margin:0; display:grid; gap:8px; }}
|
||||
.source-check {{ border-top:1px solid var(--line); padding-top:8px; display:grid; gap:3px; }}
|
||||
.source-check span {{ color:var(--ink); }}
|
||||
.source-check code, .source-check small {{ overflow-wrap:anywhere; }}
|
||||
@media (max-width:820px) {{ .stats, .mini-grid, .gate-summary {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="topbar"><div class="topbar-inner"><span class="brand">World-Class Runbook</span><div class="links"><a href="#fast-path">Fast Path</a><a href="#coordination">Coordination</a><a href="#phase-queue">Queue</a><a href="#items">Evidence</a><a href="#release-gate">Release Gate</a><a href="#boundary">Boundary</a></div></div></nav>
|
||||
<main class="shell">
|
||||
<section class="hero">
|
||||
<span class="eyebrow">Evidence Operations</span>
|
||||
<h1>World-Class Operator Runbook</h1>
|
||||
<p class="lede">A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.</p>
|
||||
<div class="stats">{stat_html}</div>
|
||||
</section>
|
||||
<section class="section panel" id="fast-path"><h2>Fast Path</h2><ol><li>Run the real external or human work for one evidence item.</li><li>Generate and fill the matching submission draft.</li><li>Validate intake and inspect the submission review queue.</li><li>Refresh the ledger and run the claim guard before making any completion claim.</li></ol></section>
|
||||
<section class="section panel" id="coordination"><h2>Coordination Plan</h2><p class="muted">These steps tell the assistant what to run and where the user must supply real provider, reviewer, client, or telemetry input. Every row is operational guidance only and keeps counts_as_completion false.</p>{coordination_html}</section>
|
||||
<section class="section panel" id="phase-queue"><h2>Phase Queue</h2>{html_phase_queue(report.get('phase_queue', []))}</section>
|
||||
<section class="section" id="items"><h2>Evidence Items</h2><div class="item-grid">{item_html}</div></section>
|
||||
<section class="section panel" id="release-gate"><h2>Release Gate</h2>{release_gate_html}</section>
|
||||
<section class="section panel" id="boundary"><h2>Boundary</h2><ul><li>Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion.</li><li>Valid intake means ready for submission review; ledger review still requires passing source evidence.</li><li>The world-class ledger and claim guard remain the source of truth.</li></ul></section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render an operator runbook for collecting pending world-class evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--submissions-dir")
|
||||
parser.add_argument("--output-json", default="reports/world_class_operator_runbook.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_operator_runbook.md")
|
||||
parser.add_argument("--output-html", default="reports/world_class_operator_runbook.html")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
|
||||
report = build_operator_runbook(skill_dir, args.generated_at, submissions_dir=submissions_dir)
|
||||
outputs = {
|
||||
"json": Path(args.output_json),
|
||||
"markdown": Path(args.output_md),
|
||||
"html": Path(args.output_html),
|
||||
}
|
||||
for key, path in outputs.items():
|
||||
if not path.is_absolute():
|
||||
path = skill_dir / path
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report["artifacts"][key] = output_path(str(path), skill_dir)
|
||||
if key == "json":
|
||||
path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
elif key == "markdown":
|
||||
path.write_text(render_markdown(report), encoding="utf-8")
|
||||
else:
|
||||
path.write_text(render_html(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,471 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from prepare_world_class_submission_kit import build_artifact_checklist
|
||||
from render_world_class_evidence_intake import build_intake
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
from render_world_class_submission_review import build_submission_review
|
||||
from world_class_evidence_contract import rel_path
|
||||
from world_class_phase_queue import build_phase_queue, summarize_phase_queue
|
||||
from world_class_preflight_markdown import render_markdown
|
||||
from world_class_preflight_layout import render_html
|
||||
from world_class_repair_checklist import build_preflight_repair_checklist, summarize_repair_checklist
|
||||
from world_class_source_checks import summarize_source_checklist
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a preflight checklist for collecting pending world-class evidence without accepting evidence."
|
||||
|
||||
|
||||
PREFLIGHT_SPECS: dict[str, list[dict[str, Any]]] = {
|
||||
"provider-holdout": [
|
||||
{
|
||||
"key": "output-cases",
|
||||
"label": "Output eval cases",
|
||||
"kind": "file",
|
||||
"path": "evals/output/cases.jsonl",
|
||||
"required": True,
|
||||
"next_action": "Keep output holdout cases available before provider execution.",
|
||||
},
|
||||
{
|
||||
"key": "provider-runner",
|
||||
"label": "Provider runner",
|
||||
"kind": "file",
|
||||
"path": "scripts/provider_output_eval_runner.py",
|
||||
"required": True,
|
||||
"next_action": "Use the provider runner instead of the local command runner for model-backed evidence.",
|
||||
},
|
||||
{
|
||||
"key": "provider-api-key",
|
||||
"label": "Provider credential",
|
||||
"kind": "env_any",
|
||||
"names": ["OPENAI_API_KEY", "DEEPSEEK_API_KEY"],
|
||||
"required": True,
|
||||
"secret": True,
|
||||
"next_action": "Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.",
|
||||
},
|
||||
{
|
||||
"key": "provider-model",
|
||||
"label": "Provider model",
|
||||
"kind": "env",
|
||||
"name": "YAO_OUTPUT_EVAL_MODEL",
|
||||
"required": False,
|
||||
"default": "provider-specific model, or pass --provider-model",
|
||||
"next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL, or pass --provider-model for the selected provider.",
|
||||
},
|
||||
],
|
||||
"human-adjudication": [
|
||||
{
|
||||
"key": "review-kit",
|
||||
"label": "Blind review kit",
|
||||
"kind": "file",
|
||||
"path": "reports/output_review_kit.html",
|
||||
"required": True,
|
||||
"next_action": "Open the blind review kit and record real reviewer choices with required rationale.",
|
||||
},
|
||||
{
|
||||
"key": "decision-template",
|
||||
"label": "Decision template",
|
||||
"kind": "file",
|
||||
"path": "reports/output_review_decisions.json",
|
||||
"required": True,
|
||||
"next_action": "Import real A/B decisions with reviewer, reviewed_at, winner_variant, confidence, reason, and truthful blind-review attestation via `python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication`.",
|
||||
},
|
||||
{
|
||||
"key": "decision-importer",
|
||||
"label": "Decision importer",
|
||||
"kind": "file",
|
||||
"path": "scripts/import_output_review_decisions.py",
|
||||
"required": True,
|
||||
"next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.",
|
||||
},
|
||||
{
|
||||
"key": "human-reviewer",
|
||||
"label": "Human reviewer",
|
||||
"kind": "human",
|
||||
"required": True,
|
||||
"next_action": "Assign a real reviewer identity before claiming human adjudication.",
|
||||
},
|
||||
],
|
||||
"native-permission-enforcement": [
|
||||
{
|
||||
"key": "permission-policy",
|
||||
"label": "Permission policy",
|
||||
"kind": "file",
|
||||
"path": "security/permission_policy.json",
|
||||
"required": True,
|
||||
"next_action": "Keep approved high-permission capabilities explicit.",
|
||||
},
|
||||
{
|
||||
"key": "permission-probes",
|
||||
"label": "Runtime probes",
|
||||
"kind": "file",
|
||||
"path": "reports/runtime_permission_probes.json",
|
||||
"required": True,
|
||||
"next_action": "Refresh runtime permission probes after packaging changes.",
|
||||
},
|
||||
{
|
||||
"key": "native-guard",
|
||||
"label": "Native guard",
|
||||
"kind": "external",
|
||||
"required": True,
|
||||
"next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.",
|
||||
},
|
||||
],
|
||||
"native-client-telemetry": [
|
||||
{
|
||||
"key": "native-host",
|
||||
"label": "Native telemetry host",
|
||||
"kind": "file",
|
||||
"path": "scripts/telemetry_native_host.py",
|
||||
"required": True,
|
||||
"next_action": "Use the native host to receive metadata-only client events.",
|
||||
},
|
||||
{
|
||||
"key": "hook-recipes",
|
||||
"label": "Hook recipes",
|
||||
"kind": "file",
|
||||
"path": "reports/telemetry_hook_recipes.json",
|
||||
"required": True,
|
||||
"next_action": "Refresh telemetry hook recipes before external client installation.",
|
||||
},
|
||||
{
|
||||
"key": "external-client",
|
||||
"label": "External client",
|
||||
"kind": "external",
|
||||
"required": True,
|
||||
"next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def env_present(name: str) -> bool:
|
||||
return bool(os.environ.get(name))
|
||||
|
||||
|
||||
def shell_path(path: Path, root: Path) -> str:
|
||||
return shlex.quote(rel_path(path, root))
|
||||
|
||||
|
||||
def build_submission_commands(skill_dir: Path, submissions_dir: Path, evidence_key: str | None = None) -> dict[str, str]:
|
||||
output_dir = shell_path(submissions_dir, skill_dir)
|
||||
prepare = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir}"
|
||||
if evidence_key:
|
||||
prepare = f"python3 scripts/yao.py world-class-submission-kit . --evidence-key {evidence_key} --output-dir {output_dir}"
|
||||
prefilled_prepare = f"{prepare} --prefill-artifacts"
|
||||
return {
|
||||
"prepare_submission": prepare,
|
||||
"prepare_prefilled_submission": prefilled_prepare,
|
||||
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir}",
|
||||
"submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir}",
|
||||
"refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir}",
|
||||
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
|
||||
}
|
||||
|
||||
|
||||
def build_artifact_role_contract(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
submission_ref_rows = [row for row in rows if row.get("submission_ref_required") is True]
|
||||
supporting_rows = [row for row in rows if row.get("submission_ref_required") is not True]
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"role_source": "world-class-submission-kit",
|
||||
"counts_as_evidence": False,
|
||||
"artifact_prefill_counts_as_evidence": False,
|
||||
"submission_ref_total_count": len(submission_ref_rows),
|
||||
"submission_ref_ready_count": sum(1 for row in submission_ref_rows if row.get("artifact_ref_ready")),
|
||||
"supporting_evidence_total_count": len(supporting_rows),
|
||||
"supporting_evidence_ready_count": sum(1 for row in supporting_rows if row.get("artifact_ref_ready")),
|
||||
"roles": [
|
||||
{
|
||||
"role": "submission-ref",
|
||||
"label": "Submission refs",
|
||||
"copy_to_artifact_refs": True,
|
||||
"description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs.",
|
||||
},
|
||||
{
|
||||
"role": "supporting-evidence",
|
||||
"label": "Supporting evidence",
|
||||
"copy_to_artifact_refs": False,
|
||||
"description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def artifact_role_contract_for_key(contract: dict[str, Any], rows: list[dict[str, Any]], evidence_key: str) -> dict[str, Any]:
|
||||
item_rows = [row for row in rows if row.get("evidence_key") == evidence_key]
|
||||
role_contract = build_artifact_role_contract(item_rows)
|
||||
return {
|
||||
**role_contract,
|
||||
"schema_version": contract.get("schema_version", "1.0"),
|
||||
"role_source": contract.get("role_source", "world-class-submission-kit"),
|
||||
}
|
||||
|
||||
|
||||
def build_precheck(skill_dir: Path, evidence_key: str, spec: dict[str, Any]) -> dict[str, Any]:
|
||||
kind = str(spec.get("kind", ""))
|
||||
required = spec.get("required") is True
|
||||
row = {
|
||||
"evidence_key": evidence_key,
|
||||
"key": spec.get("key", ""),
|
||||
"label": spec.get("label", ""),
|
||||
"kind": kind,
|
||||
"required": required,
|
||||
"next_action": spec.get("next_action", ""),
|
||||
"secret_value_redacted": spec.get("secret") is True,
|
||||
}
|
||||
if kind == "file":
|
||||
path = skill_dir / str(spec.get("path", ""))
|
||||
exists = path.exists()
|
||||
row.update(
|
||||
{
|
||||
"path": rel_path(path, skill_dir),
|
||||
"status": "pass" if exists else ("missing" if required else "optional"),
|
||||
"actual": "present" if exists else "missing",
|
||||
}
|
||||
)
|
||||
return row
|
||||
if kind == "env":
|
||||
name = str(spec.get("name", ""))
|
||||
present = env_present(name)
|
||||
row.update(
|
||||
{
|
||||
"env": name,
|
||||
"status": "pass" if present else ("missing" if required else "optional"),
|
||||
"actual": "set" if present else "not-set",
|
||||
"default": spec.get("default", ""),
|
||||
}
|
||||
)
|
||||
return row
|
||||
if kind == "env_any":
|
||||
names = [str(name) for name in spec.get("names", []) if str(name).strip()]
|
||||
set_names = [name for name in names if env_present(name)]
|
||||
row.update(
|
||||
{
|
||||
"env_any": names,
|
||||
"status": "pass" if set_names else ("missing" if required else "optional"),
|
||||
"actual": "set" if set_names else "not-set",
|
||||
"set_count": len(set_names),
|
||||
"set_envs": set_names,
|
||||
}
|
||||
)
|
||||
return row
|
||||
if kind == "human":
|
||||
row.update(
|
||||
{
|
||||
"status": "human-required",
|
||||
"actual": "external-human-action",
|
||||
}
|
||||
)
|
||||
return row
|
||||
if kind == "external":
|
||||
row.update(
|
||||
{
|
||||
"status": "external-required",
|
||||
"actual": "external-integration-required",
|
||||
}
|
||||
)
|
||||
return row
|
||||
row.update({"status": "unknown", "actual": "unknown"})
|
||||
return row
|
||||
|
||||
|
||||
def item_status(entry: dict[str, Any], prechecks: list[dict[str, Any]], source_rows: list[dict[str, Any]]) -> str:
|
||||
if entry.get("status") == "accepted":
|
||||
return "accepted"
|
||||
blocking = {row.get("status") for row in prechecks if row.get("required") is True}
|
||||
if "missing" in blocking or "external-required" in blocking:
|
||||
return "blocked"
|
||||
if "human-required" in blocking:
|
||||
return "ready-for-human-review"
|
||||
if any(row.get("status") != "pass" for row in source_rows):
|
||||
return "ready-to-collect"
|
||||
return "ready-for-submission"
|
||||
|
||||
|
||||
def first_next_action(prechecks: list[dict[str, Any]], source_rows: list[dict[str, Any]]) -> str:
|
||||
for row in prechecks:
|
||||
if row.get("required") is True and row.get("status") != "pass":
|
||||
return str(row.get("next_action", ""))
|
||||
for row in source_rows:
|
||||
if row.get("status") != "pass":
|
||||
return str(row.get("next_action", ""))
|
||||
return "Validate intake, refresh the ledger, and run the claim guard."
|
||||
|
||||
|
||||
def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
review = build_submission_review(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
review_by_key = {str(item.get("evidence_key", "")): item for item in review.get("items", [])}
|
||||
intake_by_key = {str(item.get("evidence_key", "")): item for item in intake.get("operator_checklist", [])}
|
||||
operator_items = intake.get("operator_checklist", [])
|
||||
operator_items = operator_items if isinstance(operator_items, list) else []
|
||||
artifact_checklist = build_artifact_checklist(skill_dir, operator_items)
|
||||
artifact_role_contract = build_artifact_role_contract(artifact_checklist)
|
||||
items: list[dict[str, Any]] = []
|
||||
precheck_rows: list[dict[str, Any]] = []
|
||||
source_rows: list[dict[str, Any]] = []
|
||||
for entry in ledger.get("entries", []):
|
||||
key = str(entry.get("key", ""))
|
||||
prechecks = [build_precheck(skill_dir, key, spec) for spec in PREFLIGHT_SPECS.get(key, [])]
|
||||
review_item = review_by_key.get(key, {})
|
||||
item_source_rows = review_item.get("source_checklist", entry.get("source_checklist", []))
|
||||
item_source_rows = item_source_rows if isinstance(item_source_rows, list) else []
|
||||
status = item_status(entry, prechecks, item_source_rows)
|
||||
item_commands = build_submission_commands(skill_dir, submissions_dir, key)
|
||||
item = {
|
||||
"evidence_key": key,
|
||||
"label": entry.get("label", key),
|
||||
"category": entry.get("category", "external"),
|
||||
"owner": entry.get("owner", "release reviewer"),
|
||||
"status": status,
|
||||
"ledger_status": entry.get("status", "pending"),
|
||||
"intake_readiness": intake_by_key.get(key, {}).get("readiness", "missing"),
|
||||
"review_state": review_item.get("review_state", "missing"),
|
||||
"source_accepted": entry.get("source_accepted") is True,
|
||||
"prechecks": prechecks,
|
||||
"source_checklist": item_source_rows,
|
||||
"next_action": first_next_action(prechecks, item_source_rows),
|
||||
"submission_path": intake_by_key.get(key, {}).get("submission_path", ""),
|
||||
"template_path": intake_by_key.get(key, {}).get("template_path", ""),
|
||||
"commands": item_commands,
|
||||
"submission_kit": {
|
||||
"prepare_command": item_commands["prepare_submission"],
|
||||
"prefill_command": item_commands["prepare_prefilled_submission"],
|
||||
"output_dir": rel_path(submissions_dir, skill_dir),
|
||||
"draft_path": intake_by_key.get(key, {}).get("submission_path", ""),
|
||||
"template_path": intake_by_key.get(key, {}).get("template_path", ""),
|
||||
"artifact_role_contract": artifact_role_contract_for_key(
|
||||
artifact_role_contract,
|
||||
artifact_checklist,
|
||||
key,
|
||||
),
|
||||
"drafts_count_as_evidence": False,
|
||||
"artifact_prefill_counts_as_evidence": False,
|
||||
},
|
||||
"runbook": entry.get("runbook", []),
|
||||
}
|
||||
items.append(item)
|
||||
precheck_rows.extend(prechecks)
|
||||
source_rows.extend(item_source_rows)
|
||||
repair_checklist = build_preflight_repair_checklist(items)
|
||||
repairs_by_key: dict[str, list[dict[str, Any]]] = {}
|
||||
for row in repair_checklist:
|
||||
key = str(row.get("evidence_key", ""))
|
||||
repairs_by_key.setdefault(key, []).append(row)
|
||||
for item in items:
|
||||
item_repair_rows = repairs_by_key.get(str(item.get("evidence_key", "")), [])
|
||||
item["repair_checklist"] = item_repair_rows
|
||||
item["phase_queue"] = build_phase_queue(item_repair_rows)
|
||||
precheck_status_counts: dict[str, int] = {}
|
||||
for row in precheck_rows:
|
||||
status = str(row.get("status", "unknown"))
|
||||
precheck_status_counts[status] = precheck_status_counts.get(status, 0) + 1
|
||||
source_summary = summarize_source_checklist(source_rows)
|
||||
repair_summary = summarize_repair_checklist(repair_checklist)
|
||||
phase_queue = build_phase_queue(repair_checklist)
|
||||
phase_queue_summary = summarize_phase_queue(phase_queue)
|
||||
collection_ready_count = sum(1 for item in items if item["status"] in {"ready-to-collect", "ready-for-human-review", "ready-for-submission"})
|
||||
blocked_count = sum(1 for item in items if item["status"] == "blocked")
|
||||
ready_to_claim = ledger.get("summary", {}).get("ready_to_claim_world_class") is True
|
||||
summary = {
|
||||
"evidence_item_count": len(items),
|
||||
"precheck_count": len(precheck_rows),
|
||||
"precheck_pass_count": precheck_status_counts.get("pass", 0),
|
||||
"precheck_missing_count": precheck_status_counts.get("missing", 0),
|
||||
"precheck_optional_count": precheck_status_counts.get("optional", 0),
|
||||
"precheck_human_required_count": precheck_status_counts.get("human-required", 0),
|
||||
"precheck_external_required_count": precheck_status_counts.get("external-required", 0),
|
||||
"collection_ready_count": collection_ready_count,
|
||||
"collection_blocked_count": blocked_count,
|
||||
**source_summary,
|
||||
**repair_summary,
|
||||
**phase_queue_summary,
|
||||
"pending_count": ledger.get("summary", {}).get("pending_count", 0),
|
||||
"ready_to_claim_world_class": ready_to_claim,
|
||||
"credential_value_exposed": False,
|
||||
"preflight_counts_as_evidence": False,
|
||||
"decision": "ready-for-completion-audit" if ready_to_claim else ("collection-preflight-blocked" if blocked_count else "ready-to-collect-evidence"),
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": True,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
"summary": summary,
|
||||
"status_counts": {item["status"]: sum(1 for candidate in items if candidate["status"] == item["status"]) for item in items},
|
||||
"precheck_status_counts": precheck_status_counts,
|
||||
"items": items,
|
||||
"prechecks": precheck_rows,
|
||||
"source_checklist": source_rows,
|
||||
"repair_checklist": repair_checklist,
|
||||
"phase_queue": phase_queue,
|
||||
"submissions": {
|
||||
"directory": rel_path(submissions_dir, skill_dir),
|
||||
"commands": build_submission_commands(skill_dir, submissions_dir),
|
||||
"submission_kit_command": build_submission_commands(skill_dir, submissions_dir)["prepare_submission"],
|
||||
"submission_kit_prefill_command": build_submission_commands(skill_dir, submissions_dir)[
|
||||
"prepare_prefilled_submission"
|
||||
],
|
||||
"preflight_counts_submission_as_completion": False,
|
||||
"drafts_count_as_evidence": False,
|
||||
"artifact_prefill_counts_as_evidence": False,
|
||||
"artifact_role_contract": artifact_role_contract,
|
||||
},
|
||||
"source_reports": {
|
||||
"ledger": "reports/world_class_evidence_ledger.json",
|
||||
"intake": "reports/world_class_evidence_intake.json",
|
||||
"submission_review": "reports/world_class_submission_review.json",
|
||||
"operator_runbook": "reports/world_class_operator_runbook.json",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_evidence_preflight.json",
|
||||
"markdown": "reports/world_class_evidence_preflight.md",
|
||||
"html": "reports/world_class_evidence_preflight.html",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render collection preflight checks for pending world-class evidence.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--submissions-dir")
|
||||
parser.add_argument("--output-json", default="reports/world_class_evidence_preflight.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_evidence_preflight.md")
|
||||
parser.add_argument("--output-html", default="reports/world_class_evidence_preflight.html")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
|
||||
report = build_preflight(skill_dir, args.generated_at, submissions_dir=submissions_dir)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
output_html = Path(args.output_html)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
if not output_html.is_absolute():
|
||||
output_html = skill_dir / output_html
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_html.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
output_html.write_text(render_html(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,305 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_world_class_evidence_intake import build_intake
|
||||
from render_world_class_evidence_ledger import build_ledger
|
||||
from world_class_source_checks import build_source_checklist, summarize_source_checklist
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Renders a read-only review queue for world-class evidence submissions before ledger acceptance."
|
||||
|
||||
TOP_LEVEL_SUMMARY_FIELDS = [
|
||||
"decision",
|
||||
"ready_to_claim_world_class",
|
||||
"review_item_count",
|
||||
"accepted_count",
|
||||
"awaiting_submission_count",
|
||||
"valid_packet_source_incomplete_count",
|
||||
"ready_for_ledger_review_count",
|
||||
"fix_submission_count",
|
||||
"unmatched_submission_count",
|
||||
"invalid_submission_count",
|
||||
"source_check_count",
|
||||
"source_pass_count",
|
||||
"source_blocked_count",
|
||||
"review_counts_submission_as_completion",
|
||||
]
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def by_key(items: list[dict[str, Any]], key_name: str = "evidence_key") -> dict[str, dict[str, Any]]:
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
key = str(item.get(key_name, ""))
|
||||
if key and key not in result:
|
||||
result[key] = item
|
||||
return result
|
||||
|
||||
|
||||
def top_level_summary_mirrors(summary: dict[str, Any]) -> dict[str, Any]:
|
||||
return {key: summary[key] for key in TOP_LEVEL_SUMMARY_FIELDS if key in summary}
|
||||
|
||||
|
||||
def review_state(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> tuple[str, str]:
|
||||
if entry.get("status") == "accepted":
|
||||
return "accepted", "Ledger already accepts this evidence item."
|
||||
|
||||
submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {}
|
||||
submission_status = str(submission.get("status", "missing"))
|
||||
if submission_status == "missing":
|
||||
return "awaiting-submission", "No evidence packet has been submitted for review."
|
||||
if submission_status in {"invalid-json", "invalid-contract"}:
|
||||
return "fix-submission", "Submission exists but fails the ledger submission contract."
|
||||
if intake_result is None:
|
||||
return "fix-submission", "Submission exists but is missing from the intake validation results."
|
||||
if intake_result.get("status") != "pass":
|
||||
return "fix-submission", "Submission exists but failed intake validation."
|
||||
|
||||
observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {}
|
||||
if observed.get("accepted") is True:
|
||||
return "ready-for-ledger-review", "Submission and source evidence are ready for final ledger review."
|
||||
return "source-evidence-incomplete", "Submission packet is valid, but the source evidence checks still do not pass."
|
||||
|
||||
|
||||
def build_item(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> dict[str, Any]:
|
||||
state, reason = review_state(entry, intake_result)
|
||||
submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {}
|
||||
observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {}
|
||||
source_checklist = build_source_checklist([entry])
|
||||
source_summary = summarize_source_checklist(source_checklist)
|
||||
return {
|
||||
"evidence_key": entry.get("key", ""),
|
||||
"label": entry.get("label", entry.get("key", "")),
|
||||
"category": entry.get("category", "external"),
|
||||
"owner": entry.get("owner", "release reviewer"),
|
||||
"ledger_status": entry.get("status", "pending"),
|
||||
"submission_status": submission.get("status", "missing"),
|
||||
"intake_status": intake_result.get("status", "missing") if intake_result else "missing",
|
||||
"source_accepted": observed.get("accepted") is True,
|
||||
"review_state": state,
|
||||
"blocking_reason": reason,
|
||||
"submission_path": submission.get("path", ""),
|
||||
"submitted_by": submission.get("submitted_by", ""),
|
||||
"submitted_at": submission.get("submitted_at", ""),
|
||||
"artifact_ref_count": submission.get("artifact_ref_count", 0),
|
||||
"intake_errors": intake_result.get("errors", []) if intake_result else [],
|
||||
"observed_state": observed,
|
||||
"source_checklist": source_checklist,
|
||||
**source_summary,
|
||||
"success_checks": entry.get("success_checks", []),
|
||||
"privacy_contract": entry.get("privacy_contract", []),
|
||||
"next_action": entry.get("next_action", ""),
|
||||
}
|
||||
|
||||
|
||||
def build_submission_review(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]:
|
||||
submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions")
|
||||
ledger = build_ledger(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
intake = build_intake(skill_dir, generated_at, submissions_dir=submissions_dir)
|
||||
intake_by_key = by_key(intake.get("submissions", []))
|
||||
entries = ledger.get("entries", [])
|
||||
items = [build_item(entry, intake_by_key.get(str(entry.get("key", "")))) for entry in entries]
|
||||
known_keys = {str(entry.get("key", "")) for entry in entries}
|
||||
unmatched = [item for item in intake.get("submissions", []) if str(item.get("evidence_key", "")) not in known_keys]
|
||||
counts: dict[str, int] = {}
|
||||
for item in items:
|
||||
state = str(item.get("review_state", "unknown"))
|
||||
counts[state] = counts.get(state, 0) + 1
|
||||
invalid_count = counts.get("fix-submission", 0) + len(unmatched)
|
||||
ready_count = counts.get("ready-for-ledger-review", 0)
|
||||
accepted_count = counts.get("accepted", 0)
|
||||
source_incomplete_count = counts.get("source-evidence-incomplete", 0)
|
||||
awaiting_count = counts.get("awaiting-submission", 0)
|
||||
source_rows = [row for item in items for row in item.get("source_checklist", [])]
|
||||
source_summary = summarize_source_checklist(source_rows)
|
||||
if accepted_count == len(items) and items:
|
||||
decision = "ledger-complete"
|
||||
elif invalid_count:
|
||||
decision = "fix-submissions"
|
||||
elif ready_count:
|
||||
decision = "review-ready"
|
||||
elif source_incomplete_count:
|
||||
decision = "source-evidence-incomplete"
|
||||
else:
|
||||
decision = "awaiting-submissions"
|
||||
summary = {
|
||||
"review_item_count": len(items),
|
||||
"accepted_count": accepted_count,
|
||||
"awaiting_submission_count": awaiting_count,
|
||||
"valid_packet_source_incomplete_count": source_incomplete_count,
|
||||
"ready_for_ledger_review_count": ready_count,
|
||||
"fix_submission_count": counts.get("fix-submission", 0),
|
||||
"unmatched_submission_count": len(unmatched),
|
||||
"invalid_submission_count": invalid_count,
|
||||
**source_summary,
|
||||
"ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True,
|
||||
"review_counts_submission_as_completion": False,
|
||||
"decision": decision,
|
||||
}
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"ok": invalid_count == 0,
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": rel_path(skill_dir, ROOT),
|
||||
**top_level_summary_mirrors(summary),
|
||||
"summary": summary,
|
||||
"report_contract": {
|
||||
"schema_version": "1.0",
|
||||
"contract": "world-class-submission-review",
|
||||
"top_level_mirrors_summary": True,
|
||||
"summary_fields": TOP_LEVEL_SUMMARY_FIELDS,
|
||||
"source_of_truth": "summary",
|
||||
},
|
||||
"submissions": {
|
||||
"directory": rel_path(submissions_dir, skill_dir),
|
||||
"review_counts_submission_as_completion": False,
|
||||
},
|
||||
"items": items,
|
||||
"unmatched_submissions": unmatched,
|
||||
"source_reports": {
|
||||
"ledger": "reports/world_class_evidence_ledger.json",
|
||||
"intake": "reports/world_class_evidence_intake.json",
|
||||
},
|
||||
"artifacts": {
|
||||
"json": "reports/world_class_submission_review.json",
|
||||
"markdown": "reports/world_class_submission_review.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_list(values: list[Any], empty: str) -> list[str]:
|
||||
if not values:
|
||||
return [f"- {empty}"]
|
||||
return [f"- {value}" for value in values]
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# World-Class Submission Review",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- review items: `{summary['review_item_count']}`",
|
||||
f"- accepted: `{summary['accepted_count']}`",
|
||||
f"- awaiting submission: `{summary['awaiting_submission_count']}`",
|
||||
f"- valid packet but source incomplete: `{summary['valid_packet_source_incomplete_count']}`",
|
||||
f"- ready for ledger review: `{summary['ready_for_ledger_review_count']}`",
|
||||
f"- fix submission: `{summary['fix_submission_count']}`",
|
||||
f"- unmatched submissions: `{summary['unmatched_submission_count']}`",
|
||||
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
|
||||
f"- review counts submission as completion: `{str(summary['review_counts_submission_as_completion']).lower()}`",
|
||||
"",
|
||||
"This report is a read-only reviewer queue. It does not accept evidence or make world-class completion true.",
|
||||
"",
|
||||
"## Queue",
|
||||
"",
|
||||
"| Evidence | Review state | Intake | Source accepted | Submission | Next action |",
|
||||
"| --- | --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for item in report["items"]:
|
||||
source_accepted = str(item.get("source_accepted") is True).lower()
|
||||
next_action = str(item.get("next_action", "")).replace("|", "\\|")
|
||||
lines.append(
|
||||
f"| `{item['evidence_key']}` | `{item['review_state']}` | `{item['intake_status']}` | "
|
||||
f"`{source_accepted}` | `{item['submission_status']}` | {next_action} |"
|
||||
)
|
||||
if report.get("unmatched_submissions"):
|
||||
lines.extend(["", "## Unmatched Submissions", ""])
|
||||
for item in report["unmatched_submissions"]:
|
||||
errors = "; ".join(item.get("errors", [])) or "unknown evidence key"
|
||||
lines.append(f"- `{item.get('path', '')}`: {errors}")
|
||||
lines.extend(["", "## Details", ""])
|
||||
for item in report["items"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {item['label']}",
|
||||
"",
|
||||
f"- review state: `{item['review_state']}`",
|
||||
f"- blocking reason: {item['blocking_reason']}",
|
||||
f"- ledger status: `{item['ledger_status']}`",
|
||||
f"- submission status: `{item['submission_status']}`",
|
||||
f"- intake status: `{item['intake_status']}`",
|
||||
f"- source accepted: `{str(item['source_accepted']).lower()}`",
|
||||
f"- submission path: `{item.get('submission_path') or 'missing'}`",
|
||||
"",
|
||||
"#### Source Checks",
|
||||
"",
|
||||
*render_list(
|
||||
[
|
||||
f"{row['label']}: {row['actual']} / {row['expected']} => {row['status']}"
|
||||
for row in item.get("source_checklist", [])
|
||||
],
|
||||
"No source checks listed.",
|
||||
),
|
||||
"",
|
||||
"#### Completion Assertions",
|
||||
"",
|
||||
*render_list(item.get("success_checks", []), "No completion assertions listed."),
|
||||
"",
|
||||
"#### Intake Errors",
|
||||
"",
|
||||
*render_list(item.get("intake_errors", []), "No intake errors."),
|
||||
"",
|
||||
"#### Privacy Contract",
|
||||
"",
|
||||
*render_list(item.get("privacy_contract", []), "No privacy contract listed."),
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"## Boundary",
|
||||
"",
|
||||
"- A valid submission packet is not accepted evidence by itself.",
|
||||
"- Planned work, metadata fallback, pending human review, and local command-runner output still do not count.",
|
||||
"- The world-class ledger remains the source of truth for `ready_to_claim_world_class`.",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render a read-only review queue for world-class evidence submissions.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--submissions-dir")
|
||||
parser.add_argument("--output-json", default="reports/world_class_submission_review.json")
|
||||
parser.add_argument("--output-md", default="reports/world_class_submission_review.md")
|
||||
parser.add_argument("--generated-at", default=date.today().isoformat())
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
submissions_dir = Path(args.submissions_dir).resolve() if args.submissions_dir else None
|
||||
report = build_submission_review(skill_dir, args.generated_at, submissions_dir=submissions_dir)
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
if not output_json.is_absolute():
|
||||
output_json = skill_dir / output_json
|
||||
if not output_md.is_absolute():
|
||||
output_md = skill_dir / output_md
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -14,6 +14,27 @@ IGNORED_RELATIVE_DIRS = {
|
||||
Path("tests") / "tmp_snapshot",
|
||||
Path("tests") / "tmp_cli",
|
||||
}
|
||||
IGNORED_FILE_PATTERNS = {
|
||||
"reports/benchmark_reproducibility*.json",
|
||||
"reports/benchmark_reproducibility*.md",
|
||||
"reports/context_budget*.json",
|
||||
"reports/context_budget*.md",
|
||||
"reports/evidence_consistency*.json",
|
||||
"reports/evidence_consistency*.md",
|
||||
"reports/review-studio*.html",
|
||||
"reports/review-studio*.json",
|
||||
"reports/review-viewer*.html",
|
||||
"reports/review-viewer*.json",
|
||||
"reports/skill-interpretation*.html",
|
||||
"reports/skill-interpretation*.json",
|
||||
"reports/skill-overview*.html",
|
||||
"reports/skill-overview*.json",
|
||||
"reports/world_class_evidence_preflight*.json",
|
||||
"reports/world_class_evidence_preflight*.md",
|
||||
"reports/world_class_evidence_preflight*.html",
|
||||
"reports/*pattern-analysis*.md",
|
||||
"reports/*research-plan*.md",
|
||||
}
|
||||
CANONICAL_PATHS = (
|
||||
"SKILL.md",
|
||||
"manifest.json",
|
||||
@@ -37,6 +58,14 @@ CONTEXT_BUDGETS = {
|
||||
}
|
||||
SKILL_BODY_BUFFER = 100
|
||||
SKILL_BODY_WARN_RATIO = 0.85
|
||||
DEFERRED_RESOURCE_DIRS = {"references", "scripts", "evals", "templates", "assets", "input", "outputs"}
|
||||
DEFERRED_RESOURCE_WARN_TOKENS = 120_000
|
||||
DEFERRED_RESOURCE_WARN_DIR_TOKENS = 80_000
|
||||
SCRIPT_GOVERNANCE_REPORTS = (
|
||||
"reports/security_trust_report.json",
|
||||
"reports/architecture_maintainability.json",
|
||||
"reports/python_compatibility.json",
|
||||
)
|
||||
|
||||
|
||||
def has_files(path: Path) -> bool:
|
||||
@@ -47,6 +76,8 @@ def should_ignore(path: Path, root: Path) -> bool:
|
||||
rel = path.relative_to(root)
|
||||
if any(rel == ignored or ignored in rel.parents for ignored in IGNORED_RELATIVE_DIRS):
|
||||
return True
|
||||
if any(rel.match(pattern) for pattern in IGNORED_FILE_PATTERNS):
|
||||
return True
|
||||
return len(rel.parts) >= 2 and rel.parts[0] == "tests" and rel.parts[1].startswith("tmp_")
|
||||
|
||||
|
||||
@@ -56,6 +87,16 @@ def load_manifest(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def iter_relevant_files(root: Path) -> list[Path]:
|
||||
files = []
|
||||
for entry in CANONICAL_PATHS:
|
||||
@@ -103,6 +144,111 @@ def quality_signal_points(root: Path, manifest: dict, governance_score: int) ->
|
||||
return points
|
||||
|
||||
|
||||
def script_resource_governance(root: Path, expected_file_count: int) -> dict:
|
||||
trust = load_json(root / "reports" / "security_trust_report.json")
|
||||
architecture = load_json(root / "reports" / "architecture_maintainability.json")
|
||||
python_compat = load_json(root / "reports" / "python_compatibility.json")
|
||||
trust_summary = trust.get("summary", {}) if isinstance(trust.get("summary", {}), dict) else {}
|
||||
architecture_summary = architecture.get("summary", {}) if isinstance(architecture.get("summary", {}), dict) else {}
|
||||
python_summary = python_compat.get("summary", {}) if isinstance(python_compat.get("summary", {}), dict) else {}
|
||||
|
||||
reasons = []
|
||||
missing = []
|
||||
checks = [
|
||||
(
|
||||
"trust report covers scripts",
|
||||
trust.get("ok") is True
|
||||
and int(trust_summary.get("script_count", 0) or 0) >= expected_file_count
|
||||
and int(trust_summary.get("secret_findings", 0) or 0) == 0
|
||||
and int(trust_summary.get("help_smoke_failed_count", 0) or 0) == 0,
|
||||
),
|
||||
(
|
||||
"architecture report has no script hotspots or blockers",
|
||||
architecture.get("ok") is True
|
||||
and int(architecture_summary.get("hotspot_count", 0) or 0) == 0
|
||||
and int(architecture_summary.get("blocker_count", 0) or 0) == 0,
|
||||
),
|
||||
(
|
||||
"Python compatibility report has no issues",
|
||||
python_compat.get("ok") is True
|
||||
and int(python_summary.get("issue_count", 0) or 0) == 0,
|
||||
),
|
||||
]
|
||||
for label, ok in checks:
|
||||
if ok:
|
||||
reasons.append(label)
|
||||
else:
|
||||
missing.append(label)
|
||||
return {
|
||||
"status": "governed" if not missing else "needs-review",
|
||||
"evidence": list(SCRIPT_GOVERNANCE_REPORTS),
|
||||
"reasons": reasons,
|
||||
"missing": missing,
|
||||
}
|
||||
|
||||
|
||||
def deferred_dir_governance(
|
||||
root: Path,
|
||||
dirname: str,
|
||||
payload: dict,
|
||||
manifest: dict,
|
||||
skill_text: str,
|
||||
) -> dict:
|
||||
file_count = int(payload.get("file_count", 0) or 0)
|
||||
if dirname == "scripts":
|
||||
governance = script_resource_governance(root, file_count)
|
||||
governance.update(
|
||||
{
|
||||
"path": dirname,
|
||||
"estimated_tokens": int(payload.get("estimated_tokens", 0) or 0),
|
||||
"file_count": file_count,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context.",
|
||||
}
|
||||
)
|
||||
return governance
|
||||
|
||||
referenced = explicit_dir_reference(dirname, root / dirname, skill_text, manifest)
|
||||
declared = dirname in (manifest.get("factory_components") or [])
|
||||
governed = referenced or declared
|
||||
return {
|
||||
"path": dirname,
|
||||
"status": "governed" if governed else "needs-review",
|
||||
"estimated_tokens": int(payload.get("estimated_tokens", 0) or 0),
|
||||
"file_count": file_count,
|
||||
"evidence": ["SKILL.md", "manifest.json"],
|
||||
"reasons": ["directory is explicitly referenced or declared as a factory component"] if governed else [],
|
||||
"missing": [] if governed else ["directory is not referenced in SKILL.md or manifest factory_components"],
|
||||
"rationale": "Deferred resources are acceptable when they are discoverable and intentionally part of the package contract.",
|
||||
}
|
||||
|
||||
|
||||
def deferred_resource_governance(
|
||||
root: Path,
|
||||
manifest: dict,
|
||||
skill_text: str,
|
||||
deferred_resource_dirs: dict[str, dict[str, int | str]],
|
||||
large_deferred_resource_dirs: list[dict],
|
||||
) -> dict:
|
||||
governed_dirs = [
|
||||
deferred_dir_governance(root, str(item["path"]), item, manifest, skill_text)
|
||||
for item in large_deferred_resource_dirs
|
||||
]
|
||||
missing = [item for item in governed_dirs if item["status"] != "governed"]
|
||||
return {
|
||||
"status": "governed" if governed_dirs and not missing else ("not-required" if not governed_dirs else "needs-review"),
|
||||
"large_dir_count": len(governed_dirs),
|
||||
"governed_large_dir_count": len(governed_dirs) - len(missing),
|
||||
"directories": governed_dirs,
|
||||
"summary": (
|
||||
"Large deferred resources are indexed and backed by evidence."
|
||||
if governed_dirs and not missing
|
||||
else "No large deferred resource directory exceeds the per-dir threshold."
|
||||
if not governed_dirs
|
||||
else "One or more large deferred resource directories still need explicit governance evidence."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def analyze_skill(
|
||||
root: Path,
|
||||
max_initial_tokens: int | None = None,
|
||||
@@ -122,6 +268,8 @@ def analyze_skill(
|
||||
other_tokens = 0
|
||||
initial_load_tokens = 0
|
||||
total_text_tokens = 0
|
||||
deferred_resource_tokens = 0
|
||||
deferred_resource_dirs: dict[str, dict[str, int | str]] = {}
|
||||
for path in files:
|
||||
if path.suffix and path.suffix not in TEXT_EXTS and path.name != "SKILL.md":
|
||||
continue
|
||||
@@ -129,6 +277,7 @@ def analyze_skill(
|
||||
tokens = estimate_tokens(text)
|
||||
total_text_tokens += tokens
|
||||
rel = path.relative_to(root)
|
||||
top_dir = rel.parts[0] if rel.parts else str(rel)
|
||||
if rel == Path("SKILL.md"):
|
||||
skill_body_tokens += tokens
|
||||
initial_load_tokens += tokens
|
||||
@@ -136,6 +285,14 @@ def analyze_skill(
|
||||
other_tokens += tokens
|
||||
if rel.parts[0] in {"agents"}:
|
||||
initial_load_tokens += tokens
|
||||
if top_dir in DEFERRED_RESOURCE_DIRS:
|
||||
deferred_resource_tokens += tokens
|
||||
current = deferred_resource_dirs.setdefault(
|
||||
top_dir,
|
||||
{"path": top_dir, "estimated_tokens": 0, "file_count": 0},
|
||||
)
|
||||
current["estimated_tokens"] = int(current["estimated_tokens"]) + tokens
|
||||
current["file_count"] = int(current["file_count"]) + 1
|
||||
|
||||
budget_tier = budget_tier_for(manifest)
|
||||
budget_limit = max_initial_tokens if max_initial_tokens is not None else CONTEXT_BUDGETS[budget_tier]
|
||||
@@ -168,6 +325,30 @@ def analyze_skill(
|
||||
if other_tokens and skill_body_tokens / (skill_body_tokens + other_tokens) > 0.75:
|
||||
warnings.append("Most text still lives in SKILL.md; consider moving detail into references/ or scripts/.")
|
||||
|
||||
large_deferred_resource_dirs = [
|
||||
item
|
||||
for item in sorted(
|
||||
deferred_resource_dirs.values(),
|
||||
key=lambda payload: int(payload["estimated_tokens"]),
|
||||
reverse=True,
|
||||
)
|
||||
if int(item["estimated_tokens"]) > DEFERRED_RESOURCE_WARN_DIR_TOKENS
|
||||
]
|
||||
deferred_governance = deferred_resource_governance(
|
||||
root,
|
||||
manifest,
|
||||
skill_text,
|
||||
deferred_resource_dirs,
|
||||
large_deferred_resource_dirs,
|
||||
)
|
||||
|
||||
if deferred_resource_tokens > DEFERRED_RESOURCE_WARN_TOKENS and deferred_governance["status"] != "governed":
|
||||
warnings.append(
|
||||
"Deferred resource footprint is high: "
|
||||
f"{deferred_resource_tokens} estimated tokens across references/scripts/evals. "
|
||||
"Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified."
|
||||
)
|
||||
|
||||
frontmatter = read_frontmatter(skill_md)
|
||||
governance_score, _ = compute_score(root, manifest, frontmatter, skill_text, bool(manifest))
|
||||
signal_points = quality_signal_points(root, manifest, governance_score)
|
||||
@@ -184,6 +365,15 @@ def analyze_skill(
|
||||
"other_text_tokens": other_tokens,
|
||||
"estimated_initial_load_tokens": initial_load_tokens,
|
||||
"estimated_total_text_tokens": total_text_tokens,
|
||||
"deferred_resource_tokens": deferred_resource_tokens,
|
||||
"deferred_resource_warn_threshold": DEFERRED_RESOURCE_WARN_TOKENS,
|
||||
"deferred_resource_dirs": sorted(
|
||||
deferred_resource_dirs.values(),
|
||||
key=lambda payload: int(payload["estimated_tokens"]),
|
||||
reverse=True,
|
||||
),
|
||||
"large_deferred_resource_dirs": large_deferred_resource_dirs,
|
||||
"deferred_resource_governance": deferred_governance,
|
||||
"relevant_file_count": len(files),
|
||||
"unused_resource_dirs": unused_resource_dirs,
|
||||
"quality_signal_points": signal_points,
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by review_studio_actions.py to keep world-class evidence action cards out of generic Review Studio action wiring."
|
||||
|
||||
|
||||
def action_command_rows(commands: dict[str, Any]) -> list[dict[str, str]]:
|
||||
command_labels = {
|
||||
"prepare_submission": "准备提交",
|
||||
"validate_intake": "校验入口",
|
||||
"submission_review": "审查提交",
|
||||
"refresh_ledger": "刷新台账",
|
||||
"guard_claim": "声明守卫",
|
||||
}
|
||||
rows = []
|
||||
for key in ("prepare_submission", "validate_intake", "submission_review", "refresh_ledger", "guard_claim"):
|
||||
command = str(commands.get(key, "")).strip()
|
||||
if command:
|
||||
rows.append({"key": key, "label": command_labels.get(key, key), "command": command})
|
||||
return rows
|
||||
|
||||
|
||||
def first_text_items(*values: Any, limit: int = 4) -> list[str]:
|
||||
for value in values:
|
||||
if isinstance(value, list) and value:
|
||||
return [str(item) for item in value[:limit]]
|
||||
return []
|
||||
|
||||
|
||||
def artifact_role_rows(contract: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
rows: list[dict[str, Any]] = []
|
||||
roles = contract.get("roles", []) if isinstance(contract.get("roles", []), list) else []
|
||||
for role in roles:
|
||||
if not isinstance(role, dict):
|
||||
continue
|
||||
role_name = str(role.get("role", "")).strip()
|
||||
if role_name == "submission-ref":
|
||||
ready_count = int(contract.get("submission_ref_ready_count", 0) or 0)
|
||||
total_count = int(contract.get("submission_ref_total_count", 0) or 0)
|
||||
elif role_name == "supporting-evidence":
|
||||
ready_count = int(contract.get("supporting_evidence_ready_count", 0) or 0)
|
||||
total_count = int(contract.get("supporting_evidence_total_count", 0) or 0)
|
||||
else:
|
||||
ready_count = 0
|
||||
total_count = 0
|
||||
rows.append(
|
||||
{
|
||||
"role": role_name,
|
||||
"label": str(role.get("label", role_name)),
|
||||
"ready_count": ready_count,
|
||||
"total_count": total_count,
|
||||
"copy_to_artifact_refs": role.get("copy_to_artifact_refs") is True,
|
||||
"description": str(role.get("description", "")),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def compact_artifact_role_contract(contract: dict[str, Any]) -> dict[str, Any]:
|
||||
if not isinstance(contract, dict) or not contract:
|
||||
return {}
|
||||
return {
|
||||
"role_source": str(contract.get("role_source", "")),
|
||||
"counts_as_evidence": contract.get("counts_as_evidence") is True,
|
||||
"artifact_prefill_counts_as_evidence": contract.get("artifact_prefill_counts_as_evidence") is True,
|
||||
"submission_ref_ready_count": int(contract.get("submission_ref_ready_count", 0) or 0),
|
||||
"submission_ref_total_count": int(contract.get("submission_ref_total_count", 0) or 0),
|
||||
"supporting_evidence_ready_count": int(contract.get("supporting_evidence_ready_count", 0) or 0),
|
||||
"supporting_evidence_total_count": int(contract.get("supporting_evidence_total_count", 0) or 0),
|
||||
"roles": artifact_role_rows(contract),
|
||||
}
|
||||
|
||||
|
||||
def compact_phase_queue(rows: list[Any]) -> list[dict[str, Any]]:
|
||||
compact_rows: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
compact_rows.append(
|
||||
{
|
||||
"phase": str(row.get("phase", "")),
|
||||
"label": str(row.get("label", "")),
|
||||
"priority": int(row.get("priority", 90) or 90),
|
||||
"status": str(row.get("status", "blocked")),
|
||||
"blocked_count": int(row.get("blocked_count", 0) or 0),
|
||||
"row_count": int(row.get("row_count", 0) or 0),
|
||||
"owners": [str(item) for item in row.get("owners", []) if str(item).strip()],
|
||||
"evidence_keys": [str(item) for item in row.get("evidence_keys", []) if str(item).strip()],
|
||||
"next_action_id": str(row.get("next_action_id", "")),
|
||||
"next_action": str(row.get("next_action", "")),
|
||||
"verification_command": str(row.get("verification_command", "")),
|
||||
"counts_as_completion": row.get("counts_as_completion") is True,
|
||||
}
|
||||
)
|
||||
return compact_rows
|
||||
|
||||
|
||||
def world_class_action_steps(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
ledger = data.get("world_class_evidence_ledger", {}) if isinstance(data, dict) else {}
|
||||
entries = ledger.get("entries", []) if isinstance(ledger, dict) else []
|
||||
intake = data.get("world_class_evidence_intake", {}) if isinstance(data, dict) else {}
|
||||
intake_items = intake.get("operator_checklist", []) if isinstance(intake, dict) else []
|
||||
intake_by_key = {
|
||||
str(item.get("evidence_key", "")): item
|
||||
for item in intake_items
|
||||
if isinstance(item, dict) and str(item.get("evidence_key", "")).strip()
|
||||
}
|
||||
preflight = data.get("world_class_evidence_preflight", {}) if isinstance(data, dict) else {}
|
||||
preflight_items = preflight.get("items", []) if isinstance(preflight, dict) else []
|
||||
preflight_by_key = {
|
||||
str(item.get("evidence_key", "")): item
|
||||
for item in preflight_items
|
||||
if isinstance(item, dict) and str(item.get("evidence_key", "")).strip()
|
||||
}
|
||||
steps = []
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
key = str(entry.get("key", "")).strip()
|
||||
if not key:
|
||||
continue
|
||||
checklist = intake_by_key.get(key, {})
|
||||
preflight_item = preflight_by_key.get(key, {})
|
||||
submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {}
|
||||
source_checks = entry.get("source_checklist", []) if isinstance(entry.get("source_checklist", []), list) else []
|
||||
blocked_checks = [
|
||||
{
|
||||
"label": str(row.get("label", "")),
|
||||
"field": str(row.get("field", "")),
|
||||
"actual": row.get("actual", ""),
|
||||
"expected": str(row.get("expected", "")),
|
||||
"status": str(row.get("status", "blocked")),
|
||||
"next_action": str(row.get("next_action", "")),
|
||||
}
|
||||
for row in source_checks
|
||||
if isinstance(row, dict) and str(row.get("status", "")) != "pass"
|
||||
]
|
||||
repair_rows = []
|
||||
for row in preflight_item.get("repair_checklist", []) if isinstance(preflight_item, dict) else []:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
repair_rows.append(
|
||||
{
|
||||
"action_id": str(row.get("action_id", "")),
|
||||
"repair_type": str(row.get("repair_type", "")),
|
||||
"target": str(row.get("target", "")),
|
||||
"phase": str(row.get("phase", "")),
|
||||
"priority": int(row.get("priority", 90) or 90),
|
||||
"owner": str(row.get("owner", "")),
|
||||
"status": str(row.get("status", "blocked")),
|
||||
"blocking_reason": str(row.get("blocking_reason", "")),
|
||||
"next_action": str(row.get("next_action", "")),
|
||||
"verification_command": str(row.get("verification_command", "")),
|
||||
"counts_as_completion": row.get("counts_as_completion") is True,
|
||||
}
|
||||
)
|
||||
derived_pass_count = sum(1 for row in source_checks if isinstance(row, dict) and str(row.get("status", "")) == "pass")
|
||||
runbook = entry.get("runbook", [])
|
||||
if not runbook and isinstance(checklist.get("must_collect", {}), dict):
|
||||
runbook = checklist["must_collect"].get("runbook", [])
|
||||
must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {}
|
||||
submission_kit = (
|
||||
preflight_item.get("submission_kit", {})
|
||||
if isinstance(preflight_item.get("submission_kit", {}), dict)
|
||||
else {}
|
||||
)
|
||||
phase_queue = compact_phase_queue(
|
||||
preflight_item.get("phase_queue", []) if isinstance(preflight_item.get("phase_queue", []), list) else []
|
||||
)
|
||||
steps.append(
|
||||
{
|
||||
"key": key,
|
||||
"label": str(entry.get("label", key)),
|
||||
"category": str(entry.get("category", "")),
|
||||
"status": str(entry.get("status", "pending")),
|
||||
"readiness": str(checklist.get("readiness", "")),
|
||||
"current": str(entry.get("current", "")),
|
||||
"next_action": str(entry.get("next_action") or checklist.get("next_action", "")),
|
||||
"submission_path": str(submission.get("path") or checklist.get("submission_path", "")),
|
||||
"template_path": str(checklist.get("template_path", "")),
|
||||
"source_pass_count": int(entry.get("source_pass_count", 0) or derived_pass_count),
|
||||
"source_blocked_count": int(entry.get("source_blocked_count", 0) or len(blocked_checks)),
|
||||
"blocked_checks": blocked_checks,
|
||||
"repair_rows": repair_rows,
|
||||
"repair_blocked_count": sum(1 for row in repair_rows if row.get("status") != "ready"),
|
||||
"repair_counts_as_completion": any(row.get("counts_as_completion") for row in repair_rows),
|
||||
"phase_queue": phase_queue,
|
||||
"phase_queue_blocked_count": sum(1 for row in phase_queue if row.get("status") != "ready"),
|
||||
"phase_queue_counts_as_completion": any(row.get("counts_as_completion") for row in phase_queue),
|
||||
"commands": action_command_rows(checklist.get("commands", {}) if isinstance(checklist.get("commands", {}), dict) else {}),
|
||||
"runbook": [str(item) for item in runbook[:3]],
|
||||
"provenance_requirements": first_text_items(
|
||||
must_collect.get("provenance_requirements"),
|
||||
entry.get("provenance_requirements"),
|
||||
limit=3,
|
||||
),
|
||||
"success_checks": first_text_items(
|
||||
must_collect.get("success_checks"),
|
||||
entry.get("success_checks"),
|
||||
limit=9,
|
||||
),
|
||||
"evidence_artifacts": first_text_items(
|
||||
must_collect.get("evidence_artifacts"),
|
||||
entry.get("evidence_artifacts"),
|
||||
limit=5,
|
||||
),
|
||||
"artifact_role_contract": compact_artifact_role_contract(
|
||||
submission_kit.get("artifact_role_contract", {})
|
||||
if isinstance(submission_kit.get("artifact_role_contract", {}), dict)
|
||||
else {}
|
||||
),
|
||||
"privacy_contract": first_text_items(
|
||||
must_collect.get("privacy_contract"),
|
||||
entry.get("privacy_contract"),
|
||||
limit=4,
|
||||
),
|
||||
}
|
||||
)
|
||||
return steps
|
||||
|
||||
|
||||
def render_small_list(items: list[Any], empty: str, ordered: bool = False) -> str:
|
||||
if not items:
|
||||
return f"<p class='muted'>{html.escape(empty)}</p>"
|
||||
tag = "ol" if ordered else "ul"
|
||||
rows = "".join(f"<li>{html.escape(str(item))}</li>" for item in items)
|
||||
return f"<{tag}>{rows}</{tag}>"
|
||||
|
||||
|
||||
def render_artifact_role_contract(contract: dict[str, Any]) -> str:
|
||||
roles = contract.get("roles", []) if isinstance(contract, dict) else []
|
||||
if not roles:
|
||||
return "<p class='muted'>暂无资产角色。</p>"
|
||||
rows = []
|
||||
for role in roles:
|
||||
rows.append(
|
||||
"<li>"
|
||||
f"<strong>{html.escape(str(role.get('role', '')))}</strong>"
|
||||
f"<span>{html.escape(str(role.get('ready_count', 0)))} / {html.escape(str(role.get('total_count', 0)))} ready</span>"
|
||||
f"<code>artifact_refs: {html.escape(str(role.get('copy_to_artifact_refs') is True).lower())}</code>"
|
||||
f"<small>{html.escape(str(role.get('description', '')))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
source = html.escape(str(contract.get("role_source", "")))
|
||||
counts = html.escape(str(contract.get("counts_as_evidence") is True).lower())
|
||||
prefill = html.escape(str(contract.get("artifact_prefill_counts_as_evidence") is True).lower())
|
||||
return (
|
||||
f"<p class='muted'>source: <code>{source}</code>; counts as evidence: <code>{counts}</code>; "
|
||||
f"prefill counts as evidence: <code>{prefill}</code></p>"
|
||||
"<ul class='action-artifact-roles'>"
|
||||
+ "".join(rows)
|
||||
+ "</ul>"
|
||||
)
|
||||
|
||||
|
||||
def render_action_evidence_steps(steps: list[dict[str, Any]]) -> str:
|
||||
if not steps:
|
||||
return ""
|
||||
cards = []
|
||||
for step in steps:
|
||||
blocked_checks = []
|
||||
for check in step.get("blocked_checks", []):
|
||||
blocked_checks.append(
|
||||
"<li class='action-evidence-check "
|
||||
+ html.escape(str(check.get("status", "blocked")))
|
||||
+ "'>"
|
||||
f"<span>{html.escape(str(check.get('label', '')))}</span>"
|
||||
f"<code>{html.escape(str(check.get('field', '')))}: {html.escape(str(check.get('actual', '')))} / {html.escape(str(check.get('expected', '')))}</code>"
|
||||
f"<small>{html.escape(str(check.get('next_action', '')))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
command_rows = []
|
||||
for command in step.get("commands", [])[:4]:
|
||||
command_rows.append(
|
||||
"<li>"
|
||||
f"<span>{html.escape(str(command.get('label', '')))}</span>"
|
||||
f"<code>{html.escape(str(command.get('command', '')))}</code>"
|
||||
"</li>"
|
||||
)
|
||||
runbook_rows = [f"<li>{html.escape(str(item))}</li>" for item in step.get("runbook", [])]
|
||||
repair_rows = []
|
||||
for row in step.get("repair_rows", []):
|
||||
repair_rows.append(
|
||||
"<li class='action-repair-row "
|
||||
+ html.escape(str(row.get("status", "blocked")))
|
||||
+ "'>"
|
||||
f"<span>#{html.escape(str(row.get('priority', '')))} · {html.escape(str(row.get('phase', '')))} · {html.escape(str(row.get('repair_type', '')))}</span>"
|
||||
f"<strong>{html.escape(str(row.get('target', '')))}</strong>"
|
||||
f"<em>{html.escape(str(row.get('owner', '')))}</em>"
|
||||
f"<code>{html.escape(str(row.get('blocking_reason', '')))}</code>"
|
||||
f"<small>{html.escape(str(row.get('next_action', '')))}</small>"
|
||||
f"<code>{html.escape(str(row.get('verification_command', '')))}</code>"
|
||||
"</li>"
|
||||
)
|
||||
phase_rows = []
|
||||
for row in step.get("phase_queue", []):
|
||||
phase_rows.append(
|
||||
"<li class='action-phase-row "
|
||||
+ html.escape(str(row.get("status", "blocked")))
|
||||
+ "'>"
|
||||
f"<span>#{html.escape(str(row.get('priority', '')))} · {html.escape(str(row.get('phase', '')))}</span>"
|
||||
f"<strong>{html.escape(str(row.get('label', '')))}</strong>"
|
||||
f"<em>{html.escape(str(row.get('blocked_count', 0)))} / {html.escape(str(row.get('row_count', 0)))} blocked</em>"
|
||||
f"<code>owners: {html.escape(', '.join(str(item) for item in row.get('owners', [])) or 'n/a')}</code>"
|
||||
f"<code>evidence: {html.escape(', '.join(str(item) for item in row.get('evidence_keys', [])) or 'n/a')}</code>"
|
||||
f"<small>{html.escape(str(row.get('next_action', '')))}</small>"
|
||||
f"<code>{html.escape(str(row.get('verification_command', '')))}</code>"
|
||||
"</li>"
|
||||
)
|
||||
cards.append(
|
||||
"<article class='action-evidence-item "
|
||||
+ html.escape(str(step.get("status", "pending")))
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(str(step.get('status', 'pending')))} · {html.escape(str(step.get('category', '')))}</span>"
|
||||
f"<h4>{html.escape(str(step.get('label', step.get('key', 'evidence'))))}</h4></div>"
|
||||
f"<p>{html.escape(str(step.get('current', '')))}</p>"
|
||||
f"<dl><dt>提交</dt><dd><code>{html.escape(str(step.get('submission_path', '')))}</code></dd>"
|
||||
f"<dt>模板</dt><dd><code>{html.escape(str(step.get('template_path', '')))}</code></dd>"
|
||||
f"<dt>阻断</dt><dd>{html.escape(str(step.get('source_blocked_count', 0)))} blocked / {html.escape(str(step.get('source_pass_count', 0)))} pass</dd>"
|
||||
f"<dt>修复</dt><dd>{html.escape(str(step.get('repair_blocked_count', 0)))} repair rows; counts as completion: {html.escape(str(step.get('repair_counts_as_completion') is True).lower())}</dd>"
|
||||
f"<dt>阶段</dt><dd>{html.escape(str(step.get('phase_queue_blocked_count', 0)))} blocked phases; counts as completion: {html.escape(str(step.get('phase_queue_counts_as_completion') is True).lower())}</dd>"
|
||||
f"<dt>下一步</dt><dd>{html.escape(str(step.get('next_action', '')))}</dd></dl>"
|
||||
"<section><h5>阻断检查</h5>"
|
||||
+ (
|
||||
"<ul class='action-evidence-checks'>" + "".join(blocked_checks) + "</ul>"
|
||||
if blocked_checks
|
||||
else "<p class='muted'>暂无阻断检查。</p>"
|
||||
)
|
||||
+ "</section>"
|
||||
"<details class='action-phase-details'><summary>阶段队列</summary>"
|
||||
+ (
|
||||
"<ul class='action-phase-list'>" + "".join(phase_rows) + "</ul>"
|
||||
if phase_rows
|
||||
else "<p class='muted'>暂无阶段队列。</p>"
|
||||
)
|
||||
+ "</details>"
|
||||
"<details class='action-repair-details'><summary>修复清单</summary>"
|
||||
+ (
|
||||
"<ul class='action-repair-list'>" + "".join(repair_rows) + "</ul>"
|
||||
if repair_rows
|
||||
else "<p class='muted'>暂无修复项。</p>"
|
||||
)
|
||||
+ "</details>"
|
||||
"<details class='action-command-details'><summary>操作命令</summary>"
|
||||
+ (
|
||||
"<ul class='action-command-list'>" + "".join(command_rows) + "</ul>"
|
||||
if command_rows
|
||||
else "<p class='muted'>暂无操作命令。</p>"
|
||||
)
|
||||
+ "</details>"
|
||||
"<details class='action-runbook-details'><summary>首要步骤</summary>"
|
||||
+ (
|
||||
"<ol class='action-runbook-list'>" + "".join(runbook_rows) + "</ol>"
|
||||
if runbook_rows
|
||||
else "<p class='muted'>暂无首要步骤。</p>"
|
||||
)
|
||||
+ "</details>"
|
||||
"<details class='action-collection-details'><summary>采集契约</summary>"
|
||||
"<div class='action-collection-grid'>"
|
||||
"<section><h5>来源要求</h5>"
|
||||
+ render_small_list(step.get("provenance_requirements", []), "暂无来源要求。")
|
||||
+ "</section>"
|
||||
"<section><h5>通过条件</h5>"
|
||||
+ render_small_list(step.get("success_checks", []), "暂无通过条件。")
|
||||
+ "</section>"
|
||||
"<section><h5>证据资产</h5>"
|
||||
+ render_small_list(step.get("evidence_artifacts", []), "暂无证据资产。")
|
||||
+ "</section>"
|
||||
"<section><h5>资产角色</h5>"
|
||||
+ render_artifact_role_contract(step.get("artifact_role_contract", {}))
|
||||
+ "</section>"
|
||||
"<section><h5>隐私边界</h5>"
|
||||
+ render_small_list(step.get("privacy_contract", []), "暂无隐私边界。")
|
||||
+ "</section>"
|
||||
"</div>"
|
||||
"</details>"
|
||||
"</article>"
|
||||
)
|
||||
return (
|
||||
"<section class='action-evidence-panel'>"
|
||||
"<h4>证据采集</h4>"
|
||||
"<p>以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。</p>"
|
||||
"<div class='action-evidence-grid'>"
|
||||
+ "".join(cards)
|
||||
+ "</div></section>"
|
||||
)
|
||||
@@ -0,0 +1,374 @@
|
||||
import html
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from review_studio_action_evidence import render_action_evidence_steps, world_class_action_steps
|
||||
from review_studio_gates import status_label
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep Review Studio action guidance and source refs out of HTML rendering."
|
||||
|
||||
|
||||
def link_from(output_html: Path, target: Path) -> str:
|
||||
return os.path.relpath(target.resolve(), output_html.parent.resolve())
|
||||
|
||||
|
||||
def compact_excerpt(text: str, limit: int = 180) -> str:
|
||||
normalized = " ".join(text.strip().split())
|
||||
if len(normalized) <= limit:
|
||||
return normalized
|
||||
return normalized[: limit - 1].rstrip() + "…"
|
||||
|
||||
|
||||
def find_line_anchor(path: Path, patterns: list[str] | None = None) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"line": None, "matched_pattern": "", "excerpt": ""}
|
||||
if not patterns:
|
||||
return {"line": 1, "matched_pattern": "", "excerpt": ""}
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError:
|
||||
return {"line": 1, "matched_pattern": "", "excerpt": ""}
|
||||
for pattern in patterns:
|
||||
for index, line in enumerate(lines, start=1):
|
||||
if pattern in line:
|
||||
return {
|
||||
"line": index,
|
||||
"matched_pattern": pattern,
|
||||
"excerpt": compact_excerpt(line),
|
||||
}
|
||||
first_line = compact_excerpt(lines[0]) if lines else ""
|
||||
return {"line": 1, "matched_pattern": "", "excerpt": first_line}
|
||||
|
||||
|
||||
def find_line(path: Path, patterns: list[str] | None = None) -> int | None:
|
||||
return find_line_anchor(path, patterns).get("line")
|
||||
|
||||
|
||||
def source_refs(
|
||||
skill_dir: Path,
|
||||
output_html: Path,
|
||||
specs: list[dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
refs: list[dict[str, Any]] = []
|
||||
for spec in specs:
|
||||
rel_path = str(spec.get("path", "")).strip()
|
||||
if not rel_path:
|
||||
continue
|
||||
path = skill_dir / rel_path
|
||||
exists = path.exists()
|
||||
anchor = find_line_anchor(path, spec.get("patterns", []))
|
||||
refs.append(
|
||||
{
|
||||
"path": rel_path,
|
||||
"label": str(spec.get("label", rel_path)),
|
||||
"kind": str(spec.get("kind", "source")),
|
||||
"line": anchor.get("line"),
|
||||
"matched_pattern": anchor.get("matched_pattern", ""),
|
||||
"excerpt": anchor.get("excerpt", ""),
|
||||
"exists": exists,
|
||||
"link": link_from(output_html, path) if exists else "",
|
||||
}
|
||||
)
|
||||
return refs
|
||||
|
||||
|
||||
ACTION_GUIDANCE: dict[str, dict[str, Any]] = {
|
||||
"intent-canvas": {
|
||||
"summary": "收紧真实任务、输入、输出、排除项和成功标准。",
|
||||
"why": "低 intent confidence 会让后续 Skill IR、输出评测和 Review Studio 结论建立在模糊意图上。",
|
||||
"source_fix": "reports/intent-dialogue.md + reports/intent-confidence.md",
|
||||
"source_paths": [
|
||||
{"path": "reports/intent-dialogue.md", "label": "intent dialogue", "kind": "report", "patterns": ["# Intent"]},
|
||||
{"path": "reports/intent-confidence.md", "label": "intent confidence", "kind": "report", "patterns": ["# Intent"]},
|
||||
],
|
||||
"verification": "python3 scripts/yao.py intent-confidence .",
|
||||
},
|
||||
"trigger-lab": {
|
||||
"summary": "修正 route scorecard 中的误触发、漏触发或 ambiguous case。",
|
||||
"why": "触发错误会让正确 Skill 失活,或让相邻 Skill 被错误调用。",
|
||||
"source_fix": "SKILL.md frontmatter description + evals/*/trigger_cases.json",
|
||||
"source_paths": [
|
||||
{"path": "SKILL.md", "label": "frontmatter description", "kind": "source", "patterns": ["description:"]},
|
||||
{"path": "evals/trigger_cases.json", "label": "trigger eval cases", "kind": "eval", "patterns": ["should_trigger"]},
|
||||
{"path": "reports/route_scorecard.md", "label": "route scorecard", "kind": "report", "patterns": ["# Route"]},
|
||||
],
|
||||
"verification": "python3 scripts/build_confusion_matrix.py",
|
||||
},
|
||||
"output-lab": {
|
||||
"summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。同步补足盲审声明。",
|
||||
"why": "没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md + reports/output_review_decisions.json",
|
||||
"source_paths": [
|
||||
{"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ['"case_id"', '"id"']},
|
||||
{"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]},
|
||||
{"path": "reports/output_execution_runs.md", "label": "output execution runs", "kind": "report", "patterns": ["# Output Execution"]},
|
||||
{"path": "reports/output_blind_review_pack.md", "label": "blind A/B review pack", "kind": "report", "patterns": ["# Output Blind"]},
|
||||
{"path": "reports/output_review_kit.html", "label": "reviewer cockpit", "kind": "report", "patterns": ["Output Review Kit", "Variant A"]},
|
||||
{"path": "reports/output_review_adjudication.md", "label": "review adjudication", "kind": "report", "patterns": ["# Output Review"]},
|
||||
],
|
||||
"verification": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review",
|
||||
},
|
||||
"context-budget": {
|
||||
"summary": "压缩或拆分高成本 deferred resources,保留最小可路由上下文。",
|
||||
"why": "初始加载可以安全,但后续 references、scripts、evals 体量过大时,reviewer 仍需要看到维护和读取成本。",
|
||||
"source_fix": "SKILL.md + references/ + scripts/ + evals/",
|
||||
"source_paths": [
|
||||
{"path": "SKILL.md", "label": "entrypoint", "kind": "source", "patterns": ["# Yao Meta Skill"]},
|
||||
{"path": "reports/context_budget.md", "label": "context budget", "kind": "report", "patterns": ["# Context"]},
|
||||
{"path": "scripts/resource_boundary_check.py", "label": "resource boundary checker", "kind": "source", "patterns": ["DEFERRED_RESOURCE"]},
|
||||
{"path": "references/skill-engineering-method.md", "label": "skill engineering method", "kind": "method", "patterns": ["Design Principle"]},
|
||||
],
|
||||
"verification": "python3 scripts/render_context_reports.py",
|
||||
},
|
||||
"runtime-matrix": {
|
||||
"summary": "修复目标端结构、metadata、相对路径、fallback 或 adapter target 声明。",
|
||||
"why": "runtime conformance 失败意味着包可能被目标客户端错误加载或静默降级。",
|
||||
"source_fix": "agents/interface.yaml + reports/conformance_matrix.md",
|
||||
"source_paths": [
|
||||
{"path": "agents/interface.yaml", "label": "portable interface", "kind": "source", "patterns": ["adapter_targets"]},
|
||||
{"path": "reports/conformance_matrix.md", "label": "conformance matrix", "kind": "report", "patterns": ["# Runtime"]},
|
||||
],
|
||||
"verification": "python3 scripts/run_conformance_suite.py .",
|
||||
},
|
||||
"trust-report": {
|
||||
"summary": "处理脚本 help surface、依赖 pin、network policy、secret 和权限声明。",
|
||||
"why": "团队分发时,脚本和依赖是主要供应链风险面,warning 必须有明确处置。",
|
||||
"source_fix": "reports/security_trust_report.md + security/*.md + scripts/",
|
||||
"source_paths": [
|
||||
{"path": "reports/security_trust_report.md", "label": "trust report", "kind": "report", "patterns": ["# Security"]},
|
||||
{"path": "security/script_policy.md", "label": "script policy", "kind": "policy", "patterns": ["# Script"]},
|
||||
{"path": "security/network_policy.md", "label": "network policy", "kind": "policy", "patterns": ["# Network"]},
|
||||
],
|
||||
"verification": "python3 scripts/trust_check.py .",
|
||||
},
|
||||
"python-compat": {
|
||||
"summary": "修复 Python 3.11 语法兼容问题,尤其是 f-string 表达式内的反斜杠转义。",
|
||||
"why": "目标运行环境可能仍停留在 Python 3.11,语法漂移会让 CLI、报告生成和 CI 在发布后直接失败。",
|
||||
"source_fix": "reports/python_compatibility.md + scripts/*.py + tests/*.py",
|
||||
"source_paths": [
|
||||
{"path": "reports/python_compatibility.md", "label": "Python compatibility", "kind": "report", "patterns": ["# Python"]},
|
||||
{"path": "scripts/python_compat_check.py", "label": "compatibility checker", "kind": "source", "patterns": ["SCRIPT_INTERFACE"]},
|
||||
{"path": ".github/workflows/test.yml", "label": "CI test workflow", "kind": "ci", "patterns": ["python"]},
|
||||
],
|
||||
"verification": "python3 scripts/yao.py python-compat .",
|
||||
},
|
||||
"architecture-maintainability": {
|
||||
"summary": "处理大文件和 CLI command surface 的维护性热点,优先拆分稳定职责边界。",
|
||||
"why": "Meta Skill 的门禁、报告和 CLI 会持续增长;如果不把架构债纳入审查,后续能力会越来越难验证和迁移。",
|
||||
"source_fix": "reports/architecture_maintainability.md + scripts/yao.py + scripts/render_review_studio.py",
|
||||
"source_paths": [
|
||||
{"path": "reports/architecture_maintainability.md", "label": "architecture maintainability", "kind": "report", "patterns": ["# Architecture"]},
|
||||
{"path": "scripts/yao.py", "label": "Yao CLI orchestrator", "kind": "source", "patterns": ["def command_"]},
|
||||
{"path": "scripts/render_review_studio.py", "label": "Review Studio renderer", "kind": "source", "patterns": ["def render_html"]},
|
||||
{"path": "scripts/review_studio_actions.py", "label": "Review Studio actions", "kind": "source", "patterns": ["ACTION_GUIDANCE"]},
|
||||
{"path": "scripts/render_review_viewer.py", "label": "review viewer renderer", "kind": "source", "patterns": ["def "]},
|
||||
],
|
||||
"verification": "python3 scripts/yao.py architecture-audit .",
|
||||
},
|
||||
"permission-gates": {
|
||||
"summary": "补齐高权限能力的 reviewer、scope、reason、expires_at 和目标端 enforcement 说明。",
|
||||
"why": "权限契约只有在批准人、有效期和目标端处置方式明确时,才能支撑 governed release。",
|
||||
"source_fix": "security/permission_policy.json + security/permission_policy.md",
|
||||
"source_paths": [
|
||||
{"path": "security/permission_policy.json", "label": "permission approvals", "kind": "policy", "patterns": ["approved"]},
|
||||
{"path": "security/permission_policy.md", "label": "permission method", "kind": "policy", "patterns": ["# Permission"]},
|
||||
],
|
||||
"verification": "python3 scripts/trust_check.py .",
|
||||
},
|
||||
"permission-runtime": {
|
||||
"summary": "生成并修复目标包的 runtime permission probe 报告。",
|
||||
"why": "目标端即使只能提供 metadata fallback,也必须明确 native enforcement 缺口、表示位置和 operator note。",
|
||||
"source_fix": "dist/targets/*/adapter.json + reports/runtime_permission_probes.md",
|
||||
"source_paths": [
|
||||
{"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]},
|
||||
{"path": "dist/targets/openai/adapter.json", "label": "OpenAI adapter", "kind": "package", "patterns": ["target_permission_contract"]},
|
||||
{"path": "dist/targets/claude/adapter.json", "label": "Claude adapter", "kind": "package", "patterns": ["target_permission_contract"]},
|
||||
{"path": "dist/targets/generic/adapter.json", "label": "generic adapter", "kind": "package", "patterns": ["target_permission_contract"]},
|
||||
],
|
||||
"verification": "python3 scripts/probe_runtime_permissions.py . --package-dir dist",
|
||||
},
|
||||
"skill-atlas": {
|
||||
"summary": "处理 portfolio 里的路由冲突、owner 缺口、stale skill 和重复能力。",
|
||||
"why": "单个 Skill 质量很高仍可能在团队 skill library 中被相邻 Skill 冲突削弱。",
|
||||
"source_fix": "reports/skill_atlas.html + skill_atlas/catalog.json",
|
||||
"source_paths": [
|
||||
{"path": "skill_atlas/catalog.json", "label": "skill atlas catalog", "kind": "atlas", "patterns": ["summary"]},
|
||||
{"path": "skill_atlas/policy.json", "label": "atlas scope policy", "kind": "policy", "patterns": ["scope"]},
|
||||
{"path": "reports/skill_atlas.html", "label": "skill atlas report", "kind": "report", "patterns": ["Skill Atlas"]},
|
||||
],
|
||||
"verification": "python3 scripts/build_skill_atlas.py --workspace-root .",
|
||||
},
|
||||
"operations-loop": {
|
||||
"summary": "记录 metadata-only 使用事件,或明确当前 release 缺少真实使用信号。",
|
||||
"why": "没有运营回路时,reviewer 无法判断采用率、误触发、坏输出和 review overdue 的真实影响。",
|
||||
"source_fix": "reports/adoption_drift_report.md",
|
||||
"source_paths": [
|
||||
{"path": "reports/adoption_drift_report.md", "label": "adoption drift report", "kind": "report", "patterns": ["# Adoption"]},
|
||||
{"path": "references/telemetry-drift-method.md", "label": "telemetry method", "kind": "method", "patterns": ["# Telemetry"]},
|
||||
],
|
||||
"verification": "python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted",
|
||||
},
|
||||
"review-waivers": {
|
||||
"summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。",
|
||||
"why": "warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。",
|
||||
"source_fix": "reports/review_waivers.md",
|
||||
"source_paths": [
|
||||
{"path": "reports/review_waivers.md", "label": "waiver ledger", "kind": "report", "patterns": ["# Review"]},
|
||||
{"path": "references/review-waiver-method.md", "label": "waiver method", "kind": "method", "patterns": ["# Review"]},
|
||||
],
|
||||
"verification": "python3 scripts/render_review_waivers.py .",
|
||||
},
|
||||
"world-class-evidence": {
|
||||
"summary": "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。",
|
||||
"why": "世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。",
|
||||
"source_fix": "reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md",
|
||||
"source_paths": [
|
||||
{"path": "reports/world_class_evidence_ledger.md", "label": "world-class evidence ledger", "kind": "report", "patterns": ["# World-Class Evidence Ledger"]},
|
||||
{"path": "reports/world_class_evidence_plan.md", "label": "world-class evidence plan", "kind": "report", "patterns": ["# World-Class Evidence Plan"]},
|
||||
{"path": "reports/world_class_evidence_intake.md", "label": "world-class evidence intake", "kind": "report", "patterns": ["# World-Class Evidence Intake"]},
|
||||
{"path": "reports/world_class_evidence_preflight.md", "label": "world-class evidence preflight", "kind": "report", "patterns": ["# World-Class Evidence Preflight"]},
|
||||
{"path": "reports/world_class_evidence_preflight.html", "label": "world-class preflight HTML", "kind": "report", "patterns": ["World-Class Evidence Preflight"]},
|
||||
{"path": "reports/world_class_submission_review.md", "label": "world-class submission review", "kind": "report", "patterns": ["# World-Class Submission Review"]},
|
||||
{"path": "reports/world_class_claim_guard.md", "label": "world-class claim guard", "kind": "report", "patterns": ["# World-Class Claim Guard"]},
|
||||
{"path": "evidence/world_class/intake.schema.json", "label": "evidence intake schema", "kind": "schema", "patterns": ["Yao World-Class Evidence Intake"]},
|
||||
{"path": "evidence/world_class/templates/provider-holdout.intake.json", "label": "provider intake template", "kind": "template", "patterns": ["provider-holdout"]},
|
||||
{"path": "evidence/world_class/templates/human-adjudication.intake.json", "label": "human intake template", "kind": "template", "patterns": ["human-adjudication"]},
|
||||
{"path": "evidence/world_class/templates/native-permission-enforcement.intake.json", "label": "permission intake template", "kind": "template", "patterns": ["native-permission-enforcement"]},
|
||||
{"path": "evidence/world_class/templates/native-client-telemetry.intake.json", "label": "telemetry intake template", "kind": "template", "patterns": ["native-client-telemetry"]},
|
||||
{"path": "reports/skill_os2_audit.md", "label": "Skill OS 2.0 audit", "kind": "report", "patterns": ["# Skill OS"]},
|
||||
{"path": "reports/output_review_decisions.json", "label": "human review decisions", "kind": "report", "patterns": ["winner_variant"]},
|
||||
{"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]},
|
||||
{"path": "reports/adoption_drift_report.md", "label": "adoption drift", "kind": "report", "patterns": ["# Adoption"]},
|
||||
],
|
||||
"verification": (
|
||||
"python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions "
|
||||
"&& python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions "
|
||||
"&& python3 scripts/yao.py review-studio ."
|
||||
),
|
||||
},
|
||||
"registry-audit": {
|
||||
"summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。",
|
||||
"why": "分发元数据不完整时,团队无法安全安装、升级或追溯包体来源。",
|
||||
"source_fix": "registry/ + reports/registry_audit.md",
|
||||
"source_paths": [
|
||||
{"path": "registry/packages/yao-meta-skill.json", "label": "registry package", "kind": "registry", "patterns": ["version"]},
|
||||
{"path": "reports/registry_audit.md", "label": "registry audit", "kind": "report", "patterns": ["# Registry"]},
|
||||
{"path": "reports/install_simulation.md", "label": "install simulation", "kind": "report", "patterns": ["# Install"]},
|
||||
],
|
||||
"verification": "python3 scripts/registry_audit.py .",
|
||||
},
|
||||
"release-notes": {
|
||||
"summary": "确认 promotion、upgrade diff、breaking changes、migration guide 和 known limitations。",
|
||||
"why": "发布说明不完整会让使用者无法判断升级风险和迁移动作。",
|
||||
"source_fix": "reports/upgrade_check.md + docs/migration-v2.md",
|
||||
"source_paths": [
|
||||
{"path": "reports/upgrade_check.md", "label": "upgrade check", "kind": "report", "patterns": ["# Upgrade"]},
|
||||
{"path": "docs/migration-v2.md", "label": "migration guide", "kind": "docs", "patterns": ["# Migration"]},
|
||||
{"path": "reports/promotion_decisions.md", "label": "promotion decisions", "kind": "report", "patterns": ["# Promotion"]},
|
||||
],
|
||||
"verification": "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_review_actions(
|
||||
gates: list[dict[str, str]],
|
||||
skill_dir: Path,
|
||||
output_html: Path,
|
||||
data: dict[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
actions: list[dict[str, Any]] = []
|
||||
for gate_item in gates:
|
||||
if gate_item["status"] == "pass":
|
||||
continue
|
||||
guidance = ACTION_GUIDANCE.get(
|
||||
gate_item["key"],
|
||||
{
|
||||
"summary": "打开证据报告,修复当前 gate 暴露的问题。",
|
||||
"why": "该 gate 未通过,release reviewer 需要明确处置动作。",
|
||||
"source_fix": gate_item.get("evidence", ""),
|
||||
"source_paths": [],
|
||||
"verification": "python3 scripts/render_review_studio.py .",
|
||||
},
|
||||
)
|
||||
refs = source_refs(skill_dir, output_html, guidance.get("source_paths", []))
|
||||
actions.append(
|
||||
{
|
||||
"action_id": f"review-action-{gate_item['key']}",
|
||||
"gate_key": gate_item["key"],
|
||||
"title": gate_item["label"],
|
||||
"label": gate_item["label"],
|
||||
"status": gate_item["status"],
|
||||
"priority": "blocker" if gate_item["status"] == "block" else "warning",
|
||||
"next_step": guidance["summary"],
|
||||
"summary": guidance["summary"],
|
||||
"reason": guidance["why"],
|
||||
"why": guidance["why"],
|
||||
"source_fix": guidance["source_fix"],
|
||||
"source_refs": refs,
|
||||
"evidence": gate_item.get("evidence", ""),
|
||||
"evidence_link": gate_item.get("link", ""),
|
||||
"verification_command": guidance["verification"],
|
||||
"evidence_steps": world_class_action_steps(data or {})
|
||||
if gate_item["key"] == "world-class-evidence"
|
||||
else [],
|
||||
}
|
||||
)
|
||||
return actions
|
||||
|
||||
|
||||
def render_action_source_refs(refs: list[dict[str, Any]]) -> str:
|
||||
if not refs:
|
||||
return "<p class='muted'>暂无结构化 source refs;请先打开证据报告。</p>"
|
||||
items = []
|
||||
for ref in refs:
|
||||
line_suffix = f":{ref['line']}" if ref.get("line") else ""
|
||||
label = f"{ref['path']}{line_suffix}"
|
||||
if ref.get("exists") and ref.get("link"):
|
||||
path_html = f"<a href='{html.escape(ref['link'])}'>{html.escape(label)}</a>"
|
||||
else:
|
||||
path_html = f"<span>{html.escape(label)} · missing</span>"
|
||||
pattern = str(ref.get("matched_pattern", "")).strip()
|
||||
excerpt = str(ref.get("excerpt", "")).strip()
|
||||
anchor_bits = [html.escape(str(ref.get("label", "source"))), html.escape(str(ref.get("kind", "source")))]
|
||||
if pattern:
|
||||
anchor_bits.append("pattern: " + html.escape(pattern))
|
||||
meta_html = " · ".join(anchor_bits)
|
||||
excerpt_html = f"<blockquote>{html.escape(excerpt)}</blockquote>" if excerpt else ""
|
||||
items.append(
|
||||
"<li>"
|
||||
f"{path_html}"
|
||||
f"<small>{meta_html}</small>"
|
||||
f"{excerpt_html}"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='source-ref-list'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def render_review_actions(actions: list[dict[str, Any]]) -> str:
|
||||
if not actions:
|
||||
return "<p class='muted'>当前没有 blocker 或 warning。保持现有证据链即可。</p>"
|
||||
cards = []
|
||||
for item in actions:
|
||||
link_html = f"<a href='{html.escape(item['evidence_link'])}'>打开证据</a>" if item.get("evidence_link") else ""
|
||||
source_refs_html = render_action_source_refs(item.get("source_refs", []))
|
||||
evidence_steps_html = render_action_evidence_steps(item.get("evidence_steps", []))
|
||||
card_class = "action-card " + html.escape(item["status"])
|
||||
if item.get("evidence_steps"):
|
||||
card_class += " with-evidence"
|
||||
cards.append(
|
||||
"<article class='" + card_class + "'>"
|
||||
f"<div><span>{html.escape(status_label(item['status']))}</span><h3>{html.escape(item['label'])}</h3></div>"
|
||||
f"<p>{html.escape(item['summary'])}</p>"
|
||||
f"<small>{html.escape(item['why'])}</small>"
|
||||
f"<dl><dt>修复位置</dt><dd>{html.escape(item['source_fix'])}</dd>"
|
||||
f"<dt>验证命令</dt><dd><code>{html.escape(item['verification_command'])}</code></dd></dl>"
|
||||
f"{evidence_steps_html}"
|
||||
f"{source_refs_html}"
|
||||
f"<footer>{html.escape(item['evidence'])} {link_html}</footer>"
|
||||
"</article>"
|
||||
)
|
||||
return "".join(cards)
|
||||
@@ -0,0 +1,387 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Data loading and insight card helpers for Review Studio."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from skill_ir_paths import find_skill_ir_path as find_skill_ir_path_for_name
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to load Review Studio source reports and metric cards."
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def latest_json_report(directory: Path) -> dict[str, Any]:
|
||||
if not directory.exists():
|
||||
return {}
|
||||
candidates = sorted(path for path in directory.glob("*.json") if path.is_file())
|
||||
if not candidates:
|
||||
return {}
|
||||
payload = load_json(candidates[-1])
|
||||
if payload:
|
||||
try:
|
||||
payload.setdefault("source_path", str(candidates[-1].resolve().relative_to(directory.parent.parent.parent.resolve())))
|
||||
except ValueError:
|
||||
payload.setdefault("source_path", str(candidates[-1]))
|
||||
return payload
|
||||
|
||||
|
||||
def latest_report_path(skill_dir: Path, directory: Path, suffix: str) -> str:
|
||||
if not directory.exists():
|
||||
return ""
|
||||
candidates = sorted(path for path in directory.glob(f"*{suffix}") if path.is_file())
|
||||
if not candidates:
|
||||
return ""
|
||||
try:
|
||||
return str(candidates[-1].resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
return str(candidates[-1])
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict[str, Any]:
|
||||
if not path.exists() or yaml is None:
|
||||
return {}
|
||||
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def parse_frontmatter(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}
|
||||
try:
|
||||
end_index = lines[1:].index("---") + 1
|
||||
except ValueError:
|
||||
return {}
|
||||
text = "\n".join(lines[1:end_index])
|
||||
if yaml is not None:
|
||||
payload = yaml.safe_load(text) or {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
data = {}
|
||||
for line in text.splitlines():
|
||||
if ":" in line:
|
||||
key, value = line.split(":", 1)
|
||||
data[key.strip()] = value.strip().strip('"')
|
||||
return data
|
||||
|
||||
|
||||
def find_skill_ir_path(skill_dir: Path) -> str:
|
||||
frontmatter = parse_frontmatter(skill_dir / "SKILL.md")
|
||||
name = str(frontmatter.get("name") or skill_dir.name)
|
||||
return find_skill_ir_path_for_name(skill_dir, name)
|
||||
|
||||
|
||||
def evidence_paths(skill_dir: Path) -> dict[str, str]:
|
||||
rels = {
|
||||
"skill_overview": "reports/skill-overview.html",
|
||||
"review_viewer": "reports/review-viewer.html",
|
||||
"output_eval": "reports/output_quality_scorecard.md",
|
||||
"output_execution": "reports/output_execution_runs.md",
|
||||
"output_blind_review": "reports/output_blind_review_pack.md",
|
||||
"output_review_kit": "reports/output_review_kit.md",
|
||||
"output_review_kit_html": "reports/output_review_kit.html",
|
||||
"output_review_decisions": "reports/output_review_decisions.json",
|
||||
"output_review_adjudication": "reports/output_review_adjudication.md",
|
||||
"benchmark_reproducibility": "reports/benchmark_reproducibility.md",
|
||||
"skill_os2_coverage": "reports/skill_os2_coverage.md",
|
||||
"runtime_conformance": "reports/conformance_matrix.md",
|
||||
"trust_report": "reports/security_trust_report.md",
|
||||
"python_compatibility": "reports/python_compatibility.md",
|
||||
"architecture_maintainability": "reports/architecture_maintainability.md",
|
||||
"permission_policy": "security/permission_policy.md",
|
||||
"runtime_permissions": "reports/runtime_permission_probes.md",
|
||||
"skill_atlas": "reports/skill_atlas.html",
|
||||
"compiled_targets": "reports/compiled_targets.md",
|
||||
"adoption_drift": "reports/adoption_drift_report.md",
|
||||
"review_waivers": "reports/review_waivers.md",
|
||||
"review_annotations": "reports/review_annotations.md",
|
||||
"adaptation_proposals": "reports/adaptation_proposals.md",
|
||||
"adaptation_approval_ledger": "reports/adaptation_approval_ledger.json",
|
||||
"adaptation_regression": "reports/adaptation_regression_report.md",
|
||||
"world_class_evidence_plan": "reports/world_class_evidence_plan.md",
|
||||
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.md",
|
||||
"world_class_evidence_intake": "reports/world_class_evidence_intake.md",
|
||||
"world_class_evidence_preflight": "reports/world_class_evidence_preflight.md",
|
||||
"world_class_evidence_preflight_html": "reports/world_class_evidence_preflight.html",
|
||||
"world_class_submission_review": "reports/world_class_submission_review.md",
|
||||
"world_class_operator_runbook": "reports/world_class_operator_runbook.md",
|
||||
"world_class_operator_runbook_html": "reports/world_class_operator_runbook.html",
|
||||
"world_class_claim_guard": "reports/world_class_claim_guard.md",
|
||||
"registry_audit": "reports/registry_audit.md",
|
||||
"package_verification": "reports/package_verification.md",
|
||||
"install_simulation": "reports/install_simulation.md",
|
||||
"upgrade_check": "reports/upgrade_check.md",
|
||||
"migration": "docs/migration-v2.md",
|
||||
}
|
||||
skill_ir_path = find_skill_ir_path(skill_dir)
|
||||
if skill_ir_path:
|
||||
rels["skill_ir"] = skill_ir_path
|
||||
daily_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "daily", ".md")
|
||||
weekly_md = latest_report_path(skill_dir, skill_dir / "reports" / "skillops" / "weekly", ".md")
|
||||
if daily_md:
|
||||
rels["daily_skillops"] = daily_md
|
||||
if weekly_md:
|
||||
rels["weekly_curator"] = weekly_md
|
||||
return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()}
|
||||
|
||||
|
||||
def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
reports = skill_dir / "reports"
|
||||
return {
|
||||
"overview": load_json(reports / "skill-overview.json"),
|
||||
"intent_confidence": load_json(reports / "intent-confidence.json"),
|
||||
"intent_dialogue": load_json(reports / "intent-dialogue.json"),
|
||||
"route_scorecard": load_json(reports / "route_scorecard.json"),
|
||||
"output_quality": load_json(reports / "output_quality_scorecard.json"),
|
||||
"output_execution": load_json(reports / "output_execution_runs.json"),
|
||||
"output_blind_review": load_json(reports / "output_blind_review_pack.json"),
|
||||
"output_review_kit": load_json(reports / "output_review_kit.json"),
|
||||
"output_review_adjudication": load_json(reports / "output_review_adjudication.json"),
|
||||
"benchmark_reproducibility": load_json(reports / "benchmark_reproducibility.json"),
|
||||
"skill_os2_coverage": load_json(reports / "skill_os2_coverage.json"),
|
||||
"compiled_targets": load_json(reports / "compiled_targets.json"),
|
||||
"conformance": load_json(reports / "conformance_matrix.json"),
|
||||
"runtime_permissions": load_json(reports / "runtime_permission_probes.json"),
|
||||
"trust": load_json(reports / "security_trust_report.json"),
|
||||
"python_compatibility": load_json(reports / "python_compatibility.json"),
|
||||
"architecture_maintainability": load_json(reports / "architecture_maintainability.json"),
|
||||
"context_budget": load_json(reports / "context_budget.json"),
|
||||
"promotion": load_json(reports / "promotion_decisions.json"),
|
||||
"atlas": load_json(reports / "skill_atlas.json"),
|
||||
"adoption_drift": load_json(reports / "adoption_drift_report.json"),
|
||||
"daily_skillops": latest_json_report(reports / "skillops" / "daily"),
|
||||
"weekly_curator": latest_json_report(reports / "skillops" / "weekly"),
|
||||
"review_waivers": load_json(reports / "review_waivers.json"),
|
||||
"review_annotations": load_json(reports / "review_annotations.json"),
|
||||
"adaptation_proposals": load_json(reports / "adaptation_proposals.json"),
|
||||
"adaptation_approval_ledger": load_json(reports / "adaptation_approval_ledger.json"),
|
||||
"adaptation_regression": load_json(reports / "adaptation_regression_report.json"),
|
||||
"world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"),
|
||||
"world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"),
|
||||
"world_class_evidence_preflight": load_json(reports / "world_class_evidence_preflight.json"),
|
||||
"world_class_submission_review": load_json(reports / "world_class_submission_review.json"),
|
||||
"world_class_operator_runbook": load_json(reports / "world_class_operator_runbook.json"),
|
||||
"world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"),
|
||||
"registry": load_json(reports / "registry_audit.json"),
|
||||
"package_verification": load_json(reports / "package_verification.json"),
|
||||
"install_simulation": load_json(reports / "install_simulation.json"),
|
||||
"upgrade_check": load_json(reports / "upgrade_check.json"),
|
||||
"manifest": load_json(skill_dir / "manifest.json"),
|
||||
"frontmatter": parse_frontmatter(skill_dir / "SKILL.md"),
|
||||
"interface": load_yaml(skill_dir / "agents" / "interface.yaml"),
|
||||
}
|
||||
|
||||
|
||||
def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
overview = data["overview"]
|
||||
output = data["output_quality"].get("summary", {})
|
||||
output_execution = data["output_execution"].get("summary", {})
|
||||
output_blind = data["output_blind_review"].get("summary", {})
|
||||
output_review_kit = data["output_review_kit"].get("summary", {})
|
||||
output_review = data["output_review_adjudication"].get("summary", {})
|
||||
benchmark = data["benchmark_reproducibility"].get("summary", {})
|
||||
blueprint = data["skill_os2_coverage"].get("summary", {})
|
||||
extension_partial = int(blueprint.get("extension_partial_count", 0) or 0)
|
||||
extension_planned = int(blueprint.get("extension_planned_count", 0) or 0)
|
||||
compiled = data["compiled_targets"].get("summary", {})
|
||||
conformance = data["conformance"].get("summary", {})
|
||||
runtime_permissions = data["runtime_permissions"].get("summary", {})
|
||||
trust = data["trust"].get("summary", {})
|
||||
python_compat = data["python_compatibility"].get("summary", {})
|
||||
architecture = data["architecture_maintainability"].get("summary", {})
|
||||
atlas = data["atlas"].get("summary", {})
|
||||
adoption = data["adoption_drift"].get("summary", {})
|
||||
daily_skillops = data["daily_skillops"].get("summary", {})
|
||||
weekly_curator = data["weekly_curator"].get("summary", {})
|
||||
waivers = data["review_waivers"].get("summary", {})
|
||||
annotations = data["review_annotations"].get("summary", {})
|
||||
intake = data["world_class_evidence_intake"].get("summary", {})
|
||||
claim_guard = data["world_class_claim_guard"].get("summary", {})
|
||||
registry = data["registry"].get("package", {})
|
||||
package_verification = data["package_verification"].get("summary", {})
|
||||
install_simulation = data["install_simulation"].get("summary", {})
|
||||
upgrade = data["upgrade_check"].get("summary", {})
|
||||
cards = [
|
||||
{
|
||||
"label": "Skill IR",
|
||||
"value": str(overview.get("skill_ir", {}).get("schema_version", "missing")),
|
||||
"detail": f"{overview.get('skill_ir', {}).get('target_count', 0)} targets in platform-neutral contract",
|
||||
},
|
||||
{
|
||||
"label": "Compiler",
|
||||
"value": f"{compiled.get('pass_count', 0)}/{compiled.get('target_count', 0)}",
|
||||
"detail": "target contracts compiled from Skill IR",
|
||||
},
|
||||
{
|
||||
"label": "Output Delta",
|
||||
"value": str(output.get("delta", "n/a")),
|
||||
"detail": f"{output.get('case_count', 0)} cases; {output.get('file_backed_case_count', 0)} file-backed",
|
||||
},
|
||||
{
|
||||
"label": "Exec Runs",
|
||||
"value": str(output_execution.get("variant_run_count", 0)),
|
||||
"detail": (
|
||||
f"command {output_execution.get('command_executed_count', 0)}; "
|
||||
f"model {output_execution.get('model_executed_count', 0)}; "
|
||||
f"recorded {output_execution.get('recorded_fixture_count', 0)}"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Blind A/B",
|
||||
"value": str(output_blind.get("pair_count", 0)),
|
||||
"detail": "review pairs hide baseline vs with-skill labels",
|
||||
},
|
||||
{
|
||||
"label": "Review Kit",
|
||||
"value": f"{output_review_kit.get('ready_for_adjudication_count', 0)}/{output_review_kit.get('case_count', 0)}",
|
||||
"detail": f"pending {output_review_kit.get('pending_decision_count', 0)}; answer key hidden",
|
||||
},
|
||||
{
|
||||
"label": "Review A/B",
|
||||
"value": f"{output_review.get('judgment_count', 0)}/{output_review.get('pair_count', 0)}",
|
||||
"detail": f"adjudication decisions; pending {output_review.get('pending_count', 0)}",
|
||||
},
|
||||
{
|
||||
"label": "Public Claim",
|
||||
"value": "ready" if benchmark.get("public_claim_ready") is True else "blocked",
|
||||
"detail": f"{benchmark.get('public_claim_blocker_count', 0)} blockers; local reproducible {str(benchmark.get('reproducibility_ready', False)).lower()}",
|
||||
},
|
||||
{
|
||||
"label": "Blueprint",
|
||||
"value": f"{blueprint.get('pass_count', 0)}/{blueprint.get('item_count', 0)}",
|
||||
"detail": (
|
||||
f"2.0 coverage; extensions partial {extension_partial}, "
|
||||
f"planned {extension_planned}; evidence pending {blueprint.get('world_class_evidence_pending_count', 0)}"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Runtime",
|
||||
"value": f"{conformance.get('pass_count', 0)}/{conformance.get('target_count', 0)}",
|
||||
"detail": "target conformance pass rate",
|
||||
},
|
||||
{
|
||||
"label": "Perm Probe",
|
||||
"value": f"{runtime_permissions.get('metadata_fallback_count', 0)}/{runtime_permissions.get('target_count', 0)}",
|
||||
"detail": (
|
||||
f"{runtime_permissions.get('native_enforcement_count', 0)} native; "
|
||||
f"{runtime_permissions.get('installer_enforcement_pass_count', 0)} installer-enforced"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Trust",
|
||||
"value": str(trust.get("secret_findings", 0)),
|
||||
"detail": f"{trust.get('script_count', 0)} scripts scanned; secrets found",
|
||||
},
|
||||
{
|
||||
"label": "Py Compat",
|
||||
"value": str(python_compat.get("issue_count", 0)),
|
||||
"detail": f"{python_compat.get('file_count', 0)} files scanned for Python {python_compat.get('target_python', '3.11')}",
|
||||
},
|
||||
{
|
||||
"label": "Arch Debt",
|
||||
"value": str(architecture.get("hotspot_count", 0)),
|
||||
"detail": (
|
||||
f"{architecture.get('largest_file_lines', 0)} largest lines; "
|
||||
f"{architecture.get('watchlist_count', 0)} watchlist; "
|
||||
f"{architecture.get('early_watchlist_count', 0)} early; "
|
||||
f"{architecture.get('command_handler_count', 0)} CLI handlers; "
|
||||
f"{architecture.get('entrypoint_command_handler_count', 0)} in entrypoint"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Atlas",
|
||||
"value": str(atlas.get("route_collision_count", 0)),
|
||||
"detail": f"{atlas.get('skill_count', 0)} scanned skills; route collisions",
|
||||
},
|
||||
{
|
||||
"label": "Drift",
|
||||
"value": str(adoption.get("risk_band", "n/a")),
|
||||
"detail": f"{adoption.get('event_count', 0)} metadata events; {adoption.get('missed_trigger_count', 0)} missed triggers",
|
||||
},
|
||||
{
|
||||
"label": "Daily Ops",
|
||||
"value": str(daily_skillops.get("proposal_count", 0)),
|
||||
"detail": (
|
||||
f"{daily_skillops.get('decision', 'missing')}; "
|
||||
f"approval {daily_skillops.get('approval_count', 0)}; "
|
||||
f"release lock {str(daily_skillops.get('release_lock_ready', False)).lower()}"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Weekly Queue",
|
||||
"value": str(weekly_curator.get("unique_opportunity_count", 0)),
|
||||
"detail": (
|
||||
f"{weekly_curator.get('decision', 'missing')}; "
|
||||
f"ready {weekly_curator.get('ready_for_approval_review_count', 0)}; "
|
||||
f"top score {weekly_curator.get('top_score', 0)}"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Waivers",
|
||||
"value": str(waivers.get("active_count", 0)),
|
||||
"detail": f"{waivers.get('covered_gate_count', 0)} gates covered; human risk decisions",
|
||||
},
|
||||
{
|
||||
"label": "Intake",
|
||||
"value": f"{intake.get('template_pass_count', 0)}/{intake.get('template_count', 0)}",
|
||||
"detail": (
|
||||
f"{intake.get('valid_submission_count', 0)} valid submissions; "
|
||||
f"{intake.get('invalid_submission_count', 0)} invalid"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Claim Guard",
|
||||
"value": str(claim_guard.get("violation_count", 0)),
|
||||
"detail": f"{claim_guard.get('claim_surface_count', 0)} public surfaces scanned",
|
||||
},
|
||||
{
|
||||
"label": "Notes",
|
||||
"value": f"{annotations.get('open_count', 0)}/{annotations.get('annotation_count', 0)}",
|
||||
"detail": f"{annotations.get('open_blocker_count', 0)} open blocker annotations",
|
||||
},
|
||||
{
|
||||
"label": "Registry",
|
||||
"value": str(registry.get("version", "n/a")),
|
||||
"detail": f"{len(registry.get('targets', []))} targets; {registry.get('license', 'no license')} license",
|
||||
},
|
||||
{
|
||||
"label": "Archive",
|
||||
"value": "pass" if data["package_verification"].get("ok") else "n/a",
|
||||
"detail": f"{package_verification.get('archive_entry_count', 0)} zip entries; package verification",
|
||||
},
|
||||
{
|
||||
"label": "Install",
|
||||
"value": "pass" if data["install_simulation"].get("ok") else "n/a",
|
||||
"detail": (
|
||||
f"{install_simulation.get('adapter_count', 0)} adapters; "
|
||||
f"{install_simulation.get('installer_permission_enforced_count', 0)} permissions enforced; "
|
||||
f"{install_simulation.get('installer_permission_failure_count', 0)} permission failures"
|
||||
),
|
||||
},
|
||||
{
|
||||
"label": "Upgrade",
|
||||
"value": str(upgrade.get("recommended_bump", "n/a")),
|
||||
"detail": f"declared {upgrade.get('declared_bump', 'n/a')}; {upgrade.get('breaking_change_count', 0)} breaking changes",
|
||||
},
|
||||
]
|
||||
return cards
|
||||
@@ -19,22 +19,48 @@ LABELS = {
|
||||
"archive_entry_count": "Zip 条目",
|
||||
"archive_present": "归档存在",
|
||||
"archive_sha256": "归档哈希",
|
||||
"actual_gate_count": "实际 Gate",
|
||||
"answer_revealed_count": "答案揭示",
|
||||
"baseline_pass_rate": "Baseline",
|
||||
"breaking_change_count": "破坏变更",
|
||||
"case_count": "案例数",
|
||||
"claim_surface_count": "声明面",
|
||||
"command_executed_count": "命令执行",
|
||||
"compatibility_pass_count": "兼容通过",
|
||||
"covered_gate_count": "覆盖 Gate",
|
||||
"daily_report_count": "日报数",
|
||||
"declared_bump": "声明版本",
|
||||
"decision": "决策",
|
||||
"delta": "增益",
|
||||
"duplicate_gate_keys": "重复 Gate",
|
||||
"event_count": "事件数",
|
||||
"expected_gate_count": "期望 Gate",
|
||||
"failure_count": "失败数",
|
||||
"file_count": "文件数",
|
||||
"fstring_311_violation_count": "F-string 3.11",
|
||||
"gate_pass": "Gate",
|
||||
"help_smoke_failed_count": "Help 失败",
|
||||
"human_review_complete": "人审完成",
|
||||
"install_simulated": "安装模拟",
|
||||
"installer_enforcement_pass_count": "安装拦截",
|
||||
"installer_enforcement_ready": "安装拦截就绪",
|
||||
"installer_enforcement_source_status": "安装证据",
|
||||
"installer_enforcement_target_count": "安装拦截目标",
|
||||
"installer_permission_capability_count": "安装权限能力",
|
||||
"installer_permission_enforced_count": "安装权限",
|
||||
"installer_permission_failure_count": "安装权限失败",
|
||||
"invalid_submission_count": "无效提交",
|
||||
"item_count": "项目数",
|
||||
"issue_count": "问题数",
|
||||
"ledger_pending_count": "台账待补",
|
||||
"ledger_ready_to_claim_world_class": "台账可声明",
|
||||
"license": "License",
|
||||
"local_blueprint_ready": "本地蓝图",
|
||||
"metadata_fallback_count": "Metadata fallback",
|
||||
"missed_trigger_count": "漏触发",
|
||||
"missing_count": "缺失数",
|
||||
"missing_gate_keys": "缺失 Gate",
|
||||
"module_count": "模块数",
|
||||
"model_executed_count": "模型执行",
|
||||
"name": "名称",
|
||||
"native_enforcement_count": "原生执行",
|
||||
@@ -46,24 +72,53 @@ LABELS = {
|
||||
"package_sha256": "包体哈希",
|
||||
"pass_count": "通过数",
|
||||
"pending_count": "待审",
|
||||
"pending_answer_hidden_count": "答案隐藏",
|
||||
"permission_capability_count": "权限能力",
|
||||
"permission_target_count": "权限目标",
|
||||
"provider_evidence_complete": "Provider 证据",
|
||||
"public_claim_blocker_count": "声明阻断",
|
||||
"public_claim_ready": "可公开声明",
|
||||
"public_world_class_ready": "世界级",
|
||||
"proposal_count": "提案数",
|
||||
"proposal_review_count": "提案复核",
|
||||
"reproducibility_ready": "本地复现",
|
||||
"recorded_fixture_count": "记录样本",
|
||||
"ready_for_approval_review_count": "待批准复核",
|
||||
"ready_for_external_collection": "可收证据",
|
||||
"ready_for_ledger_review": "可审台账",
|
||||
"recommended_pr_count": "建议 PR",
|
||||
"recommended_bump": "建议版本",
|
||||
"release_lock_ready": "发布锁",
|
||||
"rendered_gate_keys": "渲染 Gate",
|
||||
"residual_risk_count": "残余风险",
|
||||
"risk_band": "风险带",
|
||||
"route_collision_count": "路由冲突",
|
||||
"script_count": "脚本数",
|
||||
"schema_present": "Schema",
|
||||
"secret_findings": "Secret",
|
||||
"skill_count": "Skill 数",
|
||||
"submission_count": "提交数",
|
||||
"syntax_error_count": "语法错误",
|
||||
"target_count": "目标数",
|
||||
"target_python": "目标 Python",
|
||||
"targets": "目标平台",
|
||||
"template_count": "模板数",
|
||||
"template_pass_count": "模板通过",
|
||||
"timing_observed_count": "计时样本",
|
||||
"token_estimated_count": "估算 Token",
|
||||
"token_observed_count": "真实 Token",
|
||||
"trust_level": "信任级别",
|
||||
"unknown_gate_keys": "未知 Gate",
|
||||
"unweighted_gate_keys": "未加权 Gate",
|
||||
"unique_opportunity_count": "唯一机会",
|
||||
"variant_run_count": "运行数",
|
||||
"valid_submission_count": "有效提交",
|
||||
"version": "版本",
|
||||
"violation_count": "违规数",
|
||||
"warning_count": "警告数",
|
||||
"with_skill_pass_rate": "With Skill",
|
||||
"world_class_evidence_pending_count": "待补证据",
|
||||
"world_class_ready": "世界级就绪",
|
||||
}
|
||||
|
||||
|
||||
@@ -114,6 +169,22 @@ def render_kv_grid(
|
||||
return "<dl class='kv-grid'>" + "".join(rows) + "</dl>"
|
||||
|
||||
|
||||
def render_gate_contract_panel(contract: dict[str, Any]) -> str:
|
||||
return render_kv_grid(
|
||||
contract,
|
||||
[
|
||||
"ok",
|
||||
"expected_gate_count",
|
||||
"actual_gate_count",
|
||||
"missing_gate_keys",
|
||||
"unknown_gate_keys",
|
||||
"duplicate_gate_keys",
|
||||
"unweighted_gate_keys",
|
||||
],
|
||||
"gate contract missing",
|
||||
)
|
||||
|
||||
|
||||
def registry_package_summary(package: dict[str, Any]) -> dict[str, Any]:
|
||||
if not package:
|
||||
return {}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Review Studio gate keys, scoring, and contract checks."""
|
||||
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by review_studio_gates.py for stable gate scoring and contract checks."
|
||||
|
||||
|
||||
GATE_WEIGHTS = {
|
||||
"trigger-lab": 15,
|
||||
"output-lab": 20,
|
||||
"context-budget": 10,
|
||||
"runtime-matrix": 10,
|
||||
"trust-report": 10,
|
||||
"python-compat": 10,
|
||||
"architecture-maintainability": 10,
|
||||
"permission-gates": 10,
|
||||
"permission-runtime": 10,
|
||||
"skill-atlas": 10,
|
||||
"operations-loop": 10,
|
||||
"review-waivers": 10,
|
||||
"world-class-evidence": 10,
|
||||
"registry-audit": 10,
|
||||
"release-notes": 10,
|
||||
"intent-canvas": 10,
|
||||
}
|
||||
REVIEW_STUDIO_GATE_KEYS = frozenset(GATE_WEIGHTS)
|
||||
|
||||
|
||||
def status_label(status: str) -> str:
|
||||
return {"pass": "通过", "warn": "关注", "block": "阻断"}.get(status, status)
|
||||
|
||||
|
||||
def add_blockers_from_gate(gates: list[dict[str, str]]) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
|
||||
blockers = [item for item in gates if item["status"] == "block"]
|
||||
warnings = [item for item in gates if item["status"] == "warn"]
|
||||
return blockers, warnings
|
||||
|
||||
|
||||
def gate_contract(gates: list[dict[str, str]]) -> dict[str, Any]:
|
||||
rendered_gate_keys = [str(item.get("key", "")) for item in gates]
|
||||
rendered_gate_set = set(rendered_gate_keys)
|
||||
expected_gate_set = set(REVIEW_STUDIO_GATE_KEYS)
|
||||
duplicate_gate_keys = sorted(key for key, count in Counter(rendered_gate_keys).items() if count > 1)
|
||||
missing_gate_keys = sorted(expected_gate_set - rendered_gate_set)
|
||||
unknown_gate_keys = sorted(rendered_gate_set - expected_gate_set)
|
||||
unweighted_gate_keys = sorted(rendered_gate_set - set(GATE_WEIGHTS))
|
||||
return {
|
||||
"ok": not (missing_gate_keys or unknown_gate_keys or duplicate_gate_keys or unweighted_gate_keys),
|
||||
"expected_gate_count": len(expected_gate_set),
|
||||
"actual_gate_count": len(rendered_gate_keys),
|
||||
"expected_gate_keys": sorted(expected_gate_set),
|
||||
"rendered_gate_keys": rendered_gate_keys,
|
||||
"missing_gate_keys": missing_gate_keys,
|
||||
"unknown_gate_keys": unknown_gate_keys,
|
||||
"duplicate_gate_keys": duplicate_gate_keys,
|
||||
"unweighted_gate_keys": unweighted_gate_keys,
|
||||
}
|
||||
|
||||
|
||||
def min_output_cases(maturity: str) -> int:
|
||||
if maturity in {"library", "governed"}:
|
||||
return 5
|
||||
if maturity == "production":
|
||||
return 3
|
||||
return 1
|
||||
|
||||
|
||||
def weighted_score(gates: list[dict[str, str]]) -> int:
|
||||
earned = 0.0
|
||||
total = 0.0
|
||||
for item in gates:
|
||||
weight = GATE_WEIGHTS.get(item["key"], 5)
|
||||
total += weight
|
||||
if item["status"] == "pass":
|
||||
earned += weight
|
||||
elif item["status"] == "warn":
|
||||
earned += weight * 0.6
|
||||
return int(round(earned / total * 100)) if total else 0
|
||||
@@ -0,0 +1,140 @@
|
||||
"""Helper functions for Review Studio gate evaluation."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
from trust_check import permission_governance_status as compute_permission_governance_status
|
||||
from trust_check import script_inventory as trust_script_inventory
|
||||
except ImportError: # pragma: no cover
|
||||
compute_permission_governance_status = None
|
||||
trust_script_inventory = None
|
||||
|
||||
from review_studio_gate_contract import min_output_cases
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by review_studio_gates.py to keep reusable Review Studio gate helpers out of the main gate sequence."
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def link_from(output_html: Path, target: Path) -> str:
|
||||
return os.path.relpath(target.resolve(), output_html.parent.resolve())
|
||||
|
||||
|
||||
def report_link(output_html: Path, skill_dir: Path, rel_path: str) -> str:
|
||||
return link_from(output_html, skill_dir / rel_path)
|
||||
|
||||
|
||||
def gate(key: str, label: str, status: str, detail: str, evidence: str, link: str = "") -> dict[str, str]:
|
||||
return {
|
||||
"key": key,
|
||||
"label": label,
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
"evidence": evidence,
|
||||
"link": link,
|
||||
}
|
||||
|
||||
|
||||
def target_maturity(skill_dir: Path, overview: dict[str, Any]) -> str:
|
||||
manifest = load_json(skill_dir / "manifest.json")
|
||||
if manifest.get("maturity_tier"):
|
||||
return str(manifest["maturity_tier"])
|
||||
metadata = overview.get("metadata", {}) if isinstance(overview, dict) else {}
|
||||
if metadata.get("maturity_tier"):
|
||||
return str(metadata["maturity_tier"])
|
||||
return "scaffold"
|
||||
|
||||
|
||||
def fallback_permission_governance(skill_dir: Path) -> dict[str, Any]:
|
||||
if compute_permission_governance_status is None or trust_script_inventory is None:
|
||||
return {}
|
||||
try:
|
||||
scripts = trust_script_inventory(skill_dir)
|
||||
return compute_permission_governance_status(skill_dir, scripts)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def build_output_lab_gate(
|
||||
skill_dir: Path,
|
||||
output_html: Path,
|
||||
maturity: str,
|
||||
output: dict[str, Any],
|
||||
output_execution: dict[str, Any],
|
||||
output_blind: dict[str, Any],
|
||||
output_review: dict[str, Any],
|
||||
) -> dict[str, str]:
|
||||
output_summary = output.get("summary", {})
|
||||
output_execution_summary = output_execution.get("summary", {})
|
||||
output_blind_summary = output_blind.get("summary", {})
|
||||
output_review_summary = output_review.get("summary", {})
|
||||
required_cases = min_output_cases(maturity)
|
||||
case_count = int(output_summary.get("case_count", 0) or 0)
|
||||
file_backed = int(output_summary.get("file_backed_case_count", 0) or 0)
|
||||
near_neighbor = int(output_summary.get("near_neighbor_case_count", 0) or 0)
|
||||
boundary = int(output_summary.get("boundary_case_count", 0) or 0)
|
||||
blind_pair_count = int(output_blind_summary.get("pair_count", 0) or 0)
|
||||
execution_variant_count = int(output_execution_summary.get("variant_run_count", 0) or 0)
|
||||
execution_command_count = int(output_execution_summary.get("command_executed_count", 0) or 0)
|
||||
execution_model_count = int(output_execution_summary.get("model_executed_count", 0) or 0)
|
||||
execution_recorded_count = int(output_execution_summary.get("recorded_fixture_count", 0) or 0)
|
||||
review_pair_count = int(output_review_summary.get("pair_count", 0) or 0)
|
||||
review_judgment_count = int(output_review_summary.get("judgment_count", 0) or 0)
|
||||
review_pending_count = int(output_review_summary.get("pending_count", 0) or 0)
|
||||
review_invalid_count = int(output_review_summary.get("invalid_decision_count", 0) or 0)
|
||||
production_like = maturity in {"production", "library", "governed"}
|
||||
blind_missing = production_like and (not output_blind or blind_pair_count < case_count)
|
||||
review_missing = production_like and case_count > 0 and not output_review
|
||||
review_pending = production_like and bool(output_review) and review_pending_count > 0
|
||||
execution_failed = bool(output_execution) and (
|
||||
not output_execution.get("ok", True) or int(output_execution_summary.get("failure_count", 0) or 0) > 0
|
||||
)
|
||||
review_invalid = bool(output_review) and (not output_review.get("ok", True) or review_invalid_count > 0)
|
||||
output_blocked = (
|
||||
not output.get("ok", False)
|
||||
or not output_summary.get("gate_pass", False)
|
||||
or case_count < required_cases
|
||||
or execution_failed
|
||||
or review_invalid
|
||||
)
|
||||
output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing or review_missing or review_pending
|
||||
if not output:
|
||||
output_status = "warn"
|
||||
output_detail = "output eval scorecard is missing; generate it before production review"
|
||||
else:
|
||||
output_status = "block" if output_blocked else ("warn" if output_warn else "pass")
|
||||
output_detail = (
|
||||
f"{case_count}/{required_cases} cases; with-skill {output_summary.get('with_skill_pass_rate', 0)}; "
|
||||
f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}; "
|
||||
f"blind A/B {blind_pair_count}"
|
||||
+ (
|
||||
f"; exec {execution_variant_count}; command {execution_command_count}; "
|
||||
f"model {execution_model_count}; recorded {execution_recorded_count}"
|
||||
if output_execution
|
||||
else ""
|
||||
)
|
||||
+ (f"; reviewed {review_judgment_count}/{review_pair_count}" if output_review else "")
|
||||
+ (f"; review pending {review_pending_count}" if review_pending else "")
|
||||
+ ("; review adjudication missing" if review_missing else "")
|
||||
)
|
||||
return gate(
|
||||
"output-lab",
|
||||
"输出实验",
|
||||
output_status,
|
||||
output_detail,
|
||||
"reports/output_quality_scorecard.json",
|
||||
report_link(output_html, skill_dir, "reports/output_quality_scorecard.md"),
|
||||
)
|
||||
@@ -0,0 +1,543 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Gate evaluation contract for Review Studio 2.0."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from review_studio_gate_contract import (
|
||||
GATE_WEIGHTS,
|
||||
REVIEW_STUDIO_GATE_KEYS,
|
||||
add_blockers_from_gate,
|
||||
gate_contract,
|
||||
min_output_cases,
|
||||
status_label,
|
||||
weighted_score,
|
||||
)
|
||||
from review_studio_gate_helpers import (
|
||||
build_output_lab_gate,
|
||||
fallback_permission_governance,
|
||||
gate,
|
||||
report_link,
|
||||
target_maturity,
|
||||
)
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep Review Studio gate evaluation separate from HTML rendering."
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
overview = data["overview"]
|
||||
maturity = target_maturity(skill_dir, overview)
|
||||
gates: list[dict[str, str]] = []
|
||||
|
||||
intent = data["intent_confidence"]
|
||||
intent_score = int(intent.get("score", 0) or 0)
|
||||
intent_status = "pass" if intent.get("gate_passed") or intent_score >= 75 else "warn"
|
||||
gates.append(
|
||||
gate(
|
||||
"intent-canvas",
|
||||
"意图画布",
|
||||
intent_status,
|
||||
f"intent confidence {intent_score}/100; {intent.get('recommended_action', 'review current intent frame')}",
|
||||
"reports/intent-confidence.json",
|
||||
report_link(output_html, skill_dir, "reports/intent-confidence.md"),
|
||||
)
|
||||
)
|
||||
|
||||
route = data["route_scorecard"]
|
||||
route_summary = route.get("summary", {})
|
||||
misroutes = int(route_summary.get("misroute_count", len(route.get("misroutes", []))) or 0)
|
||||
ambiguous = int(route_summary.get("ambiguous_case_count", len(route.get("ambiguous_cases", []))) or 0)
|
||||
if not route:
|
||||
route_status = "warn"
|
||||
route_detail = "route scorecard is missing; run route-scorecard before release review"
|
||||
else:
|
||||
route_status = "block" if misroutes else ("warn" if ambiguous else "pass")
|
||||
route_detail = f"{route_summary.get('total_cases', 0)} trigger cases; {misroutes} misroutes; {ambiguous} ambiguous"
|
||||
gates.append(
|
||||
gate(
|
||||
"trigger-lab",
|
||||
"触发实验",
|
||||
route_status,
|
||||
route_detail,
|
||||
"reports/route_scorecard.json",
|
||||
report_link(output_html, skill_dir, "reports/route_scorecard.md"),
|
||||
)
|
||||
)
|
||||
|
||||
gates.append(
|
||||
build_output_lab_gate(
|
||||
skill_dir,
|
||||
output_html,
|
||||
maturity,
|
||||
data["output_quality"],
|
||||
data["output_execution"],
|
||||
data["output_blind_review"],
|
||||
data["output_review_adjudication"],
|
||||
)
|
||||
)
|
||||
|
||||
context = data["context_budget"]
|
||||
context_stats = context.get("stats", {})
|
||||
context_status = "pass" if context.get("ok") else "block"
|
||||
if context.get("warnings"):
|
||||
context_status = "warn" if context_status == "pass" else context_status
|
||||
if not context:
|
||||
context_status = "warn"
|
||||
large_deferred_dirs = context_stats.get("large_deferred_resource_dirs", []) or []
|
||||
top_deferred = "none"
|
||||
if large_deferred_dirs:
|
||||
first = large_deferred_dirs[0]
|
||||
top_deferred = f"{first.get('path', 'resource')} {first.get('estimated_tokens', 'n/a')}"
|
||||
deferred_governance = context_stats.get("deferred_resource_governance", {}) if isinstance(context_stats, dict) else {}
|
||||
governance_status = deferred_governance.get("status", "unknown") if isinstance(deferred_governance, dict) else "unknown"
|
||||
context_detail = (
|
||||
f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/"
|
||||
f"{context_stats.get('context_budget_limit', 'n/a')}; "
|
||||
f"deferred {context_stats.get('deferred_resource_tokens', 'n/a')}/"
|
||||
f"{context_stats.get('deferred_resource_warn_threshold', 'n/a')}; "
|
||||
f"top deferred {top_deferred}; "
|
||||
f"resource governance {governance_status}; "
|
||||
f"quality density {context_stats.get('quality_density', 'n/a')}"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"context-budget",
|
||||
"上下文",
|
||||
context_status,
|
||||
context_detail,
|
||||
"reports/context_budget.json",
|
||||
report_link(output_html, skill_dir, "reports/context_budget.md"),
|
||||
)
|
||||
)
|
||||
|
||||
conformance = data["conformance"]
|
||||
conformance_summary = conformance.get("summary", {})
|
||||
fail_count = int(conformance_summary.get("fail_count", 0) or 0)
|
||||
if not conformance:
|
||||
conformance_status = "warn"
|
||||
conformance_detail = "runtime conformance matrix is missing"
|
||||
else:
|
||||
conformance_status = "block" if fail_count else "pass"
|
||||
conformance_detail = f"{conformance_summary.get('pass_count', 0)} / {conformance_summary.get('target_count', 0)} targets pass"
|
||||
gates.append(
|
||||
gate(
|
||||
"runtime-matrix",
|
||||
"运行矩阵",
|
||||
conformance_status,
|
||||
conformance_detail,
|
||||
"reports/conformance_matrix.json",
|
||||
report_link(output_html, skill_dir, "reports/conformance_matrix.md"),
|
||||
)
|
||||
)
|
||||
|
||||
trust = data["trust"]
|
||||
trust_summary = trust.get("summary", {})
|
||||
if not trust:
|
||||
trust_status = "warn"
|
||||
trust_detail = "security trust report is missing"
|
||||
else:
|
||||
trust_status = "block" if trust.get("failures") else ("warn" if trust.get("warnings") else "pass")
|
||||
trust_detail = (
|
||||
f"{trust_summary.get('secret_findings', 0)} secrets; "
|
||||
f"{trust_summary.get('script_count', 0)} scripts; "
|
||||
f"{trust_summary.get('network_script_count', 0)} network-capable scripts; "
|
||||
f"{trust_summary.get('help_smoke_failed_count', 0)} help smoke failures"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"trust-report",
|
||||
"信任报告",
|
||||
trust_status,
|
||||
trust_detail,
|
||||
"reports/security_trust_report.json",
|
||||
report_link(output_html, skill_dir, "reports/security_trust_report.md"),
|
||||
)
|
||||
)
|
||||
|
||||
python_compat = data["python_compatibility"]
|
||||
python_compat_summary = python_compat.get("summary", {})
|
||||
if not python_compat:
|
||||
python_compat_status = "warn"
|
||||
python_compat_detail = "Python compatibility report is missing"
|
||||
else:
|
||||
issue_count = int(python_compat_summary.get("issue_count", 0) or 0)
|
||||
syntax_error_count = int(python_compat_summary.get("syntax_error_count", 0) or 0)
|
||||
fstring_violation_count = int(python_compat_summary.get("fstring_311_violation_count", 0) or 0)
|
||||
python_compat_status = (
|
||||
"block"
|
||||
if not python_compat.get("ok", True) or issue_count or syntax_error_count or fstring_violation_count
|
||||
else "pass"
|
||||
)
|
||||
python_compat_detail = (
|
||||
f"Python {python_compat_summary.get('target_python', '3.11')}; "
|
||||
f"{python_compat_summary.get('file_count', 0)} files; "
|
||||
f"{issue_count} compatibility issues; "
|
||||
f"{syntax_error_count} syntax; "
|
||||
f"{fstring_violation_count} f-string 3.11 hazards"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"python-compat",
|
||||
"Python 兼容",
|
||||
python_compat_status,
|
||||
python_compat_detail,
|
||||
"reports/python_compatibility.json",
|
||||
report_link(output_html, skill_dir, "reports/python_compatibility.md"),
|
||||
)
|
||||
)
|
||||
|
||||
architecture = data["architecture_maintainability"]
|
||||
architecture_summary = architecture.get("summary", {})
|
||||
if not architecture:
|
||||
if maturity in {"library", "governed"}:
|
||||
architecture_status = "warn"
|
||||
architecture_detail = "architecture maintainability report is missing for a reusable package"
|
||||
else:
|
||||
architecture_status = "pass"
|
||||
architecture_detail = "architecture maintainability report is optional until code surface grows"
|
||||
else:
|
||||
hotspot_count = int(architecture_summary.get("hotspot_count", 0) or 0)
|
||||
watchlist_count = int(architecture_summary.get("watchlist_count", 0) or 0)
|
||||
early_watchlist_count = int(architecture_summary.get("early_watchlist_count", 0) or 0)
|
||||
blocker_count = int(architecture_summary.get("blocker_count", 0) or 0)
|
||||
architecture_status = "block" if not architecture.get("ok", True) or blocker_count else ("warn" if hotspot_count else "pass")
|
||||
hotspot_label = "hotspot" if hotspot_count == 1 else "hotspots"
|
||||
watchlist_label = "watchlist file" if watchlist_count == 1 else "watchlist files"
|
||||
early_label = "early watch file" if early_watchlist_count == 1 else "early watch files"
|
||||
blocker_label = "blocker" if blocker_count == 1 else "blockers"
|
||||
architecture_detail = (
|
||||
f"{architecture_summary.get('python_file_count', 0)} Python files; "
|
||||
f"{hotspot_count} {hotspot_label}; "
|
||||
f"{watchlist_count} {watchlist_label}; "
|
||||
f"{early_watchlist_count} {early_label}; "
|
||||
f"{blocker_count} {blocker_label}; "
|
||||
f"largest {architecture_summary.get('largest_file_lines', 0)} lines; "
|
||||
f"{architecture_summary.get('command_handler_count', 0)} CLI handlers; "
|
||||
f"{architecture_summary.get('entrypoint_command_handler_count', 0)} in entrypoint"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"architecture-maintainability",
|
||||
"架构维护",
|
||||
architecture_status,
|
||||
architecture_detail,
|
||||
"reports/architecture_maintainability.json",
|
||||
report_link(output_html, skill_dir, "reports/architecture_maintainability.md"),
|
||||
)
|
||||
)
|
||||
|
||||
permission_governance = trust.get("permission_governance", {}) if isinstance(trust.get("permission_governance", {}), dict) else {}
|
||||
if trust and not permission_governance:
|
||||
permission_governance = fallback_permission_governance(skill_dir)
|
||||
if not trust:
|
||||
permission_status = "warn"
|
||||
permission_detail = "permission governance evidence is missing because trust report is missing"
|
||||
elif not permission_governance:
|
||||
permission_status = "warn"
|
||||
permission_detail = "permission governance evidence is missing from trust report"
|
||||
else:
|
||||
required = int(permission_governance.get("required_count", 0) or 0)
|
||||
approved = int(permission_governance.get("approval_count", 0) or 0)
|
||||
gaps = (
|
||||
int(permission_governance.get("missing_count", 0) or 0)
|
||||
+ int(permission_governance.get("invalid_count", 0) or 0)
|
||||
+ int(permission_governance.get("expired_count", 0) or 0)
|
||||
)
|
||||
if gaps:
|
||||
permission_status = "block" if maturity == "governed" else "warn"
|
||||
else:
|
||||
permission_status = "pass"
|
||||
required_names = ", ".join(permission_governance.get("required_capabilities", []) or []) or "none"
|
||||
permission_detail = f"{approved}/{required} permissions approved; gaps {gaps}; required {required_names}"
|
||||
gates.append(
|
||||
gate(
|
||||
"permission-gates",
|
||||
"权限批准",
|
||||
permission_status,
|
||||
permission_detail,
|
||||
"reports/security_trust_report.json + security/permission_policy.json",
|
||||
report_link(output_html, skill_dir, "security/permission_policy.md"),
|
||||
)
|
||||
)
|
||||
|
||||
runtime_permissions = data["runtime_permissions"]
|
||||
runtime_permissions_summary = runtime_permissions.get("summary", {})
|
||||
if not runtime_permissions:
|
||||
runtime_permission_status = "block" if maturity == "governed" else "warn"
|
||||
runtime_permission_detail = "runtime permission probe report is missing"
|
||||
elif runtime_permissions.get("failures"):
|
||||
runtime_permission_status = "block"
|
||||
runtime_permission_detail = f"{runtime_permissions_summary.get('failure_count', len(runtime_permissions.get('failures', [])))} runtime permission probe failures"
|
||||
else:
|
||||
runtime_permission_status = "pass"
|
||||
runtime_permission_detail = (
|
||||
f"{runtime_permissions_summary.get('pass_count', 0)}/{runtime_permissions_summary.get('target_count', 0)} targets probed; "
|
||||
f"native {runtime_permissions_summary.get('native_enforcement_count', 0)}; "
|
||||
f"metadata fallback {runtime_permissions_summary.get('metadata_fallback_count', 0)}; "
|
||||
f"installer {runtime_permissions_summary.get('installer_enforcement_pass_count', 0)}; "
|
||||
f"residual risks {runtime_permissions_summary.get('residual_risk_count', 0)}"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"permission-runtime",
|
||||
"权限探针",
|
||||
runtime_permission_status,
|
||||
runtime_permission_detail,
|
||||
"reports/runtime_permission_probes.json",
|
||||
report_link(output_html, skill_dir, "reports/runtime_permission_probes.md"),
|
||||
)
|
||||
)
|
||||
|
||||
atlas = data["atlas"]
|
||||
atlas_summary = atlas.get("summary", {})
|
||||
actionable_route_collisions = int(
|
||||
atlas_summary.get("actionable_route_collision_count", atlas_summary.get("route_collision_count", 0)) or 0
|
||||
)
|
||||
actionable_owner_gaps = int(atlas_summary.get("actionable_owner_gap_count", atlas_summary.get("owner_gap_count", 0)) or 0)
|
||||
actionable_stale = int(atlas_summary.get("actionable_stale_count", atlas_summary.get("stale_count", 0)) or 0)
|
||||
actionable_drift = int(atlas_summary.get("actionable_drift_signal_count", atlas_summary.get("drift_signal_count", 0)) or 0)
|
||||
atlas_issues = actionable_route_collisions + actionable_owner_gaps + actionable_stale + actionable_drift
|
||||
if not atlas:
|
||||
atlas_status = "warn"
|
||||
atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown"
|
||||
else:
|
||||
atlas_status = "warn" if atlas_issues else "pass"
|
||||
atlas_detail = (
|
||||
f"{atlas_summary.get('skill_count', 0)} skills, "
|
||||
f"{atlas_summary.get('actionable_skill_count', atlas_summary.get('skill_count', 0))} actionable; "
|
||||
f"{actionable_route_collisions} actionable route collisions; "
|
||||
f"{actionable_owner_gaps} actionable owner gaps; "
|
||||
f"{actionable_stale} actionable stale; "
|
||||
f"{actionable_drift} actionable drift; "
|
||||
f"{atlas_summary.get('non_actionable_issue_count', 0)} scoped non-actionable issues"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"skill-atlas",
|
||||
"组合治理",
|
||||
atlas_status,
|
||||
atlas_detail,
|
||||
"reports/skill_atlas.json",
|
||||
report_link(output_html, skill_dir, "reports/skill_atlas.html"),
|
||||
)
|
||||
)
|
||||
|
||||
adoption = data["adoption_drift"]
|
||||
adoption_summary = adoption.get("summary", {})
|
||||
daily_skillops = data.get("daily_skillops", {})
|
||||
weekly_curator = data.get("weekly_curator", {})
|
||||
daily_summary = daily_skillops.get("summary", {}) if isinstance(daily_skillops, dict) else {}
|
||||
weekly_summary = weekly_curator.get("summary", {}) if isinstance(weekly_curator, dict) else {}
|
||||
if not adoption:
|
||||
adoption_status = "warn"
|
||||
adoption_detail = "adoption drift report is missing; real usage impact is unknown"
|
||||
elif adoption.get("failures"):
|
||||
adoption_status = "block"
|
||||
adoption_detail = f"telemetry privacy or schema failures: {len(adoption.get('failures', []))}"
|
||||
else:
|
||||
risk_band = adoption_summary.get("risk_band", "no-data")
|
||||
adoption_status = "warn" if risk_band in {"no-data", "medium", "high"} else "pass"
|
||||
adoption_detail = (
|
||||
f"{adoption_summary.get('event_count', 0)} metadata events; "
|
||||
f"adoption {adoption_summary.get('adoption_rate', 0)}; "
|
||||
f"missed {adoption_summary.get('missed_trigger_count', 0)}; "
|
||||
f"bad-output {adoption_summary.get('bad_output_count', 0)}; "
|
||||
f"risk {risk_band}"
|
||||
)
|
||||
skillops_parts = []
|
||||
skillops_blocked = False
|
||||
for label, report, summary in [
|
||||
("daily", daily_skillops, daily_summary),
|
||||
("weekly", weekly_curator, weekly_summary),
|
||||
]:
|
||||
if not report:
|
||||
continue
|
||||
writes_source = bool(summary.get("writes_source_files") or report.get("writes_source_files"))
|
||||
auto_patch = bool(summary.get("auto_patch_enabled") or report.get("auto_patch_enabled"))
|
||||
failure_count = int(summary.get("failure_count", 0) or 0)
|
||||
if writes_source or auto_patch or failure_count:
|
||||
skillops_blocked = True
|
||||
if label == "daily":
|
||||
skillops_parts.append(
|
||||
f"daily proposals {summary.get('proposal_count', 0)}; "
|
||||
f"daily decision {summary.get('decision', report.get('decision', 'unknown'))}; "
|
||||
f"daily release lock {str(summary.get('release_lock_ready', False)).lower()}"
|
||||
)
|
||||
else:
|
||||
skillops_parts.append(
|
||||
f"weekly queue {summary.get('unique_opportunity_count', 0)} unique; "
|
||||
f"weekly ready {summary.get('ready_for_approval_review_count', 0)}; "
|
||||
f"weekly top {summary.get('top_score', 0)}; "
|
||||
f"weekly release lock {str(summary.get('release_lock_ready', False)).lower()}"
|
||||
)
|
||||
if skillops_blocked:
|
||||
adoption_status = "block"
|
||||
if skillops_parts:
|
||||
adoption_detail = adoption_detail + "; " + "; ".join(skillops_parts)
|
||||
gates.append(
|
||||
gate(
|
||||
"operations-loop",
|
||||
"运营回路",
|
||||
adoption_status,
|
||||
adoption_detail,
|
||||
"reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly",
|
||||
report_link(output_html, skill_dir, "reports/adoption_drift_report.md"),
|
||||
)
|
||||
)
|
||||
|
||||
waiver = data["review_waivers"]
|
||||
waiver_summary = waiver.get("summary", {})
|
||||
active_covered = set(waiver_summary.get("covered_gate_keys", []) or [])
|
||||
prior_blockers = [item for item in gates if item["status"] == "block"]
|
||||
prior_warnings = [item for item in gates if item["status"] == "warn"]
|
||||
unwaived_warnings = [item for item in prior_warnings if item["key"] not in active_covered]
|
||||
if not waiver:
|
||||
waiver_status = "warn"
|
||||
waiver_detail = "review waiver ledger is missing; warning acceptance is not auditable"
|
||||
elif waiver.get("failures"):
|
||||
waiver_status = "block"
|
||||
waiver_detail = f"{len(waiver.get('failures', []))} invalid waiver records"
|
||||
elif prior_blockers:
|
||||
waiver_status = "block"
|
||||
waiver_detail = f"{len(prior_blockers)} blocker gates cannot be waived in v0"
|
||||
elif unwaived_warnings:
|
||||
waiver_status = "warn"
|
||||
waiver_detail = (
|
||||
f"{waiver_summary.get('active_count', 0)} active waivers; "
|
||||
f"{len(unwaived_warnings)} warning gates still need reviewer decision"
|
||||
)
|
||||
else:
|
||||
waiver_status = "pass"
|
||||
waiver_detail = f"{waiver_summary.get('active_count', 0)} active waivers cover current warnings"
|
||||
gates.append(
|
||||
gate(
|
||||
"review-waivers",
|
||||
"人工批准",
|
||||
waiver_status,
|
||||
waiver_detail,
|
||||
"reports/review_waivers.json",
|
||||
report_link(output_html, skill_dir, "reports/review_waivers.md"),
|
||||
)
|
||||
)
|
||||
|
||||
world_class_ledger = data.get("world_class_evidence_ledger", {})
|
||||
world_class_summary = world_class_ledger.get("summary", {}) if isinstance(world_class_ledger, dict) else {}
|
||||
source_check_count = int(world_class_summary.get("source_check_count", 0) or 0)
|
||||
source_pass_count = int(world_class_summary.get("source_pass_count", 0) or 0)
|
||||
source_blocked_count = int(world_class_summary.get("source_blocked_count", 0) or 0)
|
||||
source_check_detail = (
|
||||
f"source checks {source_pass_count}/{source_check_count} pass; {source_blocked_count} blocked"
|
||||
if source_check_count
|
||||
else "source checks unavailable"
|
||||
)
|
||||
if not world_class_ledger and maturity == "governed":
|
||||
world_class_status = "warn"
|
||||
world_class_detail = "world-class evidence ledger is missing; public world-class readiness cannot be claimed"
|
||||
elif not world_class_ledger:
|
||||
world_class_status = "pass"
|
||||
world_class_detail = "world-class evidence ledger is optional until governed or public world-class readiness is claimed"
|
||||
elif world_class_summary.get("ready_to_claim_world_class") is True:
|
||||
world_class_status = "pass"
|
||||
world_class_detail = (
|
||||
f"{world_class_summary.get('accepted_count', 0)} accepted evidence entries; "
|
||||
f"{world_class_summary.get('pending_count', 0)} pending; "
|
||||
f"{source_check_detail}"
|
||||
)
|
||||
else:
|
||||
world_class_status = "warn"
|
||||
world_class_detail = (
|
||||
f"{world_class_summary.get('pending_count', 0)} pending world-class evidence entries; "
|
||||
f"{world_class_summary.get('human_pending_count', 0)} human pending; "
|
||||
f"{world_class_summary.get('external_pending_count', 0)} external pending; "
|
||||
f"{source_check_detail}; "
|
||||
f"overclaim guard {str(world_class_summary.get('overclaim_guard_active', False)).lower()}"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"world-class-evidence",
|
||||
"世界证据",
|
||||
world_class_status,
|
||||
world_class_detail,
|
||||
"reports/world_class_evidence_ledger.json",
|
||||
report_link(output_html, skill_dir, "reports/world_class_evidence_ledger.md"),
|
||||
)
|
||||
)
|
||||
|
||||
registry = data["registry"]
|
||||
install = data["install_simulation"]
|
||||
if not registry:
|
||||
if maturity in {"library", "governed"}:
|
||||
registry_status = "warn"
|
||||
registry_detail = "registry audit is missing; package metadata is not reviewable"
|
||||
else:
|
||||
registry_status = "pass"
|
||||
registry_detail = "registry audit is optional until team distribution is required"
|
||||
else:
|
||||
compatibility = registry.get("package", {}).get("compatibility", {})
|
||||
pass_count = sum(1 for status in compatibility.values() if status == "pass")
|
||||
registry_status = "block" if registry.get("failures") else ("warn" if registry.get("warnings") else "pass")
|
||||
registry_detail = (
|
||||
f"{registry.get('package', {}).get('name', 'package')} "
|
||||
f"{registry.get('package', {}).get('version', 'n/a')}; "
|
||||
f"{pass_count}/{len(compatibility)} compatibility entries pass"
|
||||
)
|
||||
if install:
|
||||
install_summary = install.get("summary", {})
|
||||
permission_failures = int(install_summary.get("installer_permission_failure_count", 0) or 0)
|
||||
if install.get("failures") or permission_failures:
|
||||
registry_status = "block"
|
||||
registry_detail += (
|
||||
f"; install {'pass' if install.get('ok') else 'fail'}"
|
||||
f" with {install_summary.get('adapter_count', 0)} adapters"
|
||||
f"; installer permissions {install_summary.get('installer_permission_enforced_count', 0)} enforced"
|
||||
f" / {permission_failures} failures"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"registry-audit",
|
||||
"注册审计",
|
||||
registry_status,
|
||||
registry_detail,
|
||||
"reports/registry_audit.json + reports/install_simulation.json",
|
||||
report_link(output_html, skill_dir, "reports/registry_audit.md"),
|
||||
)
|
||||
)
|
||||
|
||||
promotion = data["promotion"]
|
||||
migration_path = ROOT / "docs" / "migration-v2.md"
|
||||
if promotion:
|
||||
promotion_summary = promotion.get("summary", {})
|
||||
blocked = int(promotion_summary.get("blocked", 0) or 0)
|
||||
release_status = "block" if blocked else "pass"
|
||||
release_detail = f"{promotion_summary.get('promote', 0)} promote; {promotion_summary.get('keep_current', 0)} keep current; {blocked} blocked"
|
||||
else:
|
||||
release_status = "warn"
|
||||
release_detail = "promotion decisions are missing; release notes need reviewer confirmation"
|
||||
upgrade = data["upgrade_check"]
|
||||
if upgrade:
|
||||
upgrade_summary = upgrade.get("summary", {})
|
||||
if upgrade.get("failures"):
|
||||
release_status = "block"
|
||||
elif upgrade.get("warnings") and release_status == "pass":
|
||||
release_status = "warn"
|
||||
release_detail += (
|
||||
f"; upgrade {upgrade_summary.get('declared_bump', 'n/a')}"
|
||||
f" declared / {upgrade_summary.get('recommended_bump', 'n/a')} recommended"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"release-notes",
|
||||
"发布路线",
|
||||
release_status,
|
||||
release_detail,
|
||||
"reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md",
|
||||
report_link(output_html, skill_dir, "reports/promotion_decisions.md") if promotion else str(migration_path),
|
||||
)
|
||||
)
|
||||
|
||||
return gates
|
||||
+11
-221
@@ -2,12 +2,17 @@
|
||||
"""Static layout contract for Review Studio HTML."""
|
||||
|
||||
import html
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep Review Studio layout and CSS out of gate logic."
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
ASSET_DIR = ROOT / "assets"
|
||||
|
||||
|
||||
REVIEW_STUDIO_NAV = [
|
||||
("#overview", "审查总览"),
|
||||
("#intent", "意图画布"),
|
||||
@@ -22,11 +27,16 @@ REVIEW_STUDIO_NAV = [
|
||||
("#atlas", "组合治理"),
|
||||
("#telemetry", "运营回路"),
|
||||
("#waivers", "人工批准"),
|
||||
("#world-class", "世界证据"),
|
||||
("#registry", "注册审计"),
|
||||
("#release", "发布路线"),
|
||||
]
|
||||
|
||||
|
||||
def read_layout_asset(filename: str) -> str:
|
||||
return (ASSET_DIR / filename).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def render_review_nav(nav_items: list[tuple[str, str]] | None = None) -> str:
|
||||
items = REVIEW_STUDIO_NAV if nav_items is None else nav_items
|
||||
return "".join(
|
||||
@@ -36,224 +46,4 @@ def render_review_nav(nav_items: list[tuple[str, str]] | None = None) -> str:
|
||||
|
||||
|
||||
def review_studio_css() -> str:
|
||||
return """
|
||||
:root {
|
||||
--ink: #1B365D;
|
||||
--text: #24201d;
|
||||
--muted: #746d66;
|
||||
--line: #e7ded2;
|
||||
--soft: #faf8f5;
|
||||
--pass: #1e6b52;
|
||||
--warn: #9a6718;
|
||||
--block: #9b2c2c;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html { scroll-behavior: smooth; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: #ffffff;
|
||||
color: var(--text);
|
||||
font-family: Georgia, "Times New Roman", "Songti SC", serif;
|
||||
line-height: 1.58;
|
||||
}
|
||||
nav {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
padding: 10px 16px;
|
||||
background: rgba(255,255,255,0.94);
|
||||
border-bottom: 1px solid var(--line);
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
nav a {
|
||||
color: var(--ink);
|
||||
text-decoration: none;
|
||||
font-size: 14px;
|
||||
padding: 7px 10px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
nav a:hover { background: var(--soft); }
|
||||
main { max-width: 1180px; margin: 0 auto; padding: 44px 28px 76px; }
|
||||
header { border-bottom: 1px solid var(--line); padding-bottom: 28px; margin-bottom: 28px; }
|
||||
.eyebrow { color: var(--ink); font-size: 14px; letter-spacing: .08em; text-transform: uppercase; }
|
||||
h1, h2, h3 { color: var(--text); font-weight: 500; margin: 0; letter-spacing: 0; }
|
||||
h1 { font-size: clamp(34px, 5vw, 64px); line-height: 1.03; max-width: 920px; margin-top: 12px; }
|
||||
h2 { font-size: 30px; margin-bottom: 14px; }
|
||||
h3 { font-size: 19px; }
|
||||
p { margin: 0; }
|
||||
.lede { max-width: 820px; color: var(--muted); font-size: 20px; margin-top: 18px; }
|
||||
.decision {
|
||||
display: inline-flex;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
margin-top: 24px;
|
||||
padding: 12px 16px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
color: var(--ink);
|
||||
background: var(--soft);
|
||||
}
|
||||
.decision strong { font-size: 28px; }
|
||||
section { padding: 30px 0; border-bottom: 1px solid var(--line); scroll-margin-top: 76px; }
|
||||
.metrics, .gates {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
.gates { grid-template-columns: repeat(4, minmax(0, 1fr)); }
|
||||
.metric, .gate {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
background: #fff;
|
||||
min-width: 0;
|
||||
}
|
||||
.metric span, .gate span { display: block; color: var(--muted); font-size: 13px; }
|
||||
.metric strong { display: block; color: var(--ink); font-size: 34px; line-height: 1.1; margin: 8px 0; }
|
||||
.metric p, .gate p, .gate footer, .issues span, .evidence span { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
|
||||
.gate { display: flex; flex-direction: column; gap: 10px; }
|
||||
.gate.pass { border-top: 4px solid var(--pass); }
|
||||
.gate.warn { border-top: 4px solid var(--warn); }
|
||||
.gate.block { border-top: 4px solid var(--block); }
|
||||
.gate footer { border-top: 1px solid var(--line); padding-top: 10px; }
|
||||
a { color: var(--ink); }
|
||||
.twocol {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
|
||||
gap: 22px;
|
||||
align-items: start;
|
||||
}
|
||||
.panel {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 18px;
|
||||
background: #fff;
|
||||
}
|
||||
.panel p { color: var(--muted); }
|
||||
.kv-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 10px 12px;
|
||||
margin: 2px 0 0;
|
||||
}
|
||||
.kv-grid div {
|
||||
min-width: 0;
|
||||
padding: 9px 0 0;
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
.kv-grid dt {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
.kv-grid dd {
|
||||
margin: 2px 0 0;
|
||||
color: var(--ink);
|
||||
font-size: 15px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.issues, .evidence {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
.issues li, .evidence li {
|
||||
border-left: 3px solid var(--line);
|
||||
padding-left: 12px;
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
}
|
||||
.muted { color: var(--muted); }
|
||||
.actions-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
.annotations-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
.action-card {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 18px;
|
||||
background: #fff;
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
min-width: 0;
|
||||
}
|
||||
.action-card.warn { border-left: 4px solid var(--warn); }
|
||||
.action-card.block { border-left: 4px solid var(--block); }
|
||||
.annotation-card {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 18px;
|
||||
background: #fff;
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
min-width: 0;
|
||||
}
|
||||
.annotation-card.warning { border-left: 4px solid var(--warn); }
|
||||
.annotation-card.blocker { border-left: 4px solid var(--block); }
|
||||
.annotation-card.resolved { opacity: .72; }
|
||||
.action-card span,
|
||||
.annotation-card span,
|
||||
.action-card small,
|
||||
.annotation-card small,
|
||||
.action-card footer,
|
||||
.annotation-card footer,
|
||||
.action-card dd {
|
||||
color: var(--muted);
|
||||
font-size: 14px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
.annotation-card dd { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
|
||||
.action-card dl, .annotation-card dl {
|
||||
display: grid;
|
||||
grid-template-columns: 80px minmax(0, 1fr);
|
||||
gap: 6px 10px;
|
||||
margin: 0;
|
||||
}
|
||||
.action-card dt, .annotation-card dt { color: var(--ink); font-size: 14px; }
|
||||
.action-card dd, .annotation-card dd { margin: 0; }
|
||||
.source-ref-list {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
}
|
||||
.source-ref-list li {
|
||||
display: grid;
|
||||
gap: 2px;
|
||||
min-width: 0;
|
||||
padding: 8px 0 0;
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
.source-ref-list a,
|
||||
.source-ref-list span {
|
||||
overflow-wrap: anywhere;
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 13px;
|
||||
}
|
||||
.source-ref-list small {
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
code {
|
||||
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
font-size: 13px;
|
||||
}
|
||||
@media (max-width: 980px) {
|
||||
.metrics, .gates, .twocol, .actions-grid, .annotations-grid, .kv-grid { grid-template-columns: 1fr; }
|
||||
main { padding: 32px 18px 60px; }
|
||||
nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }
|
||||
nav a { flex: 0 0 auto; }
|
||||
}
|
||||
""".strip()
|
||||
return read_layout_asset("review-studio.css")
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Output review HTML helpers for Review Studio."""
|
||||
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to render output review checklist cards."
|
||||
|
||||
|
||||
def render_required_fields(fields: dict[str, Any]) -> str:
|
||||
if not fields:
|
||||
return "<p class='muted'>暂无字段要求。</p>"
|
||||
items = []
|
||||
for key, value in fields.items():
|
||||
items.append(f"<li><strong>{html.escape(str(key))}</strong><span>{html.escape(str(value))}</span></li>")
|
||||
return "<ul class='output-review-fields'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def render_command_list(commands: dict[str, Any]) -> str:
|
||||
if not commands:
|
||||
return "<p class='muted'>暂无命令。</p>"
|
||||
items = []
|
||||
for label, command in commands.items():
|
||||
if not command:
|
||||
continue
|
||||
items.append(
|
||||
"<li>"
|
||||
f"<span>{html.escape(str(label))}</span>"
|
||||
f"<code>{html.escape(str(command))}</code>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='output-review-commands'>" + "".join(items) + "</ul>" if items else "<p class='muted'>暂无命令。</p>"
|
||||
|
||||
|
||||
def render_contract(items: list[Any]) -> str:
|
||||
if not items:
|
||||
return "<p class='muted'>暂无边界。</p>"
|
||||
return "<ul>" + "".join(f"<li>{html.escape(str(item))}</li>" for item in items) + "</ul>"
|
||||
|
||||
|
||||
def render_output_review_checklist(adjudication: dict[str, Any]) -> str:
|
||||
checklist = adjudication.get("reviewer_checklist", []) if isinstance(adjudication, dict) else []
|
||||
if not checklist:
|
||||
return "<p class='muted'>当前没有输出评审操作清单。</p>"
|
||||
cards = []
|
||||
for item in checklist:
|
||||
readiness = str(item.get("readiness", "awaiting-decision"))
|
||||
answer_key = "可见" if item.get("answer_key_visible") else "隐藏"
|
||||
cards.append(
|
||||
"<article class='output-review-card "
|
||||
+ html.escape(readiness)
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(readiness)} · 答案{html.escape(answer_key)}</span>"
|
||||
f"<h3>{html.escape(str(item.get('case_id', 'case')))}</h3></div>"
|
||||
f"<p>{html.escape(str(item.get('blocking_reason', '')))}</p>"
|
||||
f"<dl><dt>盲评包</dt><dd><code>{html.escape(str(item.get('blind_pack_path', '')))}</code></dd>"
|
||||
f"<dt>决策表</dt><dd><code>{html.escape(str(item.get('decisions_path', '')))}</code></dd>"
|
||||
f"<dt>提示哈希</dt><dd><code>{html.escape(str(item.get('prompt_sha256', '')))}</code></dd></dl>"
|
||||
"<div class='output-review-steps'>"
|
||||
"<div><h4>操作命令</h4>"
|
||||
+ render_command_list(item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {})
|
||||
+ "</div>"
|
||||
"<div><h4>字段要求</h4>"
|
||||
+ render_required_fields(item.get("required_fields", {}) if isinstance(item.get("required_fields", {}), dict) else {})
|
||||
+ "</div>"
|
||||
"<div><h4>隐私边界</h4>"
|
||||
+ render_contract(item.get("privacy_contract", []) if isinstance(item.get("privacy_contract", []), list) else [])
|
||||
+ "</div>"
|
||||
"</div>"
|
||||
"</article>"
|
||||
)
|
||||
return "<div class='output-review-grid'>" + "".join(cards) + "</div>"
|
||||
|
||||
|
||||
def render_output_review_section(adjudication: dict[str, Any]) -> str:
|
||||
return (
|
||||
"<section>"
|
||||
"<h2>评审清单</h2>"
|
||||
"<p class='muted'>先打开 reports/output_review_kit.md;每张卡片对应一个 blind A/B case,说明当前是否待判、答案是否仍隐藏、应填写的决策文件、有效字段和复跑命令。</p>"
|
||||
f"{render_output_review_checklist(adjudication)}"
|
||||
"</section>"
|
||||
)
|
||||
@@ -0,0 +1,88 @@
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
from review_studio_gates import status_label
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep small Review Studio panel renderers out of the main page composer."
|
||||
|
||||
|
||||
def render_gate_list(gates: list[dict[str, str]]) -> str:
|
||||
items = []
|
||||
for item in gates:
|
||||
link_html = f"<a href='{html.escape(item['link'])}'>证据</a>" if item.get("link") else ""
|
||||
items.append(
|
||||
"<article class='gate "
|
||||
+ html.escape(item["status"])
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(status_label(item['status']))}</span><h3>{html.escape(item['label'])}</h3></div>"
|
||||
f"<p>{html.escape(item['detail'])}</p>"
|
||||
f"<footer>{html.escape(item['evidence'])} {link_html}</footer>"
|
||||
"</article>"
|
||||
)
|
||||
return "".join(items)
|
||||
|
||||
|
||||
def render_insights(cards: list[dict[str, str]]) -> str:
|
||||
return "".join(
|
||||
(
|
||||
"<article class='metric'>"
|
||||
f"<span>{html.escape(item['label'])}</span>"
|
||||
f"<strong>{html.escape(item['value'])}</strong>"
|
||||
f"<p>{html.escape(item['detail'])}</p>"
|
||||
"</article>"
|
||||
)
|
||||
for item in cards
|
||||
)
|
||||
|
||||
|
||||
def render_issue_list(title: str, items: list[dict[str, str]]) -> str:
|
||||
if not items:
|
||||
return f"<section><h2>{html.escape(title)}</h2><p class='muted'>无。</p></section>"
|
||||
body = "".join(
|
||||
(
|
||||
"<li>"
|
||||
f"<strong>{html.escape(item['label'])}</strong>"
|
||||
f"<span>{html.escape(item['detail'])}</span>"
|
||||
"</li>"
|
||||
)
|
||||
for item in items
|
||||
)
|
||||
return f"<section><h2>{html.escape(title)}</h2><ul class='issues'>{body}</ul></section>"
|
||||
|
||||
|
||||
def render_review_annotations_panel(annotations_report: dict[str, Any]) -> str:
|
||||
annotations = annotations_report.get("annotations", []) if isinstance(annotations_report, dict) else []
|
||||
if not annotations:
|
||||
return "<p class='muted'>当前没有 reviewer 批注。</p>"
|
||||
cards = []
|
||||
for item in annotations:
|
||||
line_suffix = f":{item['line']}" if item.get("line") else ""
|
||||
target_label = f"{item.get('target_path', '')}{line_suffix}"
|
||||
meta = " · ".join(
|
||||
part
|
||||
for part in [
|
||||
str(item.get("gate_key", "")),
|
||||
str(item.get("reviewer", "")),
|
||||
str(item.get("created_at", "")),
|
||||
]
|
||||
if part
|
||||
)
|
||||
cards.append(
|
||||
"<article class='annotation-card "
|
||||
+ html.escape(str(item.get("severity", "note")))
|
||||
+ " "
|
||||
+ html.escape(str(item.get("status", "open")))
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(str(item.get('severity', 'note')))} · {html.escape(str(item.get('status', 'open')))}</span>"
|
||||
f"<h3>{html.escape(str(item.get('id', 'annotation')))}</h3></div>"
|
||||
f"<p>{html.escape(str(item.get('body', '')))}</p>"
|
||||
f"<dl><dt>位置</dt><dd><code>{html.escape(target_label)}</code></dd>"
|
||||
f"<dt>Gate</dt><dd>{html.escape(str(item.get('gate_key', '')))}</dd>"
|
||||
f"<dt>建议</dt><dd>{html.escape(str(item.get('suggested_action', '') or '无'))}</dd></dl>"
|
||||
f"<small>{html.escape(meta)}</small>"
|
||||
f"<footer>{html.escape(str(item.get('source_excerpt', '')))}</footer>"
|
||||
"</article>"
|
||||
)
|
||||
return "".join(cards)
|
||||
@@ -0,0 +1,48 @@
|
||||
from typing import Any
|
||||
|
||||
from review_studio_formatting import render_kv_grid
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep SkillOps summary panels out of the main HTML renderer."
|
||||
|
||||
|
||||
def render_skillops_section(data: dict[str, Any]) -> str:
|
||||
daily_summary = data["daily_skillops"].get("summary", {})
|
||||
weekly_summary = data["weekly_curator"].get("summary", {})
|
||||
daily_panel = render_kv_grid(
|
||||
daily_summary,
|
||||
[
|
||||
"decision",
|
||||
"proposal_count",
|
||||
"approval_count",
|
||||
"pending_review_count",
|
||||
"release_lock_ready",
|
||||
"public_world_class_ready",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
],
|
||||
"no daily SkillOps summary",
|
||||
)
|
||||
weekly_panel = render_kv_grid(
|
||||
weekly_summary,
|
||||
[
|
||||
"decision",
|
||||
"week_id",
|
||||
"daily_report_count",
|
||||
"unique_opportunity_count",
|
||||
"ready_for_approval_review_count",
|
||||
"proposal_review_count",
|
||||
"top_score",
|
||||
"release_lock_ready",
|
||||
"writes_source_files",
|
||||
"auto_patch_enabled",
|
||||
],
|
||||
"no weekly curator summary",
|
||||
)
|
||||
return f"""
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>日常运维</h2>{daily_panel}</div>
|
||||
<div class="panel"><h2>周度队列</h2>{weekly_panel}</div>
|
||||
</section>
|
||||
"""
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Review Studio waiver candidate rendering helpers."""
|
||||
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to keep waiver candidate layout out of the main renderer."
|
||||
|
||||
|
||||
def render_waiver_candidates_panel(review_waivers: dict[str, Any]) -> str:
|
||||
candidates = review_waivers.get("waiver_candidates", []) if isinstance(review_waivers, dict) else []
|
||||
if not candidates:
|
||||
return "<p class='muted'>当前没有需要 reviewer 处理的批准候选。</p>"
|
||||
cards = []
|
||||
for item in candidates:
|
||||
allowed = bool(item.get("waiver_allowed"))
|
||||
allowed_label = "可批准" if allowed else "不可批准"
|
||||
status = str(item.get("status", "unknown"))
|
||||
required_review = item.get("required_review", [])
|
||||
required_html = "".join(
|
||||
f"<li>{html.escape(str(review_item))}</li>"
|
||||
for review_item in required_review
|
||||
) or "<li>证据不足,需要先补充审查条件。</li>"
|
||||
decision_options = item.get("decision_options", [])
|
||||
decision_options_label = ", ".join(str(option) for option in decision_options) if decision_options else "无"
|
||||
cards.append(
|
||||
"<article class='waiver-card "
|
||||
+ ("allowed" if allowed else "blocked")
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(allowed_label)} · {html.escape(status)}</span>"
|
||||
f"<h3>{html.escape(str(item.get('label') or item.get('gate_key') or 'Waiver Candidate'))}</h3></div>"
|
||||
f"<p>{html.escape(str(item.get('risk_summary') or '证据不足'))}</p>"
|
||||
"<dl>"
|
||||
f"<dt>Gate</dt><dd><code>{html.escape(str(item.get('gate_key', '')))}</code></dd>"
|
||||
f"<dt>证据</dt><dd><code>{html.escape(str(item.get('suggested_evidence', '')))}</code></dd>"
|
||||
f"<dt>选项</dt><dd>{html.escape(decision_options_label)}</dd>"
|
||||
f"<dt>边界</dt><dd>{html.escape(str(item.get('world_class_boundary', '')))}</dd>"
|
||||
"</dl>"
|
||||
"<div class='waiver-review'><h4>审查条件</h4>"
|
||||
f"<ul>{required_html}</ul></div>"
|
||||
"<div class='waiver-command'><span>建议命令</span>"
|
||||
f"<code>{html.escape(str(item.get('suggested_command', '')))}</code></div>"
|
||||
"</article>"
|
||||
)
|
||||
return "<div class='waiver-candidate-grid'>" + "".join(cards) + "</div>"
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""World-class evidence HTML helpers for Review Studio."""
|
||||
|
||||
import html
|
||||
from typing import Any
|
||||
|
||||
from world_class_source_checks import build_source_checklist
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to render world-class evidence cards."
|
||||
|
||||
|
||||
def render_inline_list(items: list[Any], empty_label: str) -> str:
|
||||
if not items:
|
||||
return f"<p class='muted'>{html.escape(empty_label)}</p>"
|
||||
return "<ul>" + "".join(f"<li>{html.escape(str(item))}</li>" for item in items) + "</ul>"
|
||||
|
||||
|
||||
def render_source_checks(entry: dict[str, Any]) -> str:
|
||||
rows = entry.get("source_checklist")
|
||||
if not isinstance(rows, list):
|
||||
rows = build_source_checklist([entry])
|
||||
if not rows:
|
||||
return "<p class='muted'>暂无源证据检查。</p>"
|
||||
items = []
|
||||
for row in rows:
|
||||
status = str(row.get("status", "blocked"))
|
||||
items.append(
|
||||
"<li class='world-source-check "
|
||||
+ html.escape(status)
|
||||
+ "'>"
|
||||
f"<span>{html.escape(str(row.get('label', '')))}</span>"
|
||||
f"<code>{html.escape(str(row.get('field', '')))}: {html.escape(str(row.get('actual', '')))} / {html.escape(str(row.get('expected', '')))}</code>"
|
||||
f"<small>{html.escape(str(row.get('next_action', '')))}</small>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='world-source-checks'>" + "".join(items) + "</ul>"
|
||||
|
||||
|
||||
def render_world_class_evidence_entries(ledger: dict[str, Any]) -> str:
|
||||
entries = ledger.get("entries", []) if isinstance(ledger, dict) else []
|
||||
if not entries:
|
||||
return "<p class='muted'>当前没有 world-class 证据条目。</p>"
|
||||
cards = []
|
||||
for entry in entries:
|
||||
observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {}
|
||||
observed_summary = "; ".join(f"{key}: {value}" for key, value in observed.items())
|
||||
submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {}
|
||||
submission_summary = "; ".join(
|
||||
f"{key}: {value}" for key, value in submission.items() if key in {"status", "path", "attested_real_evidence", "privacy_contract_satisfied"}
|
||||
)
|
||||
status = str(entry.get("status", "pending"))
|
||||
status_label_text = "已接受" if status == "accepted" else "待补证"
|
||||
cards.append(
|
||||
"<article class='world-evidence-card "
|
||||
+ html.escape(status)
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(status_label_text)} · {html.escape(str(entry.get('category', '')))}</span>"
|
||||
f"<h3>{html.escape(str(entry.get('label', entry.get('key', 'evidence'))))}</h3></div>"
|
||||
f"<p>{html.escape(str(entry.get('objective', '')))}</p>"
|
||||
f"<dl><dt>负责人</dt><dd>{html.escape(str(entry.get('owner', '')))}</dd>"
|
||||
f"<dt>当前状态</dt><dd>{html.escape(str(entry.get('current', '')))}</dd>"
|
||||
f"<dt>下一步</dt><dd>{html.escape(str(entry.get('next_action', '')))}</dd>"
|
||||
f"<dt>观测值</dt><dd>{html.escape(observed_summary or '无')}</dd>"
|
||||
f"<dt>提交态</dt><dd>{html.escape(submission_summary or 'missing')}</dd></dl>"
|
||||
"<section class='world-runbook-panel'><h4>执行步骤</h4>"
|
||||
+ render_inline_list(entry.get("runbook", []), "暂无执行步骤。")
|
||||
+ "</section>"
|
||||
"<div class='world-evidence-columns'>"
|
||||
"<div><h4>完成定义</h4>"
|
||||
+ render_inline_list(entry.get("success_checks", []), "暂无完成定义。")
|
||||
+ "</div>"
|
||||
"<div><h4>证据来源</h4>"
|
||||
+ render_inline_list(entry.get("evidence_artifacts", []), "暂无证据来源。")
|
||||
+ "</div>"
|
||||
"<div><h4>隐私约束</h4>"
|
||||
+ render_inline_list(entry.get("privacy_contract", []), "暂无隐私约束。")
|
||||
+ "</div>"
|
||||
"</div>"
|
||||
"<section class='world-source-panel'><h4>源证据检查</h4>"
|
||||
+ render_source_checks(entry)
|
||||
+ "</section>"
|
||||
"</article>"
|
||||
)
|
||||
return "<div class='world-evidence-grid'>" + "".join(cards) + "</div>"
|
||||
|
||||
|
||||
def render_command_list(commands: dict[str, Any]) -> str:
|
||||
if not commands:
|
||||
return "<p class='muted'>暂无命令。</p>"
|
||||
items = []
|
||||
for label, command in commands.items():
|
||||
if not command:
|
||||
continue
|
||||
items.append(
|
||||
"<li>"
|
||||
f"<span>{html.escape(str(label))}</span>"
|
||||
f"<code>{html.escape(str(command))}</code>"
|
||||
"</li>"
|
||||
)
|
||||
return "<ul class='world-intake-commands'>" + "".join(items) + "</ul>" if items else "<p class='muted'>暂无命令。</p>"
|
||||
|
||||
|
||||
def render_world_class_intake_checklist(intake: dict[str, Any]) -> str:
|
||||
items = intake.get("operator_checklist", []) if isinstance(intake, dict) else []
|
||||
if not items:
|
||||
return "<p class='muted'>当前没有 world-class 证据操作清单。</p>"
|
||||
cards = []
|
||||
for item in items:
|
||||
readiness = str(item.get("readiness", "awaiting-submission"))
|
||||
must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {}
|
||||
cards.append(
|
||||
"<article class='world-intake-card "
|
||||
+ html.escape(readiness)
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(readiness)} · {html.escape(str(item.get('category', '')))}</span>"
|
||||
f"<h3>{html.escape(str(item.get('label', item.get('evidence_key', 'evidence'))))}</h3></div>"
|
||||
f"<p>{html.escape(str(item.get('blocking_reason', '')))}</p>"
|
||||
f"<dl><dt>负责人</dt><dd>{html.escape(str(item.get('owner', '')))}</dd>"
|
||||
f"<dt>模板</dt><dd><code>{html.escape(str(item.get('template_path', '')))}</code></dd>"
|
||||
f"<dt>提交</dt><dd><code>{html.escape(str(item.get('submission_path', '')))}</code></dd>"
|
||||
f"<dt>下一步</dt><dd>{html.escape(str(item.get('next_action', '')))}</dd></dl>"
|
||||
"<div class='world-intake-steps'>"
|
||||
"<div><h4>操作命令</h4>"
|
||||
+ render_command_list(item.get("commands", {}) if isinstance(item.get("commands", {}), dict) else {})
|
||||
+ "</div>"
|
||||
"<div><h4>收集要求</h4>"
|
||||
+ render_inline_list(must_collect.get("provenance_requirements", []), "暂无来源要求。")
|
||||
+ "</div>"
|
||||
"<div><h4>执行步骤</h4>"
|
||||
+ render_inline_list(must_collect.get("runbook", []), "暂无执行步骤。")
|
||||
+ "</div>"
|
||||
"<div><h4>通过条件</h4>"
|
||||
+ render_inline_list(must_collect.get("success_checks", []), "暂无通过条件。")
|
||||
+ "</div>"
|
||||
"<div><h4>隐私边界</h4>"
|
||||
+ render_inline_list(must_collect.get("privacy_contract", []), "暂无隐私边界。")
|
||||
+ "</div>"
|
||||
"</div>"
|
||||
"</article>"
|
||||
)
|
||||
return "<div class='world-intake-grid'>" + "".join(cards) + "</div>"
|
||||
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Data preparation helpers for the compact review viewer."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from render_intent_confidence import render_intent_confidence
|
||||
from render_intent_dialogue import render_intent_dialogue
|
||||
from render_iteration_directions import render_iteration_directions
|
||||
from render_artifact_design_profile import render_artifact_design_profile
|
||||
from render_output_risk_profile import render_output_risk_profile
|
||||
from render_prompt_quality_profile import render_prompt_quality_profile
|
||||
from render_reference_scan import render_reference_scan
|
||||
from render_reference_synthesis import render_reference_synthesis
|
||||
from render_skill_overview import render_skill_overview
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_review_viewer.py to assemble Review Viewer data before HTML rendering."
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def load_feedback_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "feedback-log.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_baseline_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "baseline-compare.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_specific_compare(skill_dir: Path) -> dict:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "description_optimization.json",
|
||||
skill_dir.parent / "optimization" / "reports" / "description_optimization.json",
|
||||
]
|
||||
for path in candidates:
|
||||
payload = load_json(path)
|
||||
if isinstance(payload, dict) and payload:
|
||||
return payload
|
||||
return {}
|
||||
|
||||
|
||||
def load_specific_promotion(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "promotion_decisions.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_benchmark_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "github-benchmark-scan.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_reference_synthesis_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "reference-synthesis.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_output_risk_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "output-risk-profile.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_artifact_design_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "artifact-design-profile.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_prompt_quality_summary(skill_dir: Path) -> dict:
|
||||
payload = load_json(skill_dir / "reports" / "prompt-quality-profile.json")
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def ensure_report_inputs(skill_dir: Path) -> dict:
|
||||
overview_json = skill_dir / "reports" / "skill-overview.json"
|
||||
intent_confidence_json = skill_dir / "reports" / "intent-confidence.json"
|
||||
intent_json = skill_dir / "reports" / "intent-dialogue.json"
|
||||
reference_json = skill_dir / "reports" / "reference-scan.json"
|
||||
reference_synthesis_json = skill_dir / "reports" / "reference-synthesis.json"
|
||||
output_risk_json = skill_dir / "reports" / "output-risk-profile.json"
|
||||
artifact_design_json = skill_dir / "reports" / "artifact-design-profile.json"
|
||||
prompt_quality_json = skill_dir / "reports" / "prompt-quality-profile.json"
|
||||
directions_json = skill_dir / "reports" / "iteration-directions.json"
|
||||
|
||||
overview_payload = load_json(overview_json) if overview_json.exists() else {}
|
||||
intent_confidence_payload = load_json(intent_confidence_json) if intent_confidence_json.exists() else {}
|
||||
intent_payload = load_json(intent_json) if intent_json.exists() else {}
|
||||
reference_payload = load_json(reference_json) if reference_json.exists() else {}
|
||||
reference_synthesis_payload = load_json(reference_synthesis_json) if reference_synthesis_json.exists() else {}
|
||||
output_risk_payload = load_json(output_risk_json) if output_risk_json.exists() else {}
|
||||
artifact_design_payload = load_json(artifact_design_json) if artifact_design_json.exists() else {}
|
||||
prompt_quality_payload = load_json(prompt_quality_json) if prompt_quality_json.exists() else {}
|
||||
directions_payload = load_json(directions_json) if directions_json.exists() else {}
|
||||
|
||||
intent_confidence = intent_confidence_payload or render_intent_confidence(skill_dir)["summary"]
|
||||
intent = intent_payload or render_intent_dialogue(skill_dir)["summary"]
|
||||
reference = reference_payload or render_reference_scan(skill_dir, [])["summary"]
|
||||
reference_synthesis = reference_synthesis_payload or render_reference_synthesis(skill_dir)["summary"]
|
||||
output_risk = output_risk_payload or render_output_risk_profile(skill_dir)["summary"]
|
||||
artifact_design = artifact_design_payload or render_artifact_design_profile(skill_dir)["summary"]
|
||||
prompt_quality = prompt_quality_payload or render_prompt_quality_profile(skill_dir)["summary"]
|
||||
iteration_payload = directions_payload or render_iteration_directions(skill_dir)
|
||||
iteration = iteration_payload.get("summary", {})
|
||||
overview = overview_payload or render_skill_overview(skill_dir)["summary"]
|
||||
feedback = load_feedback_summary(skill_dir)
|
||||
baseline = load_baseline_summary(skill_dir)
|
||||
compare = load_specific_compare(skill_dir)
|
||||
promotion = load_specific_promotion(skill_dir)
|
||||
benchmark = load_benchmark_summary(skill_dir)
|
||||
reference_synthesis = load_reference_synthesis_summary(skill_dir)
|
||||
output_risk = load_output_risk_summary(skill_dir) or output_risk
|
||||
artifact_design = load_artifact_design_summary(skill_dir) or artifact_design
|
||||
prompt_quality = load_prompt_quality_summary(skill_dir) or prompt_quality
|
||||
return {
|
||||
"overview": overview,
|
||||
"intent_confidence": intent_confidence,
|
||||
"intent": intent,
|
||||
"reference": reference,
|
||||
"iteration": iteration_payload,
|
||||
"feedback": feedback,
|
||||
"baseline": baseline,
|
||||
"compare": compare,
|
||||
"promotion": promotion,
|
||||
"benchmark": benchmark,
|
||||
"reference_synthesis": reference_synthesis,
|
||||
"output_risk": output_risk,
|
||||
"artifact_design": artifact_design,
|
||||
"prompt_quality": prompt_quality,
|
||||
}
|
||||
|
||||
def architecture_steps(overview: dict) -> list[dict]:
|
||||
logic = overview.get("logic_steps", [])[:3]
|
||||
usage = overview.get("usage_steps", [])[:2]
|
||||
return [
|
||||
{"label": "Inputs", "detail": "workflow, prompt, transcript, docs, or notes"},
|
||||
{"label": "Boundary", "detail": overview.get("description", "Define the recurring job and exclusions.")},
|
||||
{"label": "Logic", "detail": "; ".join(logic) if logic else "Understand, execute, and validate."},
|
||||
{"label": "Usage", "detail": "; ".join(usage) if usage else "Load the skill and follow the workflow."},
|
||||
{"label": "Next", "detail": "Review the top iteration directions before growing the package."},
|
||||
]
|
||||
|
||||
|
||||
def compare_rows(compare: dict) -> list[dict]:
|
||||
if not compare:
|
||||
return []
|
||||
rows = []
|
||||
items = [
|
||||
("Baseline", compare.get("baseline", {})),
|
||||
("Current", compare.get("current_candidate", {})),
|
||||
(compare.get("winner", {}).get("label", "Winner"), compare.get("winner", {})),
|
||||
]
|
||||
for label, payload in items:
|
||||
if not payload:
|
||||
continue
|
||||
dev = payload.get("dev", {})
|
||||
holdout = payload.get("holdout", {})
|
||||
rows.append(
|
||||
{
|
||||
"label": label,
|
||||
"tokens": payload.get("estimated_tokens", 0),
|
||||
"dev_errors": dev.get("total_errors", 0),
|
||||
"holdout_errors": holdout.get("total_errors", 0),
|
||||
"strategy": payload.get("strategy", "existing"),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def benchmark_cards(benchmark: dict) -> list[dict]:
|
||||
cards = []
|
||||
for repo in benchmark.get("repositories", [])[:3]:
|
||||
cards.append(
|
||||
{
|
||||
"name": repo.get("full_name", "Unknown repo"),
|
||||
"borrow": repo.get("borrow", [])[:2],
|
||||
"avoid": repo.get("avoid", [])[:1],
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def synthesis_cards(reference_synthesis: dict) -> list[dict]:
|
||||
cards = []
|
||||
for track in reference_synthesis.get("source_tracks", [])[:3]:
|
||||
cards.append(
|
||||
{
|
||||
"name": track.get("name", "Unknown track"),
|
||||
"borrow": [track.get("borrow", "")] if track.get("borrow") else [],
|
||||
"avoid": [track.get("avoid", "")] if track.get("avoid") else [],
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def split_sentences(text: str) -> list[str]:
|
||||
if not text:
|
||||
return []
|
||||
parts = [item.strip() for item in re.split(r"(?<=[.!?])\s+", " ".join(text.split())) if item.strip()]
|
||||
return parts
|
||||
|
||||
|
||||
def metric_delta(current: int | float, baseline: int | float) -> str:
|
||||
delta = current - baseline
|
||||
if delta == 0:
|
||||
return "0"
|
||||
return f"{delta:+}"
|
||||
|
||||
|
||||
def variant_diff_cards(compare: dict) -> list[dict]:
|
||||
baseline = compare.get("baseline", {})
|
||||
current = compare.get("current_candidate", {})
|
||||
winner = compare.get("winner", {})
|
||||
variants = [
|
||||
("Baseline", baseline),
|
||||
("Current", current),
|
||||
(f"Winner — {winner.get('label', 'Winner')}", winner),
|
||||
]
|
||||
baseline_sentences = split_sentences(baseline.get("description", ""))
|
||||
baseline_set = set(baseline_sentences)
|
||||
baseline_dev = baseline.get("dev", {}).get("total_errors", 0)
|
||||
baseline_holdout = baseline.get("holdout", {}).get("total_errors", 0)
|
||||
cards = []
|
||||
seen = set()
|
||||
for label, payload in variants:
|
||||
if not payload:
|
||||
continue
|
||||
unique_key = (payload.get("description"), payload.get("strategy"), label)
|
||||
if unique_key in seen:
|
||||
continue
|
||||
seen.add(unique_key)
|
||||
description = payload.get("description", "")
|
||||
sentences = split_sentences(description)
|
||||
sentence_set = set(sentences)
|
||||
added = [item for item in sentences if item not in baseline_set][:3]
|
||||
removed = [item for item in baseline_sentences if item not in sentence_set][:2]
|
||||
dev_errors = payload.get("dev", {}).get("total_errors", 0)
|
||||
holdout_errors = payload.get("holdout", {}).get("total_errors", 0)
|
||||
cards.append(
|
||||
{
|
||||
"label": label,
|
||||
"strategy": payload.get("strategy", "existing"),
|
||||
"description": description,
|
||||
"tokens": payload.get("estimated_tokens", 0),
|
||||
"dev_errors": dev_errors,
|
||||
"holdout_errors": holdout_errors,
|
||||
"token_delta": metric_delta(payload.get("estimated_tokens", 0), baseline.get("estimated_tokens", 0)),
|
||||
"dev_delta": metric_delta(dev_errors, baseline_dev),
|
||||
"holdout_delta": metric_delta(holdout_errors, baseline_holdout),
|
||||
"added": added if label != "Baseline" else baseline_sentences[:3],
|
||||
"removed": removed,
|
||||
}
|
||||
)
|
||||
return cards
|
||||
|
||||
|
||||
def evidence_readiness(report: dict) -> dict:
|
||||
intent_confidence = report.get("intent_confidence", {})
|
||||
reference_synthesis = report.get("reference_synthesis", {})
|
||||
output_risk = report.get("output_risk", {})
|
||||
artifact_design = report.get("artifact_design", {})
|
||||
prompt_quality = report.get("prompt_quality", {})
|
||||
benchmark = report.get("benchmark", {})
|
||||
synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {}
|
||||
pattern_gate = synthesis.get("pattern_gate", {}) if isinstance(synthesis, dict) else {}
|
||||
accepted_patterns = pattern_gate.get("accepted", []) if isinstance(pattern_gate, dict) else []
|
||||
conflicts = synthesis.get("conflicts", []) if isinstance(synthesis, dict) else []
|
||||
checks = [
|
||||
{
|
||||
"label": "Intent clarity",
|
||||
"status": "ready" if intent_confidence.get("gate_passed") else "needs review",
|
||||
"detail": f"{intent_confidence.get('score', 0)}/100 intent confidence.",
|
||||
},
|
||||
{
|
||||
"label": "Benchmark coverage",
|
||||
"status": "ready" if len(benchmark.get("repositories", [])) >= 2 else "needs evidence",
|
||||
"detail": f"{len(benchmark.get('repositories', []))} GitHub benchmark repositories attached.",
|
||||
},
|
||||
{
|
||||
"label": "Pattern gate",
|
||||
"status": "ready" if accepted_patterns else "needs review",
|
||||
"detail": pattern_gate.get("summary", "No pattern gate summary attached."),
|
||||
},
|
||||
{
|
||||
"label": "Conflict handling",
|
||||
"status": "ready" if not conflicts else "decision needed",
|
||||
"detail": "No material conflicts detected." if not conflicts else conflicts[0].get("summary", "Conflict detected."),
|
||||
},
|
||||
{
|
||||
"label": "Output risk profile",
|
||||
"status": "ready" if output_risk.get("risk_families") else "needs review",
|
||||
"detail": f"{len(output_risk.get('risk_families', []))} output risk families attached.",
|
||||
},
|
||||
{
|
||||
"label": "Artifact design profile",
|
||||
"status": "ready" if artifact_design.get("primary_artifact") else "needs review",
|
||||
"detail": artifact_design.get("primary_artifact", {}).get("direction", "No artifact design profile attached."),
|
||||
},
|
||||
{
|
||||
"label": "Prompt quality profile",
|
||||
"status": "ready" if prompt_quality.get("quality_matrix") else "needs review",
|
||||
"detail": f"{prompt_quality.get('overall_quality_score', 0)}/100 prompt-facing quality score.",
|
||||
},
|
||||
]
|
||||
ready_count = sum(1 for item in checks if item["status"] == "ready")
|
||||
return {
|
||||
"score": int(ready_count / len(checks) * 100),
|
||||
"checks": checks,
|
||||
"reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery.",
|
||||
}
|
||||
@@ -10,6 +10,8 @@ try:
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
from skill_ir_paths import find_skill_ir as find_skill_ir_document
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
DEFAULT_TARGETS = ["openai", "claude", "agent-skills", "vscode", "generic"]
|
||||
@@ -64,10 +66,7 @@ def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]:
|
||||
|
||||
|
||||
def find_skill_ir(skill_dir: Path, name: str) -> dict[str, Any]:
|
||||
direct = load_json(skill_dir / "reports" / "skill-ir.json")
|
||||
if direct:
|
||||
return direct
|
||||
return load_json(skill_dir / "skill-ir" / "examples" / f"{name}.json")
|
||||
return find_skill_ir_document(skill_dir, name)[0]
|
||||
|
||||
|
||||
def add_check(checks: list[str], failures: list[str], condition: bool, passed: str, failed: str) -> None:
|
||||
@@ -144,7 +143,11 @@ def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict
|
||||
add_check(checks, failures, re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name) is not None, "name is runtime-safe", "name contains unsafe characters")
|
||||
|
||||
if target in {"agent-skills", "vscode"}:
|
||||
add_check(checks, failures, skill_dir.name == name, "directory name matches skill name", "directory name must match skill name")
|
||||
add_check(checks, failures, bool(name), "package identity derives from skill name", "Missing package identity")
|
||||
if skill_dir.name != name:
|
||||
warnings.append(
|
||||
"source checkout directory differs from skill name; package verification must enforce archive top-level identity."
|
||||
)
|
||||
|
||||
for field in ("name", "version", "owner", "status", "maturity_tier", "review_cadence"):
|
||||
add_check(checks, failures, bool(manifest.get(field)), f"manifest.{field} exists", f"Missing manifest.{field}")
|
||||
|
||||
@@ -164,7 +164,7 @@ def command_run(
|
||||
execution_kind = str(payload.get("execution_kind", "command"))
|
||||
provider = str(payload.get("provider", ""))
|
||||
model = str(payload.get("model", ""))
|
||||
model_executed = execution_kind == "model" or bool(model and provider)
|
||||
model_executed = execution_kind == "model" and bool(model and provider)
|
||||
return {
|
||||
"case_id": request["case_id"],
|
||||
"variant": variant,
|
||||
@@ -247,7 +247,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
[
|
||||
"No model-executed runs are recorded yet.",
|
||||
"",
|
||||
"Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence.",
|
||||
"Use `python3 scripts/yao.py output-exec --provider-runner openai` or `--runner-command` with a reviewed provider-backed runner to replace recorded fixtures with real model output evidence.",
|
||||
"",
|
||||
]
|
||||
)
|
||||
@@ -285,7 +285,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
"## Next Fixes",
|
||||
"",
|
||||
"- Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence.",
|
||||
"- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior.",
|
||||
"- Use `scripts/provider_output_eval_runner.py` for provider-backed holdout cases when release confidence depends on real generation behavior.",
|
||||
"- Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse.",
|
||||
]
|
||||
)
|
||||
|
||||
+118
-12
@@ -4,7 +4,7 @@ import json
|
||||
import shutil
|
||||
import tempfile
|
||||
import zipfile
|
||||
from datetime import date
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
|
||||
@@ -91,22 +91,111 @@ def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str,
|
||||
failures.append(detail)
|
||||
|
||||
|
||||
def adapter_targets(package_dir: Path) -> list[str]:
|
||||
targets_dir = package_dir / "targets"
|
||||
def package_name(package_manifest: dict[str, Any], skill_dir: Path) -> str:
|
||||
return str(package_manifest.get("name") or skill_dir.name)
|
||||
|
||||
|
||||
def adapter_targets(adapter_root: Path) -> list[str]:
|
||||
targets_dir = adapter_root / "targets"
|
||||
if not targets_dir.exists():
|
||||
return []
|
||||
return sorted(path.name for path in targets_dir.iterdir() if path.is_dir())
|
||||
|
||||
|
||||
def sorted_strings(value: Any) -> list[str]:
|
||||
return sorted(str(item) for item in value) if isinstance(value, list) else []
|
||||
|
||||
|
||||
def parse_date(value: str) -> date | None:
|
||||
try:
|
||||
return datetime.strptime(value, "%Y-%m-%d").date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def permission_policy_checks(
|
||||
installed_dir: Path | None,
|
||||
adapter_root: Path,
|
||||
targets: list[str],
|
||||
generated_at: str,
|
||||
checks: list[dict[str, str]],
|
||||
failures: list[str],
|
||||
) -> dict[str, int]:
|
||||
installed = installed_dir is not None and installed_dir.exists() and installed_dir.is_dir()
|
||||
policy = load_json(installed_dir / "security" / "permission_policy.json") if installed else {}
|
||||
capabilities = policy.get("capabilities", {}) if isinstance(policy.get("capabilities"), dict) else {}
|
||||
generated_date = parse_date(generated_at[:10])
|
||||
enforced_count = 0
|
||||
permission_failure_start = len(failures)
|
||||
declared_capabilities: set[str] = set()
|
||||
|
||||
add_check(checks, failures, "permission-policy-load", bool(capabilities), "Installed permission policy is readable")
|
||||
for target in targets:
|
||||
adapter = load_json(adapter_root / "targets" / target / "adapter.json")
|
||||
target_contract = adapter.get("target_permission_contract", {}) if isinstance(adapter, dict) else {}
|
||||
target_capabilities = sorted_strings(target_contract.get("declared_capabilities"))
|
||||
declared_capabilities.update(target_capabilities)
|
||||
add_check(
|
||||
checks,
|
||||
failures,
|
||||
f"permission-{target}-contract",
|
||||
bool(target_contract),
|
||||
f"{target} adapter exposes target permission contract for installer enforcement",
|
||||
)
|
||||
if not target_capabilities:
|
||||
continue
|
||||
for capability in target_capabilities:
|
||||
approval = capabilities.get(capability, {}) if isinstance(capabilities, dict) else {}
|
||||
enforcement = approval.get("target_enforcement", {}) if isinstance(approval.get("target_enforcement"), dict) else {}
|
||||
expires_at = str(approval.get("expires_at", "")).strip()
|
||||
expiry_date = parse_date(expires_at) if expires_at else None
|
||||
common_ok = (
|
||||
approval.get("decision") == "approved"
|
||||
and bool(str(approval.get("reviewer", "")).strip())
|
||||
and bool(str(approval.get("scope", "")).strip())
|
||||
and bool(str(approval.get("reason", "")).strip())
|
||||
and isinstance(approval.get("evidence"), list)
|
||||
and bool(approval.get("evidence"))
|
||||
and bool(enforcement)
|
||||
and expiry_date is not None
|
||||
and (generated_date is None or expiry_date >= generated_date)
|
||||
)
|
||||
add_check(
|
||||
checks,
|
||||
failures,
|
||||
f"permission-{target}-{capability}-approved",
|
||||
bool(common_ok),
|
||||
f"{target} capability {capability} has active reviewer approval",
|
||||
)
|
||||
target_enforcement_ok = bool(str(enforcement.get(target, "")).strip())
|
||||
add_check(
|
||||
checks,
|
||||
failures,
|
||||
f"permission-{target}-{capability}-target-enforcement",
|
||||
target_enforcement_ok,
|
||||
f"{target} capability {capability} has target enforcement note",
|
||||
)
|
||||
if common_ok and target_enforcement_ok:
|
||||
enforced_count += 1
|
||||
|
||||
return {
|
||||
"installer_permission_enforced_count": enforced_count,
|
||||
"installer_permission_failure_count": len(failures) - permission_failure_start,
|
||||
"permission_target_count": len(targets),
|
||||
"permission_capability_count": len(declared_capabilities),
|
||||
}
|
||||
|
||||
|
||||
def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | None, generated_at: str) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
package_dir = package_dir.resolve()
|
||||
checks: list[dict[str, str]] = []
|
||||
failures: list[str] = []
|
||||
warnings: list[str] = []
|
||||
archive_path = package_dir / f"{skill_dir.name}.zip"
|
||||
package_manifest = load_json(package_dir / "manifest.json")
|
||||
installed_dir = Path("")
|
||||
package_root = package_name(package_manifest, skill_dir)
|
||||
archive_path = package_dir / f"{package_root}.zip"
|
||||
installed_dir: Path | None = None
|
||||
archive_entries: list[str] = []
|
||||
|
||||
temp_dir: tempfile.TemporaryDirectory[str] | None = None
|
||||
@@ -117,7 +206,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No
|
||||
else:
|
||||
requested_root = install_root.resolve()
|
||||
requested_root.mkdir(parents=True, exist_ok=True)
|
||||
install_base = requested_root / f"simulate-{skill_dir.name}"
|
||||
install_base = requested_root / f"simulate-{package_root}"
|
||||
if install_base.exists():
|
||||
shutil.rmtree(install_base)
|
||||
install_base.mkdir(parents=True, exist_ok=True)
|
||||
@@ -134,7 +223,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No
|
||||
unsafe_entries = unsafe_zip_entries(archive_entries)
|
||||
add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries")
|
||||
roots = top_level_dirs(archive_entries)
|
||||
add_check(checks, failures, "single-top-level", roots == [skill_dir.name], f"Archive top-level directory is {skill_dir.name}")
|
||||
add_check(checks, failures, "single-top-level", roots == [package_root], f"Archive top-level directory is {package_root}")
|
||||
if not unsafe_entries and roots:
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
archive.extractall(install_base)
|
||||
@@ -144,18 +233,31 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No
|
||||
source_manifest = load_json(installed_dir / "manifest.json") if installed_dir else {}
|
||||
interface_doc = load_yaml(installed_dir / "agents" / "interface.yaml") if installed_dir else {}
|
||||
add_check(checks, failures, "entrypoint-load", bool(frontmatter), "Installed SKILL.md frontmatter is readable")
|
||||
add_check(checks, failures, "entrypoint-name", frontmatter.get("name") == skill_dir.name, "Installed SKILL.md name matches package directory")
|
||||
add_check(checks, failures, "entrypoint-name", frontmatter.get("name") == package_root, "Installed SKILL.md name matches package directory")
|
||||
add_check(checks, failures, "entrypoint-description", bool(frontmatter.get("description")), "Installed SKILL.md description is present")
|
||||
add_check(checks, failures, "manifest-load", bool(source_manifest), "Installed manifest.json is readable")
|
||||
add_check(checks, failures, "manifest-name", source_manifest.get("name") == package_manifest.get("name"), "Installed manifest name matches package manifest")
|
||||
add_check(checks, failures, "manifest-version", source_manifest.get("version") == package_manifest.get("version"), "Installed manifest version matches package manifest")
|
||||
add_check(checks, failures, "interface-load", bool(interface_doc.get("interface")), "Installed agents/interface.yaml is readable")
|
||||
add_check(checks, failures, "overview-report", (installed_dir / "reports" / "skill-overview.html").exists(), "Installed overview report is present")
|
||||
add_check(checks, failures, "review-studio-report", (installed_dir / "reports" / "review-studio.html").exists(), "Installed Review Studio report is present")
|
||||
add_check(
|
||||
checks,
|
||||
failures,
|
||||
"overview-report",
|
||||
installed_dir is not None and (installed_dir / "reports" / "skill-overview.html").exists(),
|
||||
"Installed overview report is present",
|
||||
)
|
||||
add_check(
|
||||
checks,
|
||||
failures,
|
||||
"review-studio-report",
|
||||
installed_dir is not None and (installed_dir / "reports" / "review-studio.html").exists(),
|
||||
"Installed Review Studio report is present",
|
||||
)
|
||||
|
||||
adapters = adapter_targets(package_dir)
|
||||
adapter_root = package_dir
|
||||
adapters = adapter_targets(adapter_root)
|
||||
for target in adapters:
|
||||
adapter = load_json(package_dir / "targets" / target / "adapter.json")
|
||||
adapter = load_json(adapter_root / "targets" / target / "adapter.json")
|
||||
add_check(checks, failures, f"adapter-{target}", bool(adapter), f"{target} adapter is readable after package install simulation")
|
||||
add_check(
|
||||
checks,
|
||||
@@ -165,6 +267,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No
|
||||
f"{target} adapter name matches package manifest",
|
||||
)
|
||||
|
||||
permission_summary = permission_policy_checks(installed_dir, adapter_root, adapters, generated_at, checks, failures)
|
||||
if not adapters:
|
||||
warnings.append("No target adapters found in package directory.")
|
||||
|
||||
@@ -191,6 +294,7 @@ def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | No
|
||||
"manifest_loaded": bool(source_manifest),
|
||||
"interface_loaded": bool(interface_doc.get("interface")),
|
||||
"adapter_count": len(adapters),
|
||||
**permission_summary,
|
||||
"install_root_is_temp": install_root_is_temp,
|
||||
"failure_count": len(failures),
|
||||
"warning_count": len(warnings),
|
||||
@@ -221,6 +325,8 @@ def render_markdown(report: dict[str, Any]) -> str:
|
||||
f"- Manifest loaded: `{summary['manifest_loaded']}`",
|
||||
f"- Interface loaded: `{summary['interface_loaded']}`",
|
||||
f"- Adapters readable: `{summary['adapter_count']}`",
|
||||
f"- Installer permissions enforced: `{summary.get('installer_permission_enforced_count', 0)}`",
|
||||
f"- Installer permission failures: `{summary.get('installer_permission_failure_count', 0)}`",
|
||||
f"- Failures: `{summary['failure_count']}`",
|
||||
f"- Warnings: `{summary['warning_count']}`",
|
||||
"",
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared Skill IR artifact discovery helpers."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by compiler, registry, conformance, and report scripts to locate canonical Skill IR artifacts."
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def candidate_paths(skill_dir: Path, name: str) -> list[Path]:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "skill-ir.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{name}.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
|
||||
]
|
||||
examples_dir = skill_dir / "skill-ir" / "examples"
|
||||
if examples_dir.exists():
|
||||
for path in sorted(examples_dir.glob("*.json")):
|
||||
if path not in candidates:
|
||||
candidates.append(path)
|
||||
seen: set[Path] = set()
|
||||
unique: list[Path] = []
|
||||
for path in candidates:
|
||||
if path in seen:
|
||||
continue
|
||||
seen.add(path)
|
||||
unique.append(path)
|
||||
return unique
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def find_skill_ir_path(skill_dir: Path, name: str, *, require_schema: bool = False, fallback_source: str = "") -> str:
|
||||
for path in candidate_paths(skill_dir, name):
|
||||
payload = load_json(path)
|
||||
if not payload:
|
||||
continue
|
||||
if require_schema and not payload.get("schema_version"):
|
||||
continue
|
||||
return display_path(path, skill_dir)
|
||||
return fallback_source
|
||||
|
||||
|
||||
def find_skill_ir(
|
||||
skill_dir: Path,
|
||||
name: str,
|
||||
*,
|
||||
require_schema: bool = False,
|
||||
fallback_source: str = "",
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
for path in candidate_paths(skill_dir, name):
|
||||
payload = load_json(path)
|
||||
if not payload:
|
||||
continue
|
||||
if require_schema and not payload.get("schema_version"):
|
||||
continue
|
||||
return payload, display_path(path, skill_dir)
|
||||
return {}, fallback_source
|
||||
@@ -0,0 +1,101 @@
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_skill_os2_coverage.py to keep coverage data assembly separate from Markdown rendering."
|
||||
|
||||
|
||||
def render_table(items: list[dict[str, Any]]) -> list[str]:
|
||||
lines = ["| Item | Status | Current | Command | Test |", "| --- | --- | --- | --- | --- |"]
|
||||
for item in items:
|
||||
lines.append(
|
||||
"| "
|
||||
+ " | ".join(
|
||||
[
|
||||
item["label"].replace("|", "\\|"),
|
||||
f"`{item['status']}`",
|
||||
item["current"].replace("|", "\\|"),
|
||||
f"`{item['command']}`",
|
||||
f"`{item['test']}`",
|
||||
]
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
return lines
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
lines = [
|
||||
"# Skill OS 2.0 Blueprint Coverage",
|
||||
"",
|
||||
f"Generated at: `{report['generated_at']}`",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
f"- decision: `{summary['decision']}`",
|
||||
f"- local blueprint ready: `{str(summary['local_blueprint_ready']).lower()}`",
|
||||
f"- public world-class ready: `{str(summary['public_world_class_ready']).lower()}`",
|
||||
f"- pass: `{summary['pass_count']}` / `{summary['item_count']}`",
|
||||
f"- missing: `{summary['missing_count']}`",
|
||||
f"- warn: `{summary['warn_count']}`",
|
||||
f"- reference extensions: `{summary['extension_track_count']}`",
|
||||
f"- extension covered: `{summary['extension_covered_count']}`",
|
||||
f"- extension partial: `{summary['extension_partial_count']}`",
|
||||
f"- extension planned: `{summary['extension_planned_count']}`",
|
||||
f"- adaptive extension ready: `{str(summary['adaptive_extension_ready']).lower()}`",
|
||||
f"- world-class evidence pending: `{summary['world_class_evidence_pending_count']}`",
|
||||
"",
|
||||
"This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, commands, and tests. It does not count pending human review, provider runs, metadata fallbacks, or planned work as public world-class evidence.",
|
||||
"",
|
||||
"## Core Modules",
|
||||
"",
|
||||
*render_table(report["modules"]),
|
||||
"",
|
||||
"## Recommended PR Coverage",
|
||||
"",
|
||||
*render_table(report["recommended_prs"]),
|
||||
"",
|
||||
"## Reference Extension Tracks",
|
||||
"",
|
||||
"| Track | Status | Current | Target | Next action |",
|
||||
"| --- | --- | --- | --- | --- |",
|
||||
]
|
||||
for item in report["reference_extension_tracks"]:
|
||||
lines.append(
|
||||
"| "
|
||||
+ " | ".join(
|
||||
[
|
||||
item["label"].replace("|", "\\|"),
|
||||
f"`{item['status']}`",
|
||||
item["current"].replace("|", "\\|"),
|
||||
item["target"].replace("|", "\\|"),
|
||||
item["next_action"].replace("|", "\\|"),
|
||||
]
|
||||
)
|
||||
+ " |"
|
||||
)
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"These extension tracks come from the user-supplied 2.0 reference plan. They are tracked separately from the formal Skill OS blueprint so the report can distinguish landed local architecture from planned explainer/adaptor evolution.",
|
||||
"",
|
||||
"## Next Highest-Leverage Moves",
|
||||
"",
|
||||
]
|
||||
)
|
||||
lines.extend(f"- {item}" for item in report["next_highest_leverage"])
|
||||
lines.extend(["", "## Evidence Detail", ""])
|
||||
for item in report["modules"] + report["recommended_prs"] + report["reference_extension_tracks"]:
|
||||
existing = [entry["path"] for entry in item["evidence"] if entry["exists"]]
|
||||
missing = [entry["path"] for entry in item["evidence"] if not entry["exists"]]
|
||||
lines.append(f"### {item['label']}")
|
||||
lines.append("")
|
||||
lines.append(f"- objective: {item['objective']}")
|
||||
lines.append(f"- status: `{item['status']}`")
|
||||
lines.append(f"- existing evidence: {', '.join(f'`{path}`' for path in existing) if existing else '`none`'}")
|
||||
if missing:
|
||||
lines.append(f"- missing evidence: {', '.join(f'`{path}`' for path in missing)}")
|
||||
lines.append(f"- next action: {item['next_action']}")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
@@ -2,6 +2,8 @@
|
||||
import html
|
||||
import math
|
||||
|
||||
from skill_report_i18n import bi_span, en_for
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to render inline SVG report charts."
|
||||
@@ -17,11 +19,26 @@ def esc(value) -> str:
|
||||
return html.escape(str(value))
|
||||
|
||||
|
||||
def figure(name: str, svg: str, caption: str) -> str:
|
||||
def svg_text(zh: str, en: str | None = None, **attrs) -> str:
|
||||
def attr_name(key: str) -> str:
|
||||
if key.endswith("_"):
|
||||
key = key[:-1]
|
||||
return key.replace("_", "-")
|
||||
|
||||
attr = " ".join(f'{attr_name(key)}="{esc(value)}"' for key, value in attrs.items() if value is not None)
|
||||
if attr:
|
||||
attr = " " + attr
|
||||
return (
|
||||
f'<text data-lang="zh-CN"{attr}>{esc(zh)}</text>'
|
||||
f'<text data-lang="en"{attr}>{esc(en_for(en if en is not None else zh))}</text>'
|
||||
)
|
||||
|
||||
|
||||
def figure(name: str, svg: str, caption: str, caption_en: str | None = None) -> str:
|
||||
return (
|
||||
f'<figure class="chart-figure" data-chart="{esc(name)}">'
|
||||
f"{svg}"
|
||||
f"<figcaption>{esc(caption)}</figcaption>"
|
||||
f"<figcaption>{bi_span(caption, caption_en)}</figcaption>"
|
||||
"</figure>"
|
||||
)
|
||||
|
||||
@@ -47,12 +64,10 @@ def render_radar(scorecard: dict) -> str:
|
||||
data_points.append(f"{center + data_radius * math.cos(angle):.1f},{center + data_radius * math.sin(angle):.1f}")
|
||||
lx = center + (radius + 32) * math.cos(angle)
|
||||
ly = center + (radius + 32) * math.sin(angle)
|
||||
label_nodes.append(
|
||||
f'<text x="{lx:.1f}" y="{ly:.1f}" text-anchor="middle" dominant-baseline="middle">{esc(labels[i])}</text>'
|
||||
)
|
||||
label_nodes.append(svg_text(labels[i], x=f"{lx:.1f}", y=f"{ly:.1f}", text_anchor="middle", dominant_baseline="middle"))
|
||||
svg = (
|
||||
'<svg viewBox="0 0 300 300" role="img" aria-label="评分雷达">'
|
||||
f'<text x="20" y="28" class="chart-title">评分雷达</text>'
|
||||
+ svg_text("评分雷达", x="20", y="28", class_="chart-title")
|
||||
+ "".join(rings)
|
||||
+ f'<polygon points="{" ".join(data_points)}" fill="#E4ECF5" stroke="{BRAND}" stroke-width="2"/>'
|
||||
+ "".join(label_nodes)
|
||||
@@ -68,12 +83,12 @@ def render_flow(summary: dict) -> str:
|
||||
x = 38 + index * 210
|
||||
nodes.append(
|
||||
f'<g><rect x="{x}" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="{BORDER}"/>'
|
||||
f'<text x="{x + 75}" y="99" text-anchor="middle">{esc(label)}</text></g>'
|
||||
f'{svg_text(label, x=str(x + 75), y="99", text_anchor="middle")}</g>'
|
||||
)
|
||||
svg = (
|
||||
'<svg viewBox="0 0 620 170" role="img" aria-label="交付流程">'
|
||||
'<text x="20" y="28" class="chart-title">交付流程</text>'
|
||||
'<path d="M188 93 H248 M398 93 H458" class="chart-line"/>'
|
||||
+ svg_text("交付流程", x="20", y="28", class_="chart-title")
|
||||
+ '<path d="M188 93 H248 M398 93 H458" class="chart-line"/>'
|
||||
+ "".join(nodes)
|
||||
+ "</svg>"
|
||||
)
|
||||
@@ -86,15 +101,15 @@ def render_matrix(profile: dict) -> str:
|
||||
y = 430 - matrix.get("knowledge_density", 60) * 3.2
|
||||
svg = (
|
||||
'<svg viewBox="0 0 520 460" role="img" aria-label="能力矩阵">'
|
||||
'<text x="20" y="30" class="chart-title">能力矩阵</text>'
|
||||
f'<rect x="70" y="70" width="380" height="320" fill="{SOFT}" stroke="{BORDER}"/>'
|
||||
f'<line x1="260" y1="70" x2="260" y2="390" stroke="{BORDER}"/>'
|
||||
f'<line x1="70" y1="230" x2="450" y2="230" stroke="{BORDER}"/>'
|
||||
'<text x="260" y="424" text-anchor="middle">执行确定性</text>'
|
||||
'<text x="22" y="230" transform="rotate(-90 22 230)" text-anchor="middle">知识密度</text>'
|
||||
f'<circle cx="{x:.1f}" cy="{y:.1f}" r="12" fill="{BRAND}"/>'
|
||||
f'<text x="{x + 18:.1f}" y="{y + 5:.1f}">{esc(profile.get("task_family", "Skill workflow"))}</text>'
|
||||
"</svg>"
|
||||
+ svg_text("能力矩阵", x="20", y="30", class_="chart-title")
|
||||
+ f'<rect x="70" y="70" width="380" height="320" fill="{SOFT}" stroke="{BORDER}"/>'
|
||||
+ f'<line x1="260" y1="70" x2="260" y2="390" stroke="{BORDER}"/>'
|
||||
+ f'<line x1="70" y1="230" x2="450" y2="230" stroke="{BORDER}"/>'
|
||||
+ svg_text("执行确定性", x="260", y="424", text_anchor="middle")
|
||||
+ svg_text("知识密度", x="22", y="230", transform="rotate(-90 22 230)", text_anchor="middle")
|
||||
+ f'<circle cx="{x:.1f}" cy="{y:.1f}" r="12" fill="{BRAND}"/>'
|
||||
+ svg_text(str(profile.get("task_family", "Skill workflow")), x=f"{x + 18:.1f}", y=f"{y + 5:.1f}")
|
||||
+ "</svg>"
|
||||
)
|
||||
return figure("matrix", svg, "能力矩阵说明这个 Skill 更偏知识密集还是执行确定。")
|
||||
|
||||
@@ -106,11 +121,11 @@ def render_layers(principle: dict) -> str:
|
||||
y = 55 + index * 48
|
||||
blocks.append(
|
||||
f'<rect x="{70 + index * 18}" y="{y}" width="{380 - index * 36}" height="34" rx="7" fill="#F6F8FB" stroke="{BORDER}"/>'
|
||||
f'<text x="260" y="{y + 22}" text-anchor="middle">{esc(layer)}</text>'
|
||||
+ svg_text(layer, x="260", y=str(y + 22), text_anchor="middle")
|
||||
)
|
||||
svg = (
|
||||
'<svg viewBox="0 0 520 320" role="img" aria-label="Skill principle flow">'
|
||||
'<text x="20" y="30" class="chart-title">分层结构</text>'
|
||||
+ svg_text("分层结构", x="20", y="30", class_="chart-title")
|
||||
+ "".join(blocks)
|
||||
+ "</svg>"
|
||||
)
|
||||
@@ -132,10 +147,10 @@ def render_risk_heatmap(risk: dict) -> str:
|
||||
)
|
||||
svg = (
|
||||
'<svg viewBox="0 0 380 300" role="img" aria-label="风险热力">'
|
||||
'<text x="20" y="30" class="chart-title">风险热力</text>'
|
||||
+ svg_text("风险热力", x="20", y="30", class_="chart-title")
|
||||
+ "".join(cells)
|
||||
+ '<text x="210" y="278" text-anchor="middle">发生概率</text>'
|
||||
+ '<text x="24" y="160" transform="rotate(-90 24 160)" text-anchor="middle">影响程度</text>'
|
||||
+ svg_text("发生概率", x="210", y="278", text_anchor="middle")
|
||||
+ svg_text("影响程度", x="24", y="160", transform="rotate(-90 24 160)", text_anchor="middle")
|
||||
+ "</svg>"
|
||||
)
|
||||
return figure("risk_heatmap", svg, "风险热力图用影响程度和发生概率标出当前治理重点。")
|
||||
@@ -157,12 +172,12 @@ def render_asset_donut(assets: dict) -> str:
|
||||
'pathLength="100" transform="rotate(-90 130 130)"/>'
|
||||
)
|
||||
offset += dash
|
||||
labels.append(f'<text x="235" y="{78 + index * 22}">{esc(item.get("label", "asset"))}</text>')
|
||||
labels.append(svg_text(str(item.get("label", "asset")), x="235", y=str(78 + index * 22)))
|
||||
svg = (
|
||||
'<svg viewBox="0 0 430 270" role="img" aria-label="资产分布">'
|
||||
'<text x="20" y="30" class="chart-title">资产分布</text>'
|
||||
+ svg_text("资产分布", x="20", y="30", class_="chart-title")
|
||||
+ "".join(circles)
|
||||
+ f'<text x="130" y="136" text-anchor="middle">{assets.get("file_count", 0)}项</text>'
|
||||
+ svg_text(f'{assets.get("file_count", 0)}项', f'{assets.get("file_count", 0)} items', x="130", y="136", text_anchor="middle")
|
||||
+ "".join(labels)
|
||||
+ "</svg>"
|
||||
)
|
||||
@@ -177,15 +192,18 @@ def render_timeline(roadmap: dict) -> str:
|
||||
title = str(item.get("title", "升级"))
|
||||
if len(title) > 18:
|
||||
title = title[:17] + "…"
|
||||
title_en = en_for(title)
|
||||
if len(title_en) > 24:
|
||||
title_en = title_en[:23] + "..."
|
||||
blocks.append(
|
||||
f'<circle cx="{x}" cy="92" r="10" fill="{BRAND}"/>'
|
||||
f'<text x="{x}" y="126" text-anchor="middle">下一步 {index + 1}</text>'
|
||||
f'<text x="{x}" y="150" text-anchor="middle">{esc(title)}</text>'
|
||||
+ svg_text(f"下一步 {index + 1}", f"Next {index + 1}", x=str(x), y="126", text_anchor="middle")
|
||||
+ svg_text(title, title_en, x=str(x), y="150", text_anchor="middle")
|
||||
)
|
||||
svg = (
|
||||
'<svg viewBox="0 0 520 210" role="img" aria-label="迭代时间">'
|
||||
'<text x="20" y="30" class="chart-title">迭代时间</text>'
|
||||
f'<line x1="60" y1="92" x2="{60 + max(0, len(items) - 1) * 190}" y2="92" class="chart-line"/>'
|
||||
+ svg_text("迭代时间", x="20", y="30", class_="chart-title")
|
||||
+ f'<line x1="60" y1="92" x2="{60 + max(0, len(items) - 1) * 190}" y2="92" class="chart-line"/>'
|
||||
+ "".join(blocks)
|
||||
+ "</svg>"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,386 @@
|
||||
"""Bilingual text helpers for the static skill overview report."""
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to keep bilingual report copy and fallback rules out of HTML rendering."
|
||||
|
||||
|
||||
TEXT_ZH = {
|
||||
"Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.": "从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill;适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。",
|
||||
"Understand the request.": "理解用户请求。",
|
||||
"Execute the main task.": "执行核心任务。",
|
||||
"Validate the result.": "校验交付结果。",
|
||||
"Understand the request": "理解用户请求。",
|
||||
"Execute the main task": "执行核心任务。",
|
||||
"Validate the result": "校验交付结果。",
|
||||
"Decide whether the request should become a skill and choose the lightest fit.": "判断请求是否应该沉淀为 Skill,并选择最轻量可靠的模式。",
|
||||
"Capture job, output, exclusions, constraints, and standards.": "捕捉任务、输出、排除项、约束和质量标准。",
|
||||
"Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.": "运行参考扫描:先看外部 benchmark,再看用户材料,最后校验本地适配;只暴露不确定性或冲突。",
|
||||
"Write the `description` early and test route quality before expanding the package.": "尽早写出 `description`,先测试路由质量,再扩展包体。",
|
||||
"Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.": "只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。",
|
||||
"Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.": "当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。",
|
||||
"Turn rough requests into a compact reusable demo skill.": "把粗糙请求整理成紧凑、可复用的演示 Skill。",
|
||||
"Tighten trigger and exclusions": "收紧触发与排除边界",
|
||||
"Add the first execution asset": "补上第一个执行资产",
|
||||
"Promote from scaffold to production-ready": "从脚手架推进到生产可用",
|
||||
"Borrow one proven pattern on purpose": "有选择地借鉴一个成熟模式",
|
||||
"Harden portability semantics": "加固跨环境语义",
|
||||
"Create an iteration evidence loop": "建立迭代证据回路",
|
||||
"补齐世界证据": "补齐世界证据",
|
||||
"The package needs clearer near-neighbor exclusions before it grows.": "在继续扩展前,需要先把相邻但不应触发的场景说清楚。",
|
||||
"The package is still mostly prose. Add one asset that removes repeated manual work.": "当前包体仍偏文本说明,应先增加一个能减少重复人工操作的资产。",
|
||||
"The first version exists; the next gain usually comes from adding the smallest useful gates.": "第一版已经存在,下一步收益通常来自补上最小但有效的质量门禁。",
|
||||
"You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.": "已经有公开 benchmark 对象,下一步应主动选择一个模式借鉴,而不是松散吸收所有做法。",
|
||||
"The skill already signals reuse across environments, so contract clarity matters early.": "这个 Skill 已经面向跨环境复用,因此早期就需要把契约语义说清楚。",
|
||||
"The package should show what changed and why after the first draft.": "第一版之后,包体应该能说明改了什么以及为什么改。",
|
||||
"Add 3 to 5 should-trigger and should-not-trigger examples.": "增加 3 到 5 个应触发和不应触发的例子。",
|
||||
"Refine the frontmatter description to name the recurring job and non-goals.": "精炼 frontmatter description,明确重复任务和非目标。",
|
||||
"Run a first trigger evaluation pass before expanding the package.": "扩展包体前先跑一轮触发评估。",
|
||||
"Move stable procedural guidance into references if users will need it repeatedly.": "如果用户会反复使用某段流程说明,把它沉淀到 references。",
|
||||
"Create one deterministic helper script if a repeated step can be executed instead of described.": "如果某个重复步骤可以执行而不是描述,就沉淀成一个确定性 helper script。",
|
||||
"Keep the main SKILL.md compact and route-oriented.": "保持主 SKILL.md 简洁,并围绕路由与入口组织。",
|
||||
"Decide whether this skill is personal, team-reused, or library-grade.": "判断这个 Skill 是个人使用、团队复用,还是库级基础能力。",
|
||||
"Add only the gates that match that risk level.": "只添加与风险等级匹配的质量门禁。",
|
||||
"Record lifecycle metadata and review cadence once reuse becomes real.": "一旦进入真实复用,就记录生命周期元数据和评审节奏。",
|
||||
"Decide whether to borrow method, structure, execution, or portability, but only one of them first.": "先判断要借鉴的是方法、结构、执行方式还是可迁移性,并且第一轮只借鉴其中一个。",
|
||||
"Record what you will not borrow so the package stays light.": "记录本轮不借鉴的内容,避免包体过重。",
|
||||
"Confirm activation mode, execution context, and trust assumptions.": "确认激活模式、执行上下文和信任假设。",
|
||||
"Add or review degradation strategy for non-native targets.": "补充或复核非原生目标端的降级策略。",
|
||||
"Package the skill once to verify adapter expectations.": "至少打包一次 Skill,用来验证 adapter 预期。",
|
||||
"Generate the HTML skill report and keep it aligned with the package.": "生成 HTML Skill 报告,并保持它与包体内容一致。",
|
||||
"Record reference scan choices and non-goals.": "记录参考扫描的取舍和非目标。",
|
||||
"Capture the next iteration choice explicitly before adding more files.": "在继续增加文件前,明确记录下一轮迭代选择。",
|
||||
"Cleaner routing and fewer accidental activations.": "路由更清晰,误触发更少。",
|
||||
"Stronger execution quality without bloating the entrypoint.": "在不膨胀入口文件的前提下提升执行质量。",
|
||||
"A clearer path from exploratory package to maintained asset.": "更清晰地从探索性包体走向可维护资产。",
|
||||
"A cleaner package shape with less accidental over-design.": "包体形态更清晰,也减少偶然过度设计。",
|
||||
"Safer cross-environment reuse with less target drift.": "跨环境复用更安全,目标漂移更少。",
|
||||
"A clearer path for the next author or reviewer.": "让下一位作者或评审者更容易接手。",
|
||||
"提交有效 intake packet,并让 ledger 通过 artifact SHA-256 校验。": "提交有效 intake packet,并让 ledger 通过 artifact SHA-256 校验。",
|
||||
"全部外部/人工证据被 ledger 接受后,才能进入公开 world-class claim 复核。": "全部外部/人工证据被 ledger 接受后,才能进入公开 world-class claim 复核。",
|
||||
}
|
||||
|
||||
TEXT_EN = {
|
||||
"把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。": "Turn one-off experience into a reusable, evaluable, and portable skill package.",
|
||||
"Skill 作者、复用团队和后续 reviewer。": "Skill authors, reuse teams, and later reviewers.",
|
||||
"创建完成后建议先打开 reports/skill-overview.html,再继续扩展包体。": "After creation, open reports/skill-overview.html before expanding the package further.",
|
||||
"触发面保持精简,并锚定在 frontmatter description。": "The trigger surface stays lean and anchored in the frontmatter description.",
|
||||
"已生成 Skill IR,核心语义可先于平台打包被审查和迁移。": "Skill IR is generated so core semantics can be reviewed and migrated before platform packaging.",
|
||||
"已生成目标编译报告,可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。": "Target compilation evidence is generated to review how IR maps to OpenAI, Claude, generic, and other target contracts.",
|
||||
"已生成 Output Eval Lab scorecard,可比较 with-skill 与 baseline 输出质量。": "Output Eval Lab scorecard is generated to compare with-skill and baseline output quality.",
|
||||
"已生成 Output Execution Runs,可区分记录样本、命令执行和模型执行证据。": "Output Execution Runs is generated to distinguish recorded fixtures, command runs, and model-run evidence.",
|
||||
"已生成 Output Review Adjudication,可记录盲评决策、一致率和待评审项。": "Output Review Adjudication is generated to record blind-review decisions, agreement rate, and pending cases.",
|
||||
"已生成 Runtime Conformance Matrix,可审查目标端消费能力。": "Runtime Conformance Matrix is generated to review target-side consumption capability.",
|
||||
"已生成 Security Trust Report,可审查脚本、依赖、secret 和包完整性风险。": "Security Trust Report is generated to review scripts, dependencies, secrets, and package-integrity risk.",
|
||||
"已生成 Skill Atlas,可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。": "Skill Atlas is generated to review route collisions, stale assets, and owner gaps across a skill library.",
|
||||
"已生成 Registry Audit,可审查版本、owner、license、checksum 和目标兼容矩阵。": "Registry Audit is generated to review version, owner, license, checksum, and target compatibility metadata.",
|
||||
"已生成 Install Simulation,可审查 zip 解压、入口加载、接口元数据和 adapter 可读性。": "Install Simulation is generated to review zip extraction, entrypoint loading, interface metadata, and adapter readability.",
|
||||
"已生成 Adoption Drift Report,可把本地使用反馈转为下一轮迭代信号。": "Adoption Drift Report is generated to turn local usage feedback into next-iteration signals.",
|
||||
"已生成 Review Waivers 台账,可记录 reviewer 对 warning 风险的批准、理由和到期时间。": "Review Waivers ledger is generated to record reviewer approval, rationale, scope, and expiry for accepted warning risk.",
|
||||
"已生成 Review Annotations 台账,可把 reviewer 批注挂到 gate、文件和行号。": "Review Annotations ledger is generated to attach reviewer notes to gates, files, and line numbers.",
|
||||
"已生成 Review Studio 2.0,可在一页中查看 blocker、warning、证据路径和发布闸门。": "Review Studio 2.0 is generated to inspect blockers, warnings, evidence paths, and release gates on one page.",
|
||||
"已打包 agents/interface.yaml,便于后续做跨平台适配。": "Portable interface metadata is packaged for later adapter-based export.",
|
||||
"长指导被拆到 references 中,入口文件可以保持轻量。": "Extended guidance is separated into references so the entrypoint can stay compact.",
|
||||
"确定性辅助逻辑放在 scripts 中,而不是藏在提示词里。": "Deterministic helper logic lives in scripts instead of hidden prompt text.",
|
||||
"包内包含可随 Skill 迁移的质量门禁或触发检查。": "The package includes portable quality gates or trigger checks.",
|
||||
"这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。": "Use this report to quickly understand the generated skill's role, principles, trigger boundary, and deliverables.",
|
||||
"先确认重复任务、真实输入形态和可交付输出,再决定是否继续加 references、scripts 或 evals。": "Clarify the recurring job, real input shape, and deliverable output before adding references, scripts, or evals.",
|
||||
"如果需求仍然模糊,优先回到 intent dialogue 收紧边界,再扩展包体结构。": "If the request is still fuzzy, tighten the boundary through intent dialogue before expanding the package.",
|
||||
"尚未生成盲评审定报告。": "The blind review adjudication report has not been generated yet.",
|
||||
"尚未生成输出执行证据报告。": "The output execution evidence report has not been generated yet.",
|
||||
"先记录 reviewer 对 A/B 的选择,再打开答案 key 计算一致率。": "Record the reviewer's A/B choice before opening the answer key and calculating agreement.",
|
||||
"缺少真实 reviewer 决策时只显示待评审,不伪造人工结论。": "When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.",
|
||||
"recorded fixture 只能证明可复现样本,不等同于模型执行。": "A recorded fixture proves reproducible samples only; it is not model execution.",
|
||||
"只有 provider runner 返回 model metadata 时才计入 model-executed。": "Only provider runners that return model metadata count as model-executed.",
|
||||
"SKILL.md 已存在,是 Skill 的入口。": "SKILL.md exists and acts as the skill entrypoint.",
|
||||
"README.md 已存在,便于人工阅读。": "README.md exists for human-readable usage.",
|
||||
"agents/interface.yaml 已存在,便于跨平台适配。": "agents/interface.yaml exists for cross-platform adaptation.",
|
||||
"manifest.json 已存在,生命周期信息可追踪。": "manifest.json exists so lifecycle metadata is traceable.",
|
||||
"reports/ 已存在,生成证据可以随包体迁移。": "reports/ exists so generated evidence can travel with the package.",
|
||||
"references/ 已存在,长指导可以从入口文件拆出。": "references/ exists so long guidance can stay out of the entrypoint.",
|
||||
"scripts/ 已存在,确定性逻辑有位置承载。": "scripts/ exists to hold deterministic logic.",
|
||||
"evals/ 已存在,触发或质量检查可以随包体迁移。": "evals/ exists so trigger or quality checks can travel with the package.",
|
||||
"frontmatter description 已存在,具备基础路由面。": "The frontmatter description exists, giving the skill a basic routing surface.",
|
||||
"description 有足够长度说明任务边界。": "The description is long enough to explain the task boundary.",
|
||||
"description 已包含使用场景或排除边界信号。": "The description includes usage-scenario or exclusion-boundary signals.",
|
||||
"description 证据不足,触发边界不稳定。": "Description evidence is insufficient, so the trigger boundary is unstable.",
|
||||
"description 偏短,建议补充输入、输出或非目标。": "The description is short; add inputs, outputs, or non-goals.",
|
||||
"description 缺少明确使用场景或排除边界。": "The description lacks clear usage scenarios or exclusion boundaries.",
|
||||
"evals/ 已存在,可承载触发样例或质量检查。": "evals/ exists and can hold trigger examples or quality checks.",
|
||||
"evals/ 证据不足,误触发检查仍偏弱。": "evals/ evidence is insufficient, so accidental-trigger checks remain weak.",
|
||||
"intent-confidence 报告已生成,可辅助判断触发稳定性。": "The intent-confidence report exists and helps judge trigger stability.",
|
||||
"intent-confidence 证据不足。": "intent-confidence evidence is insufficient.",
|
||||
"入口文件保持克制,可维护性较好。": "The entrypoint stays restrained, which supports maintainability.",
|
||||
"入口文件偏长,建议继续拆到 references/。": "The entrypoint is long; continue splitting durable guidance into references/.",
|
||||
"references/ 已承载扩展指导。": "references/ carries extended guidance.",
|
||||
"references/ 证据不足,长指导可能堆在入口。": "references/ evidence is insufficient; long guidance may still be crowded into the entrypoint.",
|
||||
"scripts/ 已承载确定性逻辑。": "scripts/ carries deterministic logic.",
|
||||
"scripts/ 证据不足,重复执行逻辑可能仍靠人工。": "scripts/ evidence is insufficient; repeated execution may still rely on manual work.",
|
||||
"evals/ 已承载可迁移检查。": "evals/ carries portable checks.",
|
||||
"agents/interface.yaml 已存在。": "agents/interface.yaml exists.",
|
||||
"manifest.json 已存在。": "manifest.json exists.",
|
||||
"目标平台或 adapter target 已声明。": "Target platforms or adapter targets are declared.",
|
||||
"目标平台证据不足。": "Target-platform evidence is insufficient.",
|
||||
"入口文件未发现明显私有绝对路径。": "No obvious private absolute paths were found in the entrypoint.",
|
||||
"入口文件含私有绝对路径,迁移风险较高。": "The entrypoint contains private absolute paths, increasing portability risk.",
|
||||
"分数越高代表上下文成本越低。": "A higher score means lower context cost.",
|
||||
"上下文成本处于可控区间。": "Context cost is within a controlled range.",
|
||||
"上下文成本偏高,建议压缩入口或拆分 references。": "Context cost is high; compress the entrypoint or split references further.",
|
||||
"手动触发 + description 路由": "Manual activation plus description-based routing",
|
||||
"跨平台": "Cross-platform",
|
||||
"本地复用": "Local reuse",
|
||||
"输入材料": "Input material",
|
||||
"用户提供的工作流、提示词、文档、记录或散乱笔记": "User-provided workflows, prompts, documents, records, or rough notes.",
|
||||
"期望沉淀的复用场景、排除项、约束和质量标准": "The reusable scenario, exclusions, constraints, and quality standards to capture.",
|
||||
"Skill 包体": "Skill package",
|
||||
"可路由的 SKILL.md": "A routeable SKILL.md.",
|
||||
"agents/interface.yaml 元数据": "agents/interface.yaml metadata.",
|
||||
"必要的 references、scripts、evals、reports 证据": "Necessary references, scripts, evals, and reports evidence.",
|
||||
"可复用能力": "Reusable capability",
|
||||
"入口层": "Entrypoint layer",
|
||||
"参考层": "Reference layer",
|
||||
"脚本层": "Script layer",
|
||||
"评估层": "Evaluation layer",
|
||||
"报告层": "Report layer",
|
||||
"评分雷达": "Rating Radar",
|
||||
"交付流程": "Delivery Flow",
|
||||
"能力矩阵": "Capability Matrix",
|
||||
"分层结构": "Layered Structure",
|
||||
"风险热力": "Risk Heatmap",
|
||||
"资产分布": "Asset Distribution",
|
||||
"迭代时间": "Iteration Timeline",
|
||||
"执行确定性": "Execution certainty",
|
||||
"知识密度": "Knowledge density",
|
||||
"发生概率": "Probability",
|
||||
"影响程度": "Impact",
|
||||
"评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。": "The radar chart compares completeness, trigger clarity, evidence, maintainability, and portability.",
|
||||
"交付流程把用户输入、生成的包体和可复用能力放在一条线上。": "The delivery flow places user input, generated package, and reusable capability on one path.",
|
||||
"能力矩阵说明这个 Skill 更偏知识密集还是执行确定。": "The capability matrix shows whether the skill leans toward knowledge density or execution certainty.",
|
||||
"分层结构展示入口、参考、脚本、评估和报告如何各司其职。": "The layered structure shows how entrypoint, references, scripts, evals, and reports each carry a distinct role.",
|
||||
"风险热力图用影响程度和发生概率标出当前治理重点。": "The risk heatmap marks governance priorities by impact and probability.",
|
||||
"资产分布图展示当前包体的文件和目录重心。": "The asset distribution chart shows where files and directories are concentrated.",
|
||||
"迭代时间线把下一步升级收束成少数可执行动作。": "The iteration timeline narrows the next upgrade into a few executable moves.",
|
||||
"只需要一次性回答、没有复用价值的临时请求。": "One-off requests that do not need reusable skill behavior.",
|
||||
"要求直接执行相邻任务,而不是沉淀或使用这个 Skill。": "Requests to perform an adjacent task directly rather than create or use this skill.",
|
||||
"缺少必要事实且用户不允许澄清的场景。": "Cases that lack required facts and do not allow clarification.",
|
||||
"相邻任务需要先确认是否应转为独立 Skill。": "Adjacent tasks should first be checked for whether they need a separate skill.",
|
||||
"不替代人工事实核查,也不静默扩大职责。": "Does not replace human fact checking or silently expand responsibility.",
|
||||
"先改触发边界,再扩展工作流。": "Tighten trigger boundaries before expanding the workflow.",
|
||||
"只把重复且稳定的步骤沉淀为脚本。": "Turn only repeated and stable steps into scripts.",
|
||||
"每次升级后重新生成报告并检查分数原因。": "Regenerate the report after each upgrade and inspect score reasons.",
|
||||
"先补证据和边界,再增加包体复杂度。": "Improve evidence and boundaries before adding package complexity.",
|
||||
"补齐世界证据": "Close world-class evidence",
|
||||
"提交有效 intake packet,并让 ledger 通过 artifact SHA-256 校验。": "Submit valid intake packets and let the ledger verify artifact SHA-256 digests.",
|
||||
"全部外部/人工证据被 ledger 接受后,才能进入公开 world-class claim 复核。": "Only after the ledger accepts all external and human evidence should the public world-class claim move to review.",
|
||||
"缺少真实 provider 模型运行和 token metadata。": "Missing a real provider model run and token metadata.",
|
||||
"盲评 pair 仍待真实 reviewer 决策。": "Blind-review pairs still need real reviewer decisions.",
|
||||
"原生 runtime enforcement 仍待目标客户端或外部安装器证明。": "Native runtime enforcement still needs target-client or external-installer proof.",
|
||||
"真实外部客户端 metadata-only 事件仍未导入。": "Real external-client metadata-only events have not been imported yet.",
|
||||
}
|
||||
|
||||
MODE_ZH = {
|
||||
"scaffold": "脚手架",
|
||||
"production": "生产",
|
||||
"library": "库级",
|
||||
"governed": "治理",
|
||||
"manual": "手动",
|
||||
"inline": "内联",
|
||||
"agent-skills": "Agent Skills",
|
||||
}
|
||||
|
||||
PACKAGE_LABEL_ZH = {
|
||||
"SKILL.md": "Skill 入口文件",
|
||||
"README.md": "人类可读使用说明",
|
||||
"agents/interface.yaml": "跨平台接口元数据",
|
||||
"manifest.json": "生命周期与打包元数据",
|
||||
"references": "扩展指导与复用资料",
|
||||
"scripts": "确定性脚本或本地工具",
|
||||
"evals": "触发与质量检查",
|
||||
"reports": "生成的证据与总结报告",
|
||||
}
|
||||
|
||||
KIND_ZH = {"file": "文件", "folder": "目录"}
|
||||
|
||||
LABEL_EN = {
|
||||
"强项": "Strength",
|
||||
"缺口": "Gap",
|
||||
"保留并复用": "Keep",
|
||||
"纳入下一轮修复": "Fix next",
|
||||
"误触发风险": "Trigger risk",
|
||||
"输出漂移风险": "Output drift risk",
|
||||
"证据不足风险": "Evidence gap risk",
|
||||
"包体膨胀风险": "Package bloat risk",
|
||||
"跨平台迁移风险": "Portability risk",
|
||||
}
|
||||
|
||||
METRIC_LABEL_EN = {
|
||||
"完整度": "Completeness",
|
||||
"触发清晰": "Trigger clarity",
|
||||
"证据充分": "Evidence depth",
|
||||
"可维护性": "Maintainability",
|
||||
"可迁移性": "Portability",
|
||||
"上下文成本": "Context cost",
|
||||
}
|
||||
|
||||
WORLD_CLASS_LABEL_EN = {
|
||||
"提供商留出": "provider holdout",
|
||||
"人工盲评": "human adjudication",
|
||||
"原生权限": "native permission",
|
||||
"原生遥测": "native telemetry",
|
||||
}
|
||||
|
||||
|
||||
def contains_cjk(text: str) -> bool:
|
||||
return any("\u4e00" <= char <= "\u9fff" for char in str(text))
|
||||
|
||||
|
||||
def zh_for(text: str) -> str:
|
||||
value = str(text).strip()
|
||||
if not value:
|
||||
return ""
|
||||
if value in TEXT_ZH:
|
||||
return TEXT_ZH[value]
|
||||
if value in TEXT_EN or contains_cjk(value):
|
||||
return value
|
||||
if value.startswith("Use this skill when the request matches:"):
|
||||
return "当用户请求与该 Skill 的触发描述匹配时使用。"
|
||||
if value.startswith("用户说出类似需求时:"):
|
||||
return "当用户提出与该 Skill 触发描述相近的请求时使用。"
|
||||
if value.startswith("Use $") and " when you need to " in value:
|
||||
skill, need = value.removeprefix("Use ").split(" when you need to ", 1)
|
||||
return f"当你需要{zh_for(need).rstrip('。')}时使用 `{skill}`。"
|
||||
if value.startswith("Read the strongest pattern from "):
|
||||
repo = value.removeprefix("Read the strongest pattern from ").rstrip(".")
|
||||
return f"阅读 `{repo}` 中最值得借鉴的模式。"
|
||||
if value.startswith("Primary prompt task family:"):
|
||||
return "主要提示任务类型已记录在 prompt quality profile 中。"
|
||||
if value.startswith("Complexity:"):
|
||||
return "复杂度判断已记录在 prompt quality profile 中。"
|
||||
if value.startswith("Stability:"):
|
||||
return "系统稳定性评分已记录在 system model 中。"
|
||||
if value.startswith("Owned job:"):
|
||||
return "负责的核心任务已在 system model 中说明。"
|
||||
if value.startswith("Leverage:"):
|
||||
return "关键杠杆点已在 system model 中说明。"
|
||||
return "原始说明可切换到英文查看;默认中文报告保留结论与结构说明。"
|
||||
|
||||
|
||||
def en_for(text: str) -> str:
|
||||
value = str(text).strip()
|
||||
if not value:
|
||||
return ""
|
||||
if value in TEXT_EN:
|
||||
return TEXT_EN[value]
|
||||
if value in METRIC_LABEL_EN:
|
||||
return METRIC_LABEL_EN[value]
|
||||
if value.startswith("创建完成后建议先打开 ") and value.endswith(",再继续扩展包体。"):
|
||||
path = value.removeprefix("创建完成后建议先打开 ").removesuffix(",再继续扩展包体。")
|
||||
return f"After creation, open {path} before expanding the package further."
|
||||
if value.startswith("交付结果:"):
|
||||
return "Deliverables: " + value.removeprefix("交付结果:")
|
||||
if value.startswith("能力类型:"):
|
||||
return "Capability type: " + value.removeprefix("能力类型:")
|
||||
if value.startswith("成熟度:"):
|
||||
return "Maturity: " + value.removeprefix("成熟度:")
|
||||
if value.startswith("触发强度:"):
|
||||
return "Trigger strength: " + en_for(value.removeprefix("触发强度:"))
|
||||
if value.startswith("复用范围:"):
|
||||
return "Reuse scope: " + en_for(value.removeprefix("复用范围:"))
|
||||
if value.startswith("评审进度:"):
|
||||
return "Review progress: " + value.removeprefix("评审进度:")
|
||||
if value.startswith("待评审:"):
|
||||
return "Pending review: " + value.removeprefix("待评审:")
|
||||
if value.startswith("一致率:"):
|
||||
tail = value.removeprefix("一致率:")
|
||||
return "Agreement rate: " + ("not available yet" if tail == "暂无" else tail)
|
||||
if value.startswith("非法决策:"):
|
||||
return "Invalid decisions: " + value.removeprefix("非法决策:")
|
||||
if value.startswith("变体运行:"):
|
||||
return "Variant runs: " + value.removeprefix("变体运行:")
|
||||
if value.startswith("模型执行:"):
|
||||
return "Model executions: " + value.removeprefix("模型执行:")
|
||||
if value.startswith("记录样本:"):
|
||||
return "Recorded fixtures: " + value.removeprefix("记录样本:")
|
||||
if value.startswith("Token 估算:"):
|
||||
return "Token estimates: " + value.removeprefix("Token 估算:")
|
||||
match = re.match(r"^世界级证据仍有\s+(\d+)\s+项待补;公开完成态 claim 必须继续保持阻塞。$", value)
|
||||
if match:
|
||||
return f"World-class evidence still has {match.group(1)} pending item(s); public completion claims must stay blocked."
|
||||
match = re.match(r"^补齐(.+?)证据:(.+)$", value)
|
||||
if match:
|
||||
label = WORLD_CLASS_LABEL_EN.get(match.group(1), match.group(1))
|
||||
return f"Close {label} evidence: {en_for(match.group(2))}"
|
||||
match = re.match(r"^继续补齐剩余\s+(\d+)\s+项外部/人工证据,并保持 claim guard 为 pending 状态。$", value)
|
||||
if match:
|
||||
return f"Close the remaining {match.group(1)} external or human evidence item(s) and keep the claim guard pending."
|
||||
match = re.match(
|
||||
r"^按 ledger 总量继续补齐\s+(\d+)\s+项待补证据(外部\s+(\d+)\s+项、人工\s+(\d+)\s+项;未展开\s+(\d+)\s+项),并保持 claim guard 为 pending 状态。$",
|
||||
value,
|
||||
)
|
||||
if match:
|
||||
return (
|
||||
f"Close all {match.group(1)} pending evidence item(s) in the ledger "
|
||||
f"(external {match.group(2)}, human {match.group(3)}; {match.group(4)} not shown here) "
|
||||
"and keep the claim guard pending."
|
||||
)
|
||||
match = re.match(r"^已生成\s+(\d+)\s+/\s+(\d+)\s+类报告证据。$", value)
|
||||
if match:
|
||||
return f"Generated {match.group(1)} / {match.group(2)} evidence report types."
|
||||
match = re.match(r"^(.+?)\s+已存在。$", value)
|
||||
if match:
|
||||
return f"{match.group(1)} exists."
|
||||
match = re.match(r"^(.+?)\s+未发现或为空,完整度扣分。$", value)
|
||||
if match:
|
||||
return f"{match.group(1)} was not found or is empty, reducing completeness."
|
||||
match = re.match(r"^(.+?)\s+证据不足。$", value)
|
||||
if match:
|
||||
return f"{match.group(1)} evidence is insufficient."
|
||||
match = re.match(r"^SKILL\.md 约\s+(.+?)\s+个词/字。$", value)
|
||||
if match:
|
||||
return f"SKILL.md is about {match.group(1)} words/characters."
|
||||
match = re.match(r"^入口约\s+(.+?)\s+个词/字,references 约\s+(.+?)\s+个词/字。$", value)
|
||||
if match:
|
||||
return f"Entrypoint is about {match.group(1)} words/characters; references are about {match.group(2)}."
|
||||
match = re.match(r"^结构化 Skill 目录,共\s+(.+?)\s+类关键资产。$", value)
|
||||
if match:
|
||||
return f"Structured skill directory with {match.group(1)} key asset groups."
|
||||
if value.startswith("证据不足:缺少 "):
|
||||
return "Evidence gap: missing " + value.removeprefix("证据不足:缺少 ").rstrip("。") + "."
|
||||
for metric_label in METRIC_LABEL_EN:
|
||||
prefix = metric_label + "需要补强:"
|
||||
if value.startswith(prefix):
|
||||
return f"{METRIC_LABEL_EN[metric_label]} needs improvement: {en_for(value.removeprefix(prefix))}"
|
||||
if value.startswith("Use this skill when the request matches:"):
|
||||
return "Use this skill when the request matches the frontmatter description."
|
||||
if value.startswith("用户说出类似需求时:"):
|
||||
return "Use this skill when the user asks for a matching scenario."
|
||||
if value.startswith("当你需要") and "时使用" in value:
|
||||
return "Use this skill when the request matches its stated scenario."
|
||||
if contains_cjk(value):
|
||||
return "Skill-specific source text is authored in Chinese; switch to Simplified Chinese for the exact wording."
|
||||
return value
|
||||
|
||||
|
||||
def bi_span(zh: str, en: str | None = None) -> str:
|
||||
english = en_for(en) if en is not None else en_for(zh)
|
||||
return (
|
||||
f'<span data-lang="zh-CN">{html.escape(str(zh))}</span>'
|
||||
f'<span data-lang="en">{html.escape(str(english))}</span>'
|
||||
)
|
||||
|
||||
|
||||
def bi_item(text: str) -> str:
|
||||
return bi_span(zh_for(text), en_for(text))
|
||||
|
||||
|
||||
def mode_zh(value: str) -> str:
|
||||
return MODE_ZH.get(str(value), str(value))
|
||||
|
||||
|
||||
def readable_description_zh(description: str) -> str:
|
||||
if contains_cjk(description):
|
||||
return description
|
||||
return "该 Skill 的触发描述来自 SKILL.md frontmatter;默认中文报告先呈现能力边界,原始英文描述可切换到英文查看。"
|
||||
+11
-615
@@ -2,6 +2,7 @@
|
||||
"""Static layout contract for skill overview report HTML."""
|
||||
|
||||
import html
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@@ -9,6 +10,14 @@ SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to keep overview report layout and CSS out of data rendering."
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
ASSET_DIR = ROOT / "assets"
|
||||
|
||||
|
||||
def read_layout_asset(filename: str) -> str:
|
||||
return (ASSET_DIR / filename).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def bi_span(zh: str, en: str) -> str:
|
||||
return (
|
||||
f'<span data-lang="zh-CN">{html.escape(str(zh))}</span>'
|
||||
@@ -33,621 +42,8 @@ def render_language_switch() -> str:
|
||||
|
||||
|
||||
def skill_overview_css() -> str:
|
||||
return """
|
||||
:root {
|
||||
--paper: #ffffff;
|
||||
--wash: #f8fafc;
|
||||
--wash-strong: #f2f5f8;
|
||||
--line: #e6e0d4;
|
||||
--line-soft: #eee9df;
|
||||
--brand: #1B365D;
|
||||
--brand-soft: #EEF3F8;
|
||||
--brand-mid: #315982;
|
||||
--ink: #151515;
|
||||
--text: #2f2d29;
|
||||
--muted: #68625a;
|
||||
--faint: #8b857b;
|
||||
--success: #2f6f5e;
|
||||
--warn: #8a5a19;
|
||||
--serif: "TsangerJinKai02", "Source Han Serif SC", "Noto Serif CJK SC", "Songti SC", "STSong", Charter, Georgia, serif;
|
||||
--mono: "JetBrains Mono", "SF Mono", ui-monospace, Menlo, Consolas, monospace;
|
||||
--shadow-soft: 0 1px 2px rgba(27, 54, 93, 0.06), 0 16px 44px rgba(27, 54, 93, 0.08);
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html { scroll-behavior: smooth; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: #ffffff;
|
||||
color: var(--ink);
|
||||
font-family: var(--serif);
|
||||
line-height: 1.62;
|
||||
letter-spacing: 0;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
text-rendering: optimizeLegibility;
|
||||
}
|
||||
body[data-report-lang="zh-CN"] [data-lang="en"],
|
||||
body[data-report-lang="en"] [data-lang="zh-CN"] { display: none !important; }
|
||||
.skip-link {
|
||||
position: fixed;
|
||||
left: 18px;
|
||||
top: 10px;
|
||||
z-index: 40;
|
||||
transform: translateY(-140%);
|
||||
padding: 8px 12px;
|
||||
border-radius: 8px;
|
||||
background: var(--brand);
|
||||
color: #fff;
|
||||
text-decoration: none;
|
||||
transition: transform 160ms cubic-bezier(0.16, 1, 0.3, 1);
|
||||
}
|
||||
.skip-link:focus-visible { transform: translateY(0); outline: 2px solid var(--brand-mid); outline-offset: 2px; }
|
||||
.topbar {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 20;
|
||||
background: rgba(255, 255, 255, 0.96);
|
||||
border-bottom: 1px solid var(--line);
|
||||
}
|
||||
.progress-track {
|
||||
height: 2px;
|
||||
background: transparent;
|
||||
}
|
||||
.progress-bar {
|
||||
display: block;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: var(--brand);
|
||||
transition: transform 160ms cubic-bezier(0.16, 1, 0.3, 1);
|
||||
transform-origin: left center;
|
||||
transform: scaleX(0);
|
||||
}
|
||||
.topbar-inner {
|
||||
max-width: 1240px;
|
||||
margin: 0 auto;
|
||||
padding: 9px 28px;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 18px;
|
||||
align-items: center;
|
||||
}
|
||||
.nav-shell {
|
||||
display: grid;
|
||||
grid-template-columns: auto minmax(0, 1fr);
|
||||
gap: 18px;
|
||||
align-items: center;
|
||||
min-width: 0;
|
||||
}
|
||||
.report-mark {
|
||||
color: var(--brand);
|
||||
font-family: var(--mono);
|
||||
font-size: 11px;
|
||||
line-height: 1;
|
||||
text-transform: uppercase;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.report-nav {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
overflow-x: auto;
|
||||
scrollbar-width: none;
|
||||
min-width: 0;
|
||||
}
|
||||
.report-nav::-webkit-scrollbar { display: none; }
|
||||
.report-nav a {
|
||||
flex: 0 0 auto;
|
||||
min-width: 76px;
|
||||
min-height: 40px;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 0 10px;
|
||||
color: var(--brand);
|
||||
text-decoration: none;
|
||||
text-align: center;
|
||||
font-size: 13px;
|
||||
border-radius: 8px;
|
||||
white-space: nowrap;
|
||||
transition-property: background-color, color, transform;
|
||||
transition-duration: 160ms;
|
||||
transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1);
|
||||
}
|
||||
@media (hover: hover) {
|
||||
.report-nav a:hover { background: var(--brand-soft); }
|
||||
}
|
||||
.report-nav a:focus-visible {
|
||||
outline: 2px solid var(--brand-mid);
|
||||
outline-offset: 2px;
|
||||
background: var(--brand-soft);
|
||||
}
|
||||
.report-nav a[aria-current="true"] {
|
||||
background: var(--brand);
|
||||
color: #ffffff;
|
||||
}
|
||||
.report-nav a[aria-current="true"] span { color: #ffffff; }
|
||||
.language-switch {
|
||||
display: inline-flex;
|
||||
gap: 3px;
|
||||
padding: 3px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
background: var(--paper);
|
||||
box-shadow: 0 1px 2px rgba(27, 54, 93, 0.05);
|
||||
}
|
||||
.language-switch button {
|
||||
appearance: none;
|
||||
min-width: 42px;
|
||||
min-height: 34px;
|
||||
border: 0;
|
||||
border-radius: 6px;
|
||||
padding: 0 10px;
|
||||
background: transparent;
|
||||
color: var(--muted);
|
||||
font: inherit;
|
||||
font-size: 13px;
|
||||
cursor: pointer;
|
||||
transition-property: background-color, color, transform;
|
||||
transition-duration: 160ms;
|
||||
transition-timing-function: cubic-bezier(0.16, 1, 0.3, 1);
|
||||
}
|
||||
.language-switch button:active { transform: scale(0.96); }
|
||||
.language-switch button:focus-visible { outline: 2px solid var(--brand-mid); outline-offset: 2px; }
|
||||
.language-switch button[aria-pressed="true"] {
|
||||
background: var(--brand-soft);
|
||||
color: var(--brand);
|
||||
}
|
||||
.wrap {
|
||||
max-width: 1240px;
|
||||
margin: 0 auto;
|
||||
padding: 58px 28px 92px;
|
||||
}
|
||||
.hero {
|
||||
padding: 16px 0 34px;
|
||||
}
|
||||
.hero-grid {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) 360px;
|
||||
gap: 44px;
|
||||
align-items: end;
|
||||
}
|
||||
.eyebrow, .report-mark {
|
||||
letter-spacing: 0;
|
||||
}
|
||||
.eyebrow {
|
||||
margin: 0 0 12px;
|
||||
color: var(--brand);
|
||||
font-family: var(--mono);
|
||||
font-size: 12px;
|
||||
text-transform: uppercase;
|
||||
}
|
||||
.slug {
|
||||
margin: 0 0 18px;
|
||||
color: var(--faint);
|
||||
font-family: var(--mono);
|
||||
font-size: 13px;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
h1, h2, h3 {
|
||||
margin: 0;
|
||||
font-weight: 500;
|
||||
letter-spacing: 0;
|
||||
text-wrap: balance;
|
||||
}
|
||||
h1 {
|
||||
max-width: 8em;
|
||||
color: var(--ink);
|
||||
font-size: 4rem;
|
||||
line-height: 1.02;
|
||||
}
|
||||
.lead {
|
||||
max-width: 760px;
|
||||
margin: 20px 0 0;
|
||||
color: var(--text);
|
||||
font-size: 1.08rem;
|
||||
text-wrap: pretty;
|
||||
}
|
||||
.hero-meta, .badges {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 20px;
|
||||
}
|
||||
.hero-meta span, .badges span, .tag {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 30px;
|
||||
padding: 5px 10px;
|
||||
border-radius: 6px;
|
||||
background: var(--brand-soft);
|
||||
color: var(--brand);
|
||||
font-size: 13px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.hero-card {
|
||||
position: relative;
|
||||
padding: 22px 22px 24px;
|
||||
border-radius: 8px;
|
||||
background: linear-gradient(180deg, #ffffff 0%, #fbfaf7 100%);
|
||||
box-shadow: var(--shadow-soft);
|
||||
border: 1px solid var(--line-soft);
|
||||
}
|
||||
.hero-card::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 18px;
|
||||
right: 18px;
|
||||
width: 38px;
|
||||
height: 2px;
|
||||
background: var(--brand);
|
||||
}
|
||||
.score-strip {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 10px;
|
||||
margin-top: 32px;
|
||||
padding-top: 20px;
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
.score-chip {
|
||||
padding: 14px 14px 12px;
|
||||
border-radius: 8px;
|
||||
background: var(--wash);
|
||||
border: 1px solid var(--line-soft);
|
||||
}
|
||||
.score-chip span {
|
||||
display: block;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
.score-chip strong {
|
||||
display: block;
|
||||
margin: 5px 0 8px;
|
||||
color: var(--brand);
|
||||
font-family: var(--mono);
|
||||
font-size: 1.65rem;
|
||||
font-weight: 500;
|
||||
line-height: 1;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.score-chip i {
|
||||
display: block;
|
||||
height: 3px;
|
||||
border-radius: 999px;
|
||||
background: linear-gradient(90deg, var(--brand) var(--score), #dfe6ee var(--score));
|
||||
}
|
||||
.score-chip small {
|
||||
display: block;
|
||||
margin-top: 8px;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
line-height: 1.45;
|
||||
}
|
||||
section {
|
||||
scroll-margin-top: 78px;
|
||||
padding-top: 44px;
|
||||
margin-top: 44px;
|
||||
border-top: 1px solid var(--line);
|
||||
}
|
||||
section.hero {
|
||||
scroll-margin-top: 0;
|
||||
padding-top: 16px;
|
||||
margin-top: 0;
|
||||
border-top: 0;
|
||||
}
|
||||
.section-head {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 246px) minmax(0, 1fr);
|
||||
gap: 36px;
|
||||
align-items: start;
|
||||
}
|
||||
h2 {
|
||||
color: var(--ink);
|
||||
font-size: 1.5rem;
|
||||
line-height: 1.18;
|
||||
}
|
||||
h2::before {
|
||||
content: "";
|
||||
display: block;
|
||||
width: 32px;
|
||||
height: 2px;
|
||||
margin-bottom: 12px;
|
||||
background: var(--brand);
|
||||
}
|
||||
h3 {
|
||||
color: var(--ink);
|
||||
font-size: 1.02rem;
|
||||
line-height: 1.3;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.section-head p {
|
||||
margin: 12px 0 0;
|
||||
color: var(--muted);
|
||||
max-width: 43ch;
|
||||
text-wrap: pretty;
|
||||
}
|
||||
.two-col, .metric-grid, .chart-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
align-items: stretch;
|
||||
}
|
||||
.quality-panels {
|
||||
margin-top: 16px;
|
||||
}
|
||||
.metrics-stack {
|
||||
display: grid;
|
||||
gap: 18px;
|
||||
}
|
||||
.metrics-lead {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(420px, 1.15fr) minmax(300px, 0.85fr);
|
||||
gap: 18px;
|
||||
align-items: stretch;
|
||||
}
|
||||
.metrics-note {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
gap: 16px;
|
||||
}
|
||||
.metrics-note p {
|
||||
margin: 0;
|
||||
max-width: none;
|
||||
color: var(--text);
|
||||
}
|
||||
.metric-summary-list {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
list-style: none;
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
}
|
||||
.metric-summary-list li {
|
||||
display: grid;
|
||||
grid-template-columns: auto minmax(0, 1fr) auto;
|
||||
gap: 8px 10px;
|
||||
align-items: baseline;
|
||||
padding: 10px 0;
|
||||
border-bottom: 1px solid var(--line-soft);
|
||||
}
|
||||
.metric-summary-list li:last-child { border-bottom: 0; }
|
||||
.metric-summary-list b {
|
||||
color: var(--ink);
|
||||
font-weight: 500;
|
||||
}
|
||||
.metric-summary-list em {
|
||||
color: var(--brand);
|
||||
font-family: var(--mono);
|
||||
font-style: normal;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.metric-summary-list small {
|
||||
grid-column: 2 / 4;
|
||||
color: var(--muted);
|
||||
line-height: 1.5;
|
||||
}
|
||||
.metric-status {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
min-height: 24px;
|
||||
padding: 2px 7px;
|
||||
border-radius: 6px;
|
||||
background: var(--brand-soft);
|
||||
color: var(--brand);
|
||||
font-size: 12px;
|
||||
}
|
||||
.metric-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
margin-top: 2px;
|
||||
}
|
||||
.list, .compact-list, .step-list {
|
||||
margin: 0;
|
||||
padding-left: 1.15em;
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
}
|
||||
.list li::marker, .compact-list li::marker, .step-list li::marker { color: var(--brand); }
|
||||
.compact-list {
|
||||
gap: 6px;
|
||||
font-size: 13px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.panel, .metric-card, .roadmap-item {
|
||||
background: #ffffff;
|
||||
border: 1px solid var(--line-soft);
|
||||
border-radius: 8px;
|
||||
padding: 22px;
|
||||
box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04);
|
||||
}
|
||||
.metric-card {
|
||||
display: grid;
|
||||
grid-template-columns: 92px minmax(0, 1fr);
|
||||
gap: 18px;
|
||||
align-content: start;
|
||||
min-height: 0;
|
||||
}
|
||||
.metric-card-head {
|
||||
display: grid;
|
||||
align-content: start;
|
||||
}
|
||||
.metric-card strong {
|
||||
display: block;
|
||||
margin: 8px 0 10px;
|
||||
color: var(--brand);
|
||||
font-family: var(--mono);
|
||||
font-size: 2rem;
|
||||
font-weight: 500;
|
||||
line-height: 1;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
.metric-label {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
.metric-card-body {
|
||||
min-width: 0;
|
||||
}
|
||||
.chart-figure {
|
||||
margin: 0;
|
||||
padding: 18px;
|
||||
border: 1px solid var(--line-soft);
|
||||
border-radius: 8px;
|
||||
background: #ffffff;
|
||||
box-shadow: 0 1px 2px rgba(27, 54, 93, 0.04);
|
||||
}
|
||||
.chart-figure svg {
|
||||
width: 100%;
|
||||
height: auto;
|
||||
display: block;
|
||||
overflow: visible;
|
||||
}
|
||||
.chart-figure text {
|
||||
fill: var(--ink);
|
||||
font-family: var(--serif);
|
||||
font-size: 13px;
|
||||
}
|
||||
.chart-title {
|
||||
fill: var(--brand);
|
||||
font-size: 16px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.chart-line {
|
||||
fill: none;
|
||||
stroke: var(--brand);
|
||||
stroke-width: 2;
|
||||
}
|
||||
figcaption {
|
||||
margin-top: 12px;
|
||||
padding-top: 10px;
|
||||
border-top: 1px solid var(--line-soft);
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
line-height: 1.5;
|
||||
text-wrap: pretty;
|
||||
}
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 14px;
|
||||
}
|
||||
th, td {
|
||||
padding: 12px 10px;
|
||||
text-align: left;
|
||||
border-bottom: 1px solid var(--line-soft);
|
||||
vertical-align: top;
|
||||
}
|
||||
th {
|
||||
color: var(--brand);
|
||||
font-weight: 500;
|
||||
font-size: 12px;
|
||||
}
|
||||
td:first-child, th:first-child { width: 96px; }
|
||||
.roadmap {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
gap: 16px;
|
||||
}
|
||||
.step {
|
||||
display: inline-flex;
|
||||
margin-bottom: 12px;
|
||||
min-height: 28px;
|
||||
align-items: center;
|
||||
padding: 3px 8px;
|
||||
border-radius: 6px;
|
||||
background: var(--brand-soft);
|
||||
color: var(--brand);
|
||||
font-size: 12px;
|
||||
}
|
||||
.unlock {
|
||||
margin-top: 14px;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
@media (max-width: 980px) {
|
||||
.topbar-inner {
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
padding: 8px 16px;
|
||||
}
|
||||
.nav-shell { grid-template-columns: 1fr; gap: 6px; }
|
||||
.report-mark { display: none; }
|
||||
.report-nav a { min-width: 72px; }
|
||||
.hero-grid, .section-head { grid-template-columns: 1fr; }
|
||||
.hero-grid { gap: 24px; }
|
||||
.score-strip { grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
.metrics-lead, .two-col, .metric-grid, .chart-grid, .roadmap { grid-template-columns: 1fr; }
|
||||
.metric-card { grid-template-columns: 86px minmax(0, 1fr); }
|
||||
h1 { font-size: 2.75rem; }
|
||||
}
|
||||
@media (max-width: 540px) {
|
||||
.wrap { padding: 34px 16px 72px; }
|
||||
.topbar-inner { gap: 10px; }
|
||||
.report-nav {
|
||||
display: flex;
|
||||
gap: 2px;
|
||||
}
|
||||
.report-nav a {
|
||||
min-width: 66px;
|
||||
min-height: 38px;
|
||||
padding: 0 8px;
|
||||
font-size: 12px;
|
||||
}
|
||||
.language-switch button { min-width: 38px; min-height: 32px; padding: 0 8px; }
|
||||
.hero { padding-top: 8px; }
|
||||
h1 { max-width: 8em; font-size: 2.35rem; }
|
||||
.lead { font-size: 1rem; }
|
||||
.hero-meta span, .badges span { font-size: 12px; }
|
||||
.score-strip { grid-template-columns: 1fr; }
|
||||
section { margin-top: 34px; padding-top: 34px; }
|
||||
.panel, .metric-card, .roadmap-item, .chart-figure, .hero-card { padding: 18px; }
|
||||
.metric-card { grid-template-columns: 1fr; gap: 10px; }
|
||||
.metric-card strong { margin-bottom: 2px; }
|
||||
.metric-summary-list li { grid-template-columns: auto minmax(0, 1fr) auto; }
|
||||
table { font-size: 13px; }
|
||||
th, td { padding: 10px 7px; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
html { scroll-behavior: auto; }
|
||||
*, *::before, *::after { transition-duration: 0.01ms !important; }
|
||||
}
|
||||
""".strip()
|
||||
return read_layout_asset("skill-overview.css")
|
||||
|
||||
|
||||
def skill_overview_script() -> str:
|
||||
return """
|
||||
(function () {
|
||||
var buttons = Array.prototype.slice.call(document.querySelectorAll("[data-set-lang]"));
|
||||
var navLinks = Array.prototype.slice.call(document.querySelectorAll(".report-nav a"));
|
||||
var sections = navLinks
|
||||
.map(function (link) { return document.querySelector(link.getAttribute("href")); })
|
||||
.filter(Boolean);
|
||||
var progressBar = document.querySelector(".progress-bar");
|
||||
function setLanguage(lang) {
|
||||
document.body.setAttribute("data-report-lang", lang);
|
||||
document.documentElement.setAttribute("lang", lang);
|
||||
buttons.forEach(function (button) {
|
||||
button.setAttribute("aria-pressed", button.getAttribute("data-set-lang") === lang ? "true" : "false");
|
||||
});
|
||||
}
|
||||
function updateProgress() {
|
||||
var scrollTop = window.scrollY || document.documentElement.scrollTop;
|
||||
var height = Math.max(1, document.documentElement.scrollHeight - window.innerHeight);
|
||||
var pct = Math.min(100, Math.max(0, (scrollTop / height) * 100));
|
||||
if (progressBar) progressBar.style.transform = "scaleX(" + pct / 100 + ")";
|
||||
var active = sections[0];
|
||||
sections.forEach(function (section) {
|
||||
if (section.getBoundingClientRect().top <= 110) active = section;
|
||||
});
|
||||
navLinks.forEach(function (link) {
|
||||
link.setAttribute("aria-current", link.getAttribute("href") === "#" + active.id ? "true" : "false");
|
||||
});
|
||||
}
|
||||
buttons.forEach(function (button) {
|
||||
button.addEventListener("click", function () {
|
||||
setLanguage(button.getAttribute("data-set-lang"));
|
||||
});
|
||||
});
|
||||
window.addEventListener("scroll", updateProgress, { passive: true });
|
||||
window.addEventListener("resize", updateProgress);
|
||||
setLanguage("zh-CN");
|
||||
updateProgress();
|
||||
})();
|
||||
""".strip()
|
||||
return read_layout_asset("skill-overview.js")
|
||||
|
||||
+72
-265
@@ -1,33 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from skill_ir_paths import find_skill_ir
|
||||
from skill_report_metrics import calculate_scorecard
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
from skill_report_sources import (
|
||||
extract_title,
|
||||
load_json,
|
||||
load_yaml,
|
||||
package_entries,
|
||||
parse_frontmatter,
|
||||
parse_sections,
|
||||
summarize_logic,
|
||||
summarize_usage,
|
||||
)
|
||||
from skill_report_sections import package_assets, quality_review, risk_governance
|
||||
from skill_report_world_class import world_class_readiness, world_class_roadmap_item
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to build the v2 report data model."
|
||||
|
||||
KNOWN_ENTRIES = [
|
||||
("SKILL.md", "Skill entrypoint"),
|
||||
("README.md", "Human-readable usage guide"),
|
||||
("agents/interface.yaml", "Neutral interface metadata"),
|
||||
("manifest.json", "Lifecycle and portability metadata"),
|
||||
("references", "Extended guidance and reusable notes"),
|
||||
("scripts", "Deterministic helpers or local tooling"),
|
||||
("evals", "Trigger and quality checks"),
|
||||
("reports", "Generated evidence and overview artifacts"),
|
||||
]
|
||||
|
||||
IGNORED_PACKAGE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"}
|
||||
|
||||
REPORT_NAV_V2 = [
|
||||
{"label": "技能概述", "label_en": "Overview", "href": "overview"},
|
||||
{"label": "总览指标", "label_en": "Metrics", "href": "metrics"},
|
||||
@@ -41,131 +34,6 @@ REPORT_NAV_V2 = [
|
||||
]
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict, str]:
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}, text
|
||||
try:
|
||||
end_index = lines[1:].index("---") + 1
|
||||
except ValueError:
|
||||
return {}, text
|
||||
frontmatter_text = "\n".join(lines[1:end_index])
|
||||
body = "\n".join(lines[end_index + 1 :]).lstrip()
|
||||
if yaml is not None:
|
||||
data = yaml.safe_load(frontmatter_text) or {}
|
||||
return data if isinstance(data, dict) else {}, body
|
||||
data = {}
|
||||
for line in frontmatter_text.splitlines():
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, value = line.split(":", 1)
|
||||
data[key.strip()] = value.strip().strip('"')
|
||||
return data, body
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if yaml is not None:
|
||||
payload = yaml.safe_load(text) or {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
return {}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def extract_title(body: str, fallback: str) -> str:
|
||||
for line in body.splitlines():
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()
|
||||
return fallback
|
||||
|
||||
|
||||
def parse_sections(body: str) -> dict[str, str]:
|
||||
sections: dict[str, list[str]] = {}
|
||||
current = "_preamble"
|
||||
sections[current] = []
|
||||
for line in body.splitlines():
|
||||
if line.startswith("## "):
|
||||
current = line[3:].strip()
|
||||
sections[current] = []
|
||||
continue
|
||||
sections[current].append(line)
|
||||
return {name: "\n".join(lines).strip() for name, lines in sections.items()}
|
||||
|
||||
|
||||
def extract_list_items(text: str) -> list[str]:
|
||||
items = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
ordered = re.match(r"^\d+\.\s+(.*)$", stripped)
|
||||
bullet = re.match(r"^[-*]\s+(.*)$", stripped)
|
||||
match = ordered or bullet
|
||||
if match:
|
||||
items.append(match.group(1).strip())
|
||||
return items
|
||||
|
||||
|
||||
def summarize_logic(sections: dict[str, str]) -> list[str]:
|
||||
for key in ("Compact Workflow", "Workflow", "How It Works", "Logic", "Quick Start"):
|
||||
if key in sections:
|
||||
items = extract_list_items(sections[key])
|
||||
if items:
|
||||
return items[:5]
|
||||
return extract_list_items(sections.get("_preamble", ""))[:5] or [
|
||||
"Understand the request",
|
||||
"Execute the main task",
|
||||
"Validate the result",
|
||||
]
|
||||
|
||||
|
||||
def summarize_usage(sections: dict[str, str], default_prompt: str, description: str) -> list[str]:
|
||||
for key in ("How To Use", "Quick Start", "Usage", "Runbook"):
|
||||
if key in sections:
|
||||
items = extract_list_items(sections[key])
|
||||
if items:
|
||||
return items[:5]
|
||||
usage = []
|
||||
if default_prompt:
|
||||
usage.append(default_prompt)
|
||||
usage.append(f"Use this skill when the request matches: {description}")
|
||||
return usage[:5]
|
||||
|
||||
|
||||
def package_entries(skill_dir: Path) -> list[dict]:
|
||||
items = []
|
||||
for rel_path, label in KNOWN_ENTRIES:
|
||||
target = skill_dir / rel_path
|
||||
if target.exists():
|
||||
kind = "folder" if target.is_dir() else "file"
|
||||
if target.is_dir():
|
||||
count = len(
|
||||
[
|
||||
path
|
||||
for path in target.rglob("*")
|
||||
if path.is_file()
|
||||
and not path.is_symlink()
|
||||
and not any(part in IGNORED_PACKAGE_PARTS for part in path.relative_to(target).parts)
|
||||
and path.suffix not in {".pyc", ".pyo"}
|
||||
]
|
||||
)
|
||||
else:
|
||||
count = 1
|
||||
items.append({"path": rel_path, "label": label, "kind": kind, "file_count": count})
|
||||
return items
|
||||
|
||||
|
||||
def context_payload(intent: dict) -> dict:
|
||||
context = intent.get("context", {}) if isinstance(intent, dict) else {}
|
||||
return context if isinstance(context, dict) else {}
|
||||
@@ -307,9 +175,12 @@ def principle_nodes(system_model: dict) -> list[dict]:
|
||||
]
|
||||
|
||||
|
||||
def roadmap_items(iteration: dict) -> list[dict]:
|
||||
def roadmap_items(iteration: dict, readiness: dict | None = None) -> list[dict]:
|
||||
directions = iteration.get("directions", []) if isinstance(iteration, dict) else []
|
||||
items = []
|
||||
evidence_item = world_class_roadmap_item(readiness or {})
|
||||
if evidence_item:
|
||||
items.append(evidence_item)
|
||||
for item in directions[:3]:
|
||||
items.append(
|
||||
{
|
||||
@@ -319,6 +190,8 @@ def roadmap_items(iteration: dict) -> list[dict]:
|
||||
"unlocks": item.get("unlocks", ""),
|
||||
}
|
||||
)
|
||||
if len(items) >= 3:
|
||||
break
|
||||
if items:
|
||||
return items
|
||||
return [
|
||||
@@ -331,42 +204,6 @@ def roadmap_items(iteration: dict) -> list[dict]:
|
||||
]
|
||||
|
||||
|
||||
def artifact_design_highlights(profile: dict) -> list[str]:
|
||||
primary = profile.get("primary_artifact", {})
|
||||
highlights = []
|
||||
if primary.get("direction"):
|
||||
highlights.append(primary["direction"])
|
||||
highlights.extend(profile.get("quality_gates", [])[:3])
|
||||
return highlights[:4]
|
||||
|
||||
|
||||
def prompt_quality_highlights(profile: dict) -> list[str]:
|
||||
highlights = []
|
||||
primary = profile.get("primary_task_family", {})
|
||||
complexity = profile.get("complexity", {})
|
||||
if primary.get("label"):
|
||||
highlights.append(f"Primary prompt task family: {primary['label']}.")
|
||||
if complexity.get("band"):
|
||||
highlights.append(f"Complexity: {complexity['band']} — {complexity.get('reason', '')}")
|
||||
for item in profile.get("quality_matrix", [])[:2]:
|
||||
highlights.append(f"{item.get('label', 'Quality')}: {item.get('score', 'n/a')}/100.")
|
||||
return highlights[:4]
|
||||
|
||||
|
||||
def system_model_highlights(model: dict) -> list[str]:
|
||||
highlights = []
|
||||
stability = model.get("stability", {})
|
||||
if stability:
|
||||
highlights.append(f"Stability: {stability.get('band', 'unknown')} ({stability.get('score', 'n/a')}/100).")
|
||||
boundary = model.get("boundary_map", {})
|
||||
if boundary.get("owned_job"):
|
||||
highlights.append(f"Owned job: {boundary['owned_job']}")
|
||||
for point in model.get("leverage_points", [])[:2]:
|
||||
if point.get("point"):
|
||||
highlights.append(f"Leverage: {point['point']} — {point.get('move', '')}")
|
||||
return highlights[:4]
|
||||
|
||||
|
||||
def capability_profile(manifest: dict, interface_data: dict, prompt_quality: dict) -> dict:
|
||||
maturity = manifest.get("maturity_tier", "scaffold")
|
||||
task_family = prompt_quality.get("primary_task_family", {}).get("label", "Skill workflow")
|
||||
@@ -384,82 +221,6 @@ def capability_profile(manifest: dict, interface_data: dict, prompt_quality: dic
|
||||
}
|
||||
|
||||
|
||||
def risk_governance(output_risk: dict, system_model: dict, scorecard: dict) -> dict:
|
||||
risk_names = [
|
||||
("误触发风险", "trigger_score"),
|
||||
("输出漂移风险", "evidence_score"),
|
||||
("证据不足风险", "evidence_score"),
|
||||
("包体膨胀风险", "maintainability_score"),
|
||||
("跨平台迁移风险", "portability_score"),
|
||||
]
|
||||
risks = []
|
||||
for index, (name, metric_key) in enumerate(risk_names):
|
||||
score = scorecard.get(metric_key, {}).get("score", 50)
|
||||
probability = max(1, min(3, 4 - round(score / 34)))
|
||||
impact = 3 if index in {0, 2, 4} else 2
|
||||
risks.append(
|
||||
{
|
||||
"name": name,
|
||||
"impact": impact,
|
||||
"probability": probability,
|
||||
"signal": scorecard.get(metric_key, {}).get("reasons", ["证据不足"])[0],
|
||||
"response": "先补证据和边界,再增加包体复杂度。",
|
||||
}
|
||||
)
|
||||
human_boundary = system_model.get("boundary_map", {}).get("human_judgment_boundary", [])
|
||||
return {
|
||||
"risks": risks,
|
||||
"risk_families": output_risk.get("risk_families", []),
|
||||
"human_judgment_boundary": human_boundary,
|
||||
}
|
||||
|
||||
|
||||
def quality_review(
|
||||
strengths: list[str],
|
||||
scorecard: dict,
|
||||
artifact_design: dict,
|
||||
prompt_quality: dict,
|
||||
system_model: dict,
|
||||
) -> dict:
|
||||
gaps = []
|
||||
for key, payload in scorecard.items():
|
||||
if payload.get("score", 0) < 70:
|
||||
gaps.append(f"{payload.get('label', key)}需要补强:{payload.get('reasons', ['证据不足'])[0]}")
|
||||
return {
|
||||
"strengths": strengths,
|
||||
"gaps": gaps or ["当前关键证据较完整,优先保持轻量。"],
|
||||
"recommendations": [
|
||||
"先改触发边界,再扩展工作流。",
|
||||
"只把重复且稳定的步骤沉淀为脚本。",
|
||||
"每次升级后重新生成报告并检查分数原因。",
|
||||
],
|
||||
"artifact_design": {
|
||||
"design_system": artifact_design.get("design_system", "content-led editorial"),
|
||||
"highlights": artifact_design_highlights(artifact_design),
|
||||
},
|
||||
"prompt_quality": {
|
||||
"overall_quality_score": prompt_quality.get("overall_quality_score", "n/a"),
|
||||
"highlights": prompt_quality_highlights(prompt_quality),
|
||||
},
|
||||
"system_model": {
|
||||
"stability": system_model.get("stability", {}),
|
||||
"highlights": system_model_highlights(system_model),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def package_assets(package_map: list[dict]) -> dict:
|
||||
files = sum(item.get("file_count", 0) for item in package_map if item.get("kind") == "file")
|
||||
folders = [item for item in package_map if item.get("kind") == "folder"]
|
||||
distribution = [{"label": item["path"], "value": max(1, item.get("file_count", 1))} for item in package_map]
|
||||
return {
|
||||
"entries": package_map,
|
||||
"file_count": files + sum(item.get("file_count", 0) for item in folders),
|
||||
"folder_count": len(folders),
|
||||
"distribution": distribution,
|
||||
}
|
||||
|
||||
|
||||
def build_report_model(skill_dir: Path) -> dict:
|
||||
skill_dir = skill_dir.resolve()
|
||||
skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8")
|
||||
@@ -475,7 +236,9 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json")
|
||||
output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json")
|
||||
output_blind_review = load_json(skill_dir / "reports" / "output_blind_review_pack.json")
|
||||
output_review_kit = load_json(skill_dir / "reports" / "output_review_kit.json")
|
||||
output_review_adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json")
|
||||
benchmark_reproducibility = load_json(skill_dir / "reports" / "benchmark_reproducibility.json")
|
||||
conformance = load_json(skill_dir / "reports" / "conformance_matrix.json")
|
||||
runtime_permissions = load_json(skill_dir / "reports" / "runtime_permission_probes.json")
|
||||
trust_report = load_json(skill_dir / "reports" / "security_trust_report.json")
|
||||
@@ -487,15 +250,14 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
adoption_drift = load_json(skill_dir / "reports" / "adoption_drift_report.json")
|
||||
review_waivers = load_json(skill_dir / "reports" / "review_waivers.json")
|
||||
review_annotations = load_json(skill_dir / "reports" / "review_annotations.json")
|
||||
world_class_evidence = load_json(skill_dir / "reports" / "world_class_evidence_plan.json")
|
||||
world_class_evidence_ledger = load_json(skill_dir / "reports" / "world_class_evidence_ledger.json")
|
||||
compiled_targets = load_json(skill_dir / "reports" / "compiled_targets.json")
|
||||
skill_ir = load_json(skill_dir / "reports" / "skill-ir.json")
|
||||
if not skill_ir:
|
||||
example_ir = skill_dir / "skill-ir" / "examples" / f"{frontmatter.get('name', skill_dir.name)}.json"
|
||||
skill_ir = load_json(example_ir)
|
||||
reference_synthesis = load_json(skill_dir / "reports" / "reference-synthesis.json")
|
||||
iteration = load_json(skill_dir / "reports" / "iteration-directions.json")
|
||||
|
||||
name = frontmatter.get("name", skill_dir.name)
|
||||
skill_ir, skill_ir_path = find_skill_ir(skill_dir, name)
|
||||
description = frontmatter.get("description", "No description found.")
|
||||
title = extract_title(body, name.replace("-", " ").title())
|
||||
display_name = interface_data.get("interface", {}).get("display_name", title)
|
||||
@@ -508,7 +270,8 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
trigger = trigger_contract(interface_data, description)
|
||||
io = io_contract(intent, package_map, description)
|
||||
principles = principle_nodes(system_model)
|
||||
roadmap = roadmap_items(iteration)
|
||||
readiness = world_class_readiness(world_class_evidence_ledger)
|
||||
roadmap = roadmap_items(iteration, readiness)
|
||||
metadata = {
|
||||
"canonical_format": interface_data.get("compatibility", {}).get("canonical_format", "agent-skills"),
|
||||
"targets": interface_data.get("compatibility", {}).get("adapter_targets", []),
|
||||
@@ -519,7 +282,7 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
deliverables = [
|
||||
"SKILL.md",
|
||||
"agents/interface.yaml",
|
||||
"reports/skill-ir.json",
|
||||
skill_ir_path or "reports/skill-ir.json",
|
||||
"reports/compiled_targets.md",
|
||||
"reports/output_quality_scorecard.md",
|
||||
"reports/conformance_matrix.md",
|
||||
@@ -533,6 +296,7 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"reports/review_waivers.md",
|
||||
"reports/review_annotations.md",
|
||||
"reports/review-studio.html",
|
||||
"reports/skill-interpretation.html",
|
||||
"reports/skill-overview.html",
|
||||
]
|
||||
if (skill_dir / "reports" / "runtime_permission_probes.md").exists():
|
||||
@@ -547,9 +311,25 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
if (skill_dir / "reports" / "output_blind_answer_key.json").exists():
|
||||
insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
|
||||
deliverables.insert(insert_after, "reports/output_blind_answer_key.json")
|
||||
if (skill_dir / "reports" / "output_review_kit.md").exists():
|
||||
insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
|
||||
deliverables.insert(insert_after, "reports/output_review_kit.md")
|
||||
if (skill_dir / "reports" / "output_review_adjudication.md").exists():
|
||||
insert_after = deliverables.index("reports/output_blind_answer_key.json") + 1 if "reports/output_blind_answer_key.json" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
|
||||
insert_after = deliverables.index("reports/output_review_kit.md") + 1 if "reports/output_review_kit.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
|
||||
deliverables.insert(insert_after, "reports/output_review_adjudication.md")
|
||||
if (skill_dir / "reports" / "benchmark_reproducibility.md").exists():
|
||||
insert_after = deliverables.index("reports/output_review_adjudication.md") + 1 if "reports/output_review_adjudication.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
|
||||
deliverables.insert(insert_after, "reports/benchmark_reproducibility.md")
|
||||
if (skill_dir / "reports" / "world_class_evidence_plan.md").exists():
|
||||
insert_after = deliverables.index("reports/review_waivers.md") + 1
|
||||
deliverables.insert(insert_after, "reports/world_class_evidence_plan.md")
|
||||
if (skill_dir / "reports" / "world_class_evidence_ledger.md").exists():
|
||||
insert_after = (
|
||||
deliverables.index("reports/world_class_evidence_plan.md") + 1
|
||||
if "reports/world_class_evidence_plan.md" in deliverables
|
||||
else deliverables.index("reports/review_waivers.md") + 1
|
||||
)
|
||||
deliverables.insert(insert_after, "reports/world_class_evidence_ledger.md")
|
||||
|
||||
skill_summary = {
|
||||
"name": name,
|
||||
@@ -599,6 +379,7 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"contract_boundary": contract,
|
||||
"quality_review": q_review,
|
||||
"risk_governance": risk_governance(output_risk, system_model, scorecard),
|
||||
"world_class_readiness": readiness,
|
||||
"package_assets": package_assets(package_map),
|
||||
"iteration_roadmap": {"items": roadmap},
|
||||
"report_contract": report_contract,
|
||||
@@ -625,6 +406,7 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"benchmark_highlights": [],
|
||||
"skill_ir": {
|
||||
"schema_version": skill_ir.get("schema_version", ""),
|
||||
"source_path": skill_ir_path,
|
||||
"target_count": len(skill_ir.get("targets", [])),
|
||||
"trigger_samples": len(skill_ir.get("trigger_surface", {}).get("should_trigger", [])),
|
||||
"output_eval_cases": len(skill_ir.get("eval_plan", {}).get("output", [])),
|
||||
@@ -664,6 +446,12 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"pair_count": output_blind_review.get("summary", {}).get("pair_count", 0),
|
||||
"answer_key_separate": output_blind_review.get("summary", {}).get("answer_key_separate", False),
|
||||
},
|
||||
"output_review_kit": {
|
||||
"ok": output_review_kit.get("ok", False),
|
||||
"summary": output_review_kit.get("summary", {}),
|
||||
"artifacts": output_review_kit.get("artifacts", {}),
|
||||
"failures": output_review_kit.get("failures", []),
|
||||
},
|
||||
"output_review_adjudication": {
|
||||
"ok": output_review_adjudication.get("ok", False),
|
||||
"summary": output_review_adjudication.get("summary", {}),
|
||||
@@ -671,6 +459,13 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"reviewed_at": output_review_adjudication.get("reviewed_at", ""),
|
||||
"failures": output_review_adjudication.get("failures", []),
|
||||
},
|
||||
"benchmark_reproducibility": {
|
||||
"ok": benchmark_reproducibility.get("ok", False),
|
||||
"summary": benchmark_reproducibility.get("summary", {}),
|
||||
"commit": benchmark_reproducibility.get("commit", ""),
|
||||
"missing_artifacts": benchmark_reproducibility.get("missing_artifacts", []),
|
||||
"limitations": benchmark_reproducibility.get("limitations", []),
|
||||
},
|
||||
"runtime_conformance": conformance.get("summary", {}),
|
||||
"runtime_permissions": {
|
||||
"ok": runtime_permissions.get("ok", False),
|
||||
@@ -738,6 +533,18 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"annotations": review_annotations.get("annotations", [])[:8],
|
||||
"failures": review_annotations.get("failures", []),
|
||||
},
|
||||
"world_class_evidence_plan": {
|
||||
"ok": world_class_evidence.get("ok", False),
|
||||
"summary": world_class_evidence.get("summary", {}),
|
||||
"tasks": world_class_evidence.get("tasks", [])[:8],
|
||||
"source_audit": world_class_evidence.get("source_audit", {}),
|
||||
},
|
||||
"world_class_evidence_ledger": {
|
||||
"ok": world_class_evidence_ledger.get("ok", False),
|
||||
"summary": world_class_evidence_ledger.get("summary", {}),
|
||||
"entries": world_class_evidence_ledger.get("entries", [])[:8],
|
||||
"source_plan": world_class_evidence_ledger.get("source_plan", {}),
|
||||
},
|
||||
"synthesis_highlights": synthesis,
|
||||
"artifact_design": q_review["artifact_design"],
|
||||
"prompt_quality": q_review["prompt_quality"],
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to assemble overview report quality, risk, and asset sections."
|
||||
|
||||
|
||||
def artifact_design_highlights(profile: dict) -> list[str]:
|
||||
primary = profile.get("primary_artifact", {})
|
||||
highlights = []
|
||||
if primary.get("direction"):
|
||||
highlights.append(primary["direction"])
|
||||
highlights.extend(profile.get("quality_gates", [])[:3])
|
||||
return highlights[:4]
|
||||
|
||||
|
||||
def prompt_quality_highlights(profile: dict) -> list[str]:
|
||||
highlights = []
|
||||
primary = profile.get("primary_task_family", {})
|
||||
complexity = profile.get("complexity", {})
|
||||
if primary.get("label"):
|
||||
highlights.append(f"Primary prompt task family: {primary['label']}.")
|
||||
if complexity.get("band"):
|
||||
highlights.append(f"Complexity: {complexity['band']} — {complexity.get('reason', '')}")
|
||||
for item in profile.get("quality_matrix", [])[:2]:
|
||||
highlights.append(f"{item.get('label', 'Quality')}: {item.get('score', 'n/a')}/100.")
|
||||
return highlights[:4]
|
||||
|
||||
|
||||
def system_model_highlights(model: dict) -> list[str]:
|
||||
highlights = []
|
||||
stability = model.get("stability", {})
|
||||
if stability:
|
||||
highlights.append(f"Stability: {stability.get('band', 'unknown')} ({stability.get('score', 'n/a')}/100).")
|
||||
boundary = model.get("boundary_map", {})
|
||||
if boundary.get("owned_job"):
|
||||
highlights.append(f"Owned job: {boundary['owned_job']}")
|
||||
for point in model.get("leverage_points", [])[:2]:
|
||||
if point.get("point"):
|
||||
highlights.append(f"Leverage: {point['point']} — {point.get('move', '')}")
|
||||
return highlights[:4]
|
||||
|
||||
|
||||
def risk_governance(output_risk: dict, system_model: dict, scorecard: dict) -> dict:
|
||||
risk_names = [
|
||||
("误触发风险", "trigger_score"),
|
||||
("输出漂移风险", "evidence_score"),
|
||||
("证据不足风险", "evidence_score"),
|
||||
("包体膨胀风险", "maintainability_score"),
|
||||
("跨平台迁移风险", "portability_score"),
|
||||
]
|
||||
risks = []
|
||||
for index, (name, metric_key) in enumerate(risk_names):
|
||||
score = scorecard.get(metric_key, {}).get("score", 50)
|
||||
probability = max(1, min(3, 4 - round(score / 34)))
|
||||
impact = 3 if index in {0, 2, 4} else 2
|
||||
risks.append(
|
||||
{
|
||||
"name": name,
|
||||
"impact": impact,
|
||||
"probability": probability,
|
||||
"signal": scorecard.get(metric_key, {}).get("reasons", ["证据不足"])[0],
|
||||
"response": "先补证据和边界,再增加包体复杂度。",
|
||||
}
|
||||
)
|
||||
human_boundary = system_model.get("boundary_map", {}).get("human_judgment_boundary", [])
|
||||
return {
|
||||
"risks": risks,
|
||||
"risk_families": output_risk.get("risk_families", []),
|
||||
"human_judgment_boundary": human_boundary,
|
||||
}
|
||||
|
||||
|
||||
def quality_review(
|
||||
strengths: list[str],
|
||||
scorecard: dict,
|
||||
artifact_design: dict,
|
||||
prompt_quality: dict,
|
||||
system_model: dict,
|
||||
) -> dict:
|
||||
gaps = []
|
||||
for key, payload in scorecard.items():
|
||||
if payload.get("score", 0) < 70:
|
||||
gaps.append(f"{payload.get('label', key)}需要补强:{payload.get('reasons', ['证据不足'])[0]}")
|
||||
return {
|
||||
"strengths": strengths,
|
||||
"gaps": gaps or ["当前关键证据较完整,优先保持轻量。"],
|
||||
"recommendations": [
|
||||
"先改触发边界,再扩展工作流。",
|
||||
"只把重复且稳定的步骤沉淀为脚本。",
|
||||
"每次升级后重新生成报告并检查分数原因。",
|
||||
],
|
||||
"artifact_design": {
|
||||
"design_system": artifact_design.get("design_system", "content-led editorial"),
|
||||
"highlights": artifact_design_highlights(artifact_design),
|
||||
},
|
||||
"prompt_quality": {
|
||||
"overall_quality_score": prompt_quality.get("overall_quality_score", "n/a"),
|
||||
"highlights": prompt_quality_highlights(prompt_quality),
|
||||
},
|
||||
"system_model": {
|
||||
"stability": system_model.get("stability", {}),
|
||||
"highlights": system_model_highlights(system_model),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def package_assets(package_map: list[dict]) -> dict:
|
||||
files = sum(item.get("file_count", 0) for item in package_map if item.get("kind") == "file")
|
||||
folders = [item for item in package_map if item.get("kind") == "folder"]
|
||||
distribution = [{"label": item["path"], "value": max(1, item.get("file_count", 1))} for item in package_map]
|
||||
return {
|
||||
"entries": package_map,
|
||||
"file_count": files + sum(item.get("file_count", 0) for item in folders),
|
||||
"folder_count": len(folders),
|
||||
"distribution": distribution,
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Source loading and package scanning helpers for skill overview reports."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to load source files and scan overview package assets."
|
||||
|
||||
|
||||
KNOWN_ENTRIES = [
|
||||
("SKILL.md", "Skill entrypoint"),
|
||||
("README.md", "Human-readable usage guide"),
|
||||
("agents/interface.yaml", "Neutral interface metadata"),
|
||||
("manifest.json", "Lifecycle and portability metadata"),
|
||||
("references", "Extended guidance and reusable notes"),
|
||||
("scripts", "Deterministic helpers or local tooling"),
|
||||
("evals", "Trigger and quality checks"),
|
||||
("reports", "Generated evidence and overview artifacts"),
|
||||
]
|
||||
|
||||
IGNORED_PACKAGE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"}
|
||||
|
||||
|
||||
def parse_frontmatter(text: str) -> tuple[dict, str]:
|
||||
lines = text.splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}, text
|
||||
try:
|
||||
end_index = lines[1:].index("---") + 1
|
||||
except ValueError:
|
||||
return {}, text
|
||||
frontmatter_text = "\n".join(lines[1:end_index])
|
||||
body = "\n".join(lines[end_index + 1 :]).lstrip()
|
||||
if yaml is not None:
|
||||
data = yaml.safe_load(frontmatter_text) or {}
|
||||
return data if isinstance(data, dict) else {}, body
|
||||
data = {}
|
||||
for line in frontmatter_text.splitlines():
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, value = line.split(":", 1)
|
||||
data[key.strip()] = value.strip().strip('"')
|
||||
return data, body
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
text = path.read_text(encoding="utf-8")
|
||||
if yaml is not None:
|
||||
payload = yaml.safe_load(text) or {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
return {}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def extract_title(body: str, fallback: str) -> str:
|
||||
for line in body.splitlines():
|
||||
if line.startswith("# "):
|
||||
return line[2:].strip()
|
||||
return fallback
|
||||
|
||||
|
||||
def parse_sections(body: str) -> dict[str, str]:
|
||||
sections: dict[str, list[str]] = {}
|
||||
current = "_preamble"
|
||||
sections[current] = []
|
||||
for line in body.splitlines():
|
||||
if line.startswith("## "):
|
||||
current = line[3:].strip()
|
||||
sections[current] = []
|
||||
continue
|
||||
sections[current].append(line)
|
||||
return {name: "\n".join(lines).strip() for name, lines in sections.items()}
|
||||
|
||||
|
||||
def extract_list_items(text: str) -> list[str]:
|
||||
items = []
|
||||
for line in text.splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
ordered = re.match(r"^\d+\.\s+(.*)$", stripped)
|
||||
bullet = re.match(r"^[-*]\s+(.*)$", stripped)
|
||||
match = ordered or bullet
|
||||
if match:
|
||||
items.append(match.group(1).strip())
|
||||
return items
|
||||
|
||||
|
||||
def summarize_logic(sections: dict[str, str]) -> list[str]:
|
||||
for key in ("Compact Workflow", "Workflow", "How It Works", "Logic", "Quick Start"):
|
||||
if key in sections:
|
||||
items = extract_list_items(sections[key])
|
||||
if items:
|
||||
return items[:5]
|
||||
return extract_list_items(sections.get("_preamble", ""))[:5] or [
|
||||
"Understand the request",
|
||||
"Execute the main task",
|
||||
"Validate the result",
|
||||
]
|
||||
|
||||
|
||||
def summarize_usage(sections: dict[str, str], default_prompt: str, description: str) -> list[str]:
|
||||
for key in ("How To Use", "Quick Start", "Usage", "Runbook"):
|
||||
if key in sections:
|
||||
items = extract_list_items(sections[key])
|
||||
if items:
|
||||
return items[:5]
|
||||
usage = []
|
||||
if default_prompt:
|
||||
usage.append(default_prompt)
|
||||
usage.append(f"Use this skill when the request matches: {description}")
|
||||
return usage[:5]
|
||||
|
||||
|
||||
def package_entries(skill_dir: Path) -> list[dict]:
|
||||
items = []
|
||||
for rel_path, label in KNOWN_ENTRIES:
|
||||
target = skill_dir / rel_path
|
||||
if target.exists():
|
||||
kind = "folder" if target.is_dir() else "file"
|
||||
if target.is_dir():
|
||||
count = len(
|
||||
[
|
||||
path
|
||||
for path in target.rglob("*")
|
||||
if path.is_file()
|
||||
and not path.is_symlink()
|
||||
and not any(part in IGNORED_PACKAGE_PARTS for part in path.relative_to(target).parts)
|
||||
and path.suffix not in {".pyc", ".pyo"}
|
||||
]
|
||||
)
|
||||
else:
|
||||
count = 1
|
||||
items.append({"path": rel_path, "label": label, "kind": kind, "file_count": count})
|
||||
return items
|
||||
@@ -0,0 +1,128 @@
|
||||
"""World-class evidence shaping for the static skill overview report."""
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to summarize world-class evidence readiness and roadmap actions."
|
||||
|
||||
WORLD_CLASS_ENTRY_COPY = {
|
||||
"provider-holdout": {
|
||||
"label_zh": "提供商留出",
|
||||
"label_en": "Provider Holdout",
|
||||
"summary_zh": "缺少真实 provider 模型运行和 token metadata。",
|
||||
"summary_en": "Missing a real provider model run and token metadata.",
|
||||
},
|
||||
"human-adjudication": {
|
||||
"label_zh": "人工盲评",
|
||||
"label_en": "Human Adjudication",
|
||||
"summary_zh": "盲评 pair 仍待真实 reviewer 决策。",
|
||||
"summary_en": "Blind-review pairs still need real reviewer decisions.",
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"label_zh": "原生权限",
|
||||
"label_en": "Native Permission",
|
||||
"summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。",
|
||||
"summary_en": "Native runtime enforcement still needs target-client or external-installer proof.",
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"label_zh": "原生遥测",
|
||||
"label_en": "Native Telemetry",
|
||||
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
|
||||
"summary_en": "Real external-client metadata-only events have not been imported yet.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def world_class_readiness(ledger: dict) -> dict:
|
||||
summary = ledger.get("summary", {}) if isinstance(ledger, dict) else {}
|
||||
entries = ledger.get("entries", []) if isinstance(ledger, dict) else []
|
||||
if not isinstance(entries, list):
|
||||
entries = []
|
||||
source_check_count = int(summary.get("source_check_count", 0) or 0)
|
||||
source_pass_count = int(summary.get("source_pass_count", 0) or 0)
|
||||
pending_count = int(summary.get("pending_count", len(entries)) or 0)
|
||||
accepted_count = int(summary.get("accepted_count", 0) or 0)
|
||||
entry_count = int(summary.get("ledger_entry_count", len(entries)) or 0)
|
||||
ready = bool(summary.get("ready_to_claim_world_class", False))
|
||||
decision = str(summary.get("decision", "not-generated" if entry_count == 0 else "evidence-pending"))
|
||||
if entry_count == 0:
|
||||
conclusion_zh = "未生成 world-class ledger;当前报告不会宣称世界级完成。"
|
||||
conclusion_en = "No world-class ledger was generated; this report does not claim world-class completion."
|
||||
elif ready:
|
||||
conclusion_zh = "世界级证据已被 ledger 接受,可进入公开 claim 前的最终复核。"
|
||||
conclusion_en = "World-class evidence is accepted by the ledger and can move to final claim review."
|
||||
else:
|
||||
conclusion_zh = f"世界级证据尚未完成:{pending_count} 项待补,{accepted_count} 项已接受。"
|
||||
conclusion_en = f"World-class evidence is not complete: {pending_count} pending, {accepted_count} accepted."
|
||||
|
||||
items = []
|
||||
for entry in entries[:4]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
key = str(entry.get("key", ""))
|
||||
copy = WORLD_CLASS_ENTRY_COPY.get(key, {})
|
||||
category = str(entry.get("category", "external"))
|
||||
checklist = entry.get("source_checklist", [])
|
||||
blocked_checks = [
|
||||
str(check.get("label", ""))
|
||||
for check in checklist
|
||||
if isinstance(check, dict) and check.get("status") == "blocked" and str(check.get("label", "")).strip()
|
||||
][:3]
|
||||
items.append(
|
||||
{
|
||||
"key": key,
|
||||
"label_zh": copy.get("label_zh", str(entry.get("label", key))),
|
||||
"label_en": copy.get("label_en", str(entry.get("label", key))),
|
||||
"category": category,
|
||||
"category_zh": "人工证据" if category == "human" else "外部证据",
|
||||
"category_en": "Human evidence" if category == "human" else "External evidence",
|
||||
"status": str(entry.get("status", "pending")),
|
||||
"summary_zh": copy.get("summary_zh", str(entry.get("current", ""))),
|
||||
"summary_en": copy.get("summary_en", str(entry.get("current", ""))),
|
||||
"blocked_checks": blocked_checks,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"ready": ready,
|
||||
"decision": decision,
|
||||
"entry_count": entry_count,
|
||||
"pending_count": pending_count,
|
||||
"accepted_count": accepted_count,
|
||||
"external_pending_count": int(summary.get("external_pending_count", 0) or 0),
|
||||
"human_pending_count": int(summary.get("human_pending_count", 0) or 0),
|
||||
"source_check_count": source_check_count,
|
||||
"source_pass_count": source_pass_count,
|
||||
"conclusion_zh": conclusion_zh,
|
||||
"conclusion_en": conclusion_en,
|
||||
"entries": items,
|
||||
}
|
||||
|
||||
|
||||
def world_class_roadmap_item(readiness: dict) -> dict | None:
|
||||
if not isinstance(readiness, dict):
|
||||
return None
|
||||
pending_count = int(readiness.get("pending_count", 0) or 0)
|
||||
if readiness.get("ready") or pending_count <= 0:
|
||||
return None
|
||||
external_pending_count = int(readiness.get("external_pending_count", 0) or 0)
|
||||
human_pending_count = int(readiness.get("human_pending_count", 0) or 0)
|
||||
actions = []
|
||||
entries = [entry for entry in readiness.get("entries", []) if isinstance(entry, dict)]
|
||||
for entry in entries[:2]:
|
||||
label = str(entry.get("label_zh") or entry.get("key") or "证据项")
|
||||
summary = str(entry.get("summary_zh") or "仍待补充真实证据。")
|
||||
actions.append(f"补齐{label}证据:{summary}")
|
||||
remaining = pending_count - len(actions)
|
||||
if remaining > 0:
|
||||
actions.append(
|
||||
f"按 ledger 总量继续补齐 {pending_count} 项待补证据"
|
||||
f"(外部 {external_pending_count} 项、人工 {human_pending_count} 项;"
|
||||
f"未展开 {remaining} 项),并保持 claim guard 为 pending 状态。"
|
||||
)
|
||||
else:
|
||||
actions.append("提交有效 intake packet,并让 ledger 通过 artifact SHA-256 校验。")
|
||||
return {
|
||||
"title": "补齐世界证据",
|
||||
"why": f"世界级证据仍有 {pending_count} 项待补;公开完成态 claim 必须继续保持阻塞。",
|
||||
"actions": actions[:3],
|
||||
"unlocks": "全部外部/人工证据被 ledger 接受后,才能进入公开 world-class claim 复核。",
|
||||
"source": "world_class_evidence_ledger",
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by render_daily_skillops_report.py to score SkillOps opportunities without writing files."
|
||||
|
||||
ACTION_POLICY: dict[str, dict[str, str]] = {
|
||||
"language_default": {
|
||||
"action_type": "patch_existing_skill",
|
||||
"opportunity_type": "report-experience",
|
||||
"reason": "Repeated language preferences can improve existing report templates after approval.",
|
||||
},
|
||||
"report_ui": {
|
||||
"action_type": "patch_existing_skill",
|
||||
"opportunity_type": "artifact-quality",
|
||||
"reason": "Repeated report layout feedback maps to existing renderer and design doctrine changes.",
|
||||
},
|
||||
"approval_safety": {
|
||||
"action_type": "agents_update",
|
||||
"opportunity_type": "governance",
|
||||
"reason": "Repeated privacy and approval signals should tighten durable operating guidance.",
|
||||
},
|
||||
"delivery_format": {
|
||||
"action_type": "patch_existing_skill",
|
||||
"opportunity_type": "artifact-discoverability",
|
||||
"reason": "Repeated delivery-path requests can improve CLI and README artifact discoverability.",
|
||||
},
|
||||
"evidence_testing": {
|
||||
"action_type": "add_eval",
|
||||
"opportunity_type": "quality-gate",
|
||||
"reason": "Repeated verification and evidence requests should become focused checks.",
|
||||
},
|
||||
}
|
||||
|
||||
RISK_PENALTY = {
|
||||
"low": 0,
|
||||
"medium": 8,
|
||||
"high": 18,
|
||||
}
|
||||
|
||||
|
||||
def _stable_id(pattern_id: str, target_files: list[str], title: str) -> str:
|
||||
digest = hashlib.sha1(f"{pattern_id}:{title}:{','.join(target_files)}".encode("utf-8")).hexdigest()
|
||||
return f"skillops-{digest[:10]}"
|
||||
|
||||
|
||||
def _as_int(value: Any, default: int = 0) -> int:
|
||||
if isinstance(value, bool):
|
||||
return default
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _risk_level(proposal: dict[str, Any]) -> str:
|
||||
risk = str(proposal.get("risk_level") or "medium").lower()
|
||||
return risk if risk in RISK_PENALTY else "medium"
|
||||
|
||||
|
||||
def _target_files(proposal: dict[str, Any]) -> list[str]:
|
||||
files = proposal.get("target_files")
|
||||
if not isinstance(files, list):
|
||||
return []
|
||||
return [str(item) for item in files if str(item).strip()]
|
||||
|
||||
|
||||
def _verification_commands(proposal: dict[str, Any]) -> list[str]:
|
||||
commands = proposal.get("verification_commands")
|
||||
if not isinstance(commands, list):
|
||||
return []
|
||||
return [str(item) for item in commands if str(item).strip()]
|
||||
|
||||
|
||||
def _score(pattern: dict[str, Any], proposal: dict[str, Any]) -> tuple[int, list[str]]:
|
||||
support = _as_int(pattern.get("support_count"), _as_int(proposal.get("support_count"), 0))
|
||||
confidence = pattern.get("confidence", 0.5)
|
||||
try:
|
||||
confidence_score = int(float(confidence) * 25)
|
||||
except (TypeError, ValueError):
|
||||
confidence_score = 12
|
||||
target_files = _target_files(proposal)
|
||||
verification_commands = _verification_commands(proposal)
|
||||
risk = _risk_level(proposal)
|
||||
reasons = [
|
||||
f"support_count={support}",
|
||||
f"confidence={confidence}",
|
||||
f"risk={risk}",
|
||||
]
|
||||
score = 28
|
||||
score += min(30, support * 10)
|
||||
score += max(0, min(25, confidence_score))
|
||||
if target_files:
|
||||
score += 8
|
||||
reasons.append("target_files_present")
|
||||
else:
|
||||
reasons.append("target_files_missing")
|
||||
if verification_commands:
|
||||
score += 7
|
||||
reasons.append("verification_present")
|
||||
else:
|
||||
reasons.append("verification_missing")
|
||||
score -= RISK_PENALTY[risk]
|
||||
if proposal.get("requires_approval") is True:
|
||||
reasons.append("approval_required")
|
||||
return max(0, min(100, score)), reasons
|
||||
|
||||
|
||||
def _decision(score: int, proposal: dict[str, Any]) -> str:
|
||||
if score >= 85:
|
||||
return "ready_for_approval_review"
|
||||
if score >= 70:
|
||||
return "proposal_review"
|
||||
if score >= 50:
|
||||
return "observe_more_evidence"
|
||||
if proposal.get("requires_approval") is True:
|
||||
return "report_only"
|
||||
return "no_action"
|
||||
|
||||
|
||||
def _priority(score: int, risk: str) -> str:
|
||||
if score >= 85 and risk == "low":
|
||||
return "high"
|
||||
if score >= 70:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def build_opportunities(patterns: list[dict[str, Any]], proposals: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
pattern_by_id = {str(item.get("pattern_id", "")): item for item in patterns if isinstance(item, dict)}
|
||||
opportunities: list[dict[str, Any]] = []
|
||||
for proposal in proposals:
|
||||
if not isinstance(proposal, dict):
|
||||
continue
|
||||
pattern_id = str(proposal.get("pattern_id") or "generic")
|
||||
pattern = pattern_by_id.get(pattern_id, {})
|
||||
policy = ACTION_POLICY.get(
|
||||
pattern_id,
|
||||
{
|
||||
"action_type": "report_only",
|
||||
"opportunity_type": "manual-review",
|
||||
"reason": "Unknown pattern family requires manual review before any durable change.",
|
||||
},
|
||||
)
|
||||
target_files = _target_files(proposal)
|
||||
verification_commands = _verification_commands(proposal)
|
||||
risk = _risk_level(proposal)
|
||||
score, score_reasons = _score(pattern, proposal)
|
||||
evidence_refs = proposal.get("evidence_refs")
|
||||
if not isinstance(evidence_refs, list):
|
||||
evidence_refs = []
|
||||
opportunities.append(
|
||||
{
|
||||
"opportunity_id": _stable_id(pattern_id, target_files, str(proposal.get("title") or "")),
|
||||
"proposal_id": proposal.get("proposal_id", ""),
|
||||
"pattern_id": pattern_id,
|
||||
"title": proposal.get("title", ""),
|
||||
"opportunity_type": policy["opportunity_type"],
|
||||
"action_type": policy["action_type"],
|
||||
"decision": _decision(score, proposal),
|
||||
"priority": _priority(score, risk),
|
||||
"score": score,
|
||||
"score_reasons": score_reasons,
|
||||
"policy_reason": policy["reason"],
|
||||
"risk_level": risk,
|
||||
"requires_approval": proposal.get("requires_approval") is True,
|
||||
"write_allowed_without_approval": False,
|
||||
"evidence_count": len(evidence_refs) or _as_int(pattern.get("support_count"), 0),
|
||||
"target_files": target_files,
|
||||
"verification_commands": verification_commands,
|
||||
}
|
||||
)
|
||||
opportunities.sort(key=lambda item: (-int(item["score"]), str(item["opportunity_id"])))
|
||||
return opportunities
|
||||
|
||||
|
||||
def summarize_opportunities(opportunities: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
by_action: dict[str, int] = {}
|
||||
by_decision: dict[str, int] = {}
|
||||
for item in opportunities:
|
||||
by_action[str(item.get("action_type", ""))] = by_action.get(str(item.get("action_type", "")), 0) + 1
|
||||
by_decision[str(item.get("decision", ""))] = by_decision.get(str(item.get("decision", "")), 0) + 1
|
||||
top_score = int(opportunities[0]["score"]) if opportunities else 0
|
||||
return {
|
||||
"opportunity_count": len(opportunities),
|
||||
"top_score": top_score,
|
||||
"ready_for_approval_review_count": by_decision.get("ready_for_approval_review", 0),
|
||||
"proposal_review_count": by_decision.get("proposal_review", 0),
|
||||
"observe_more_evidence_count": by_decision.get("observe_more_evidence", 0),
|
||||
"report_only_count": by_decision.get("report_only", 0),
|
||||
"action_type_counts": by_action,
|
||||
"decision_counts": by_decision,
|
||||
}
|
||||
|
||||
|
||||
def decision_policy_contract() -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": "1.0",
|
||||
"contract": "skillops-opportunity-scoring",
|
||||
"score_range": [0, 100],
|
||||
"score_bands": {
|
||||
"85-100": "ready_for_approval_review",
|
||||
"70-84": "proposal_review",
|
||||
"50-69": "observe_more_evidence",
|
||||
"0-49": "report_only_or_no_action",
|
||||
},
|
||||
"action_types": sorted({item["action_type"] for item in ACTION_POLICY.values()} | {"report_only"}),
|
||||
"writes_source_files": False,
|
||||
"auto_patch_enabled": False,
|
||||
"approval_required_for_writes": True,
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT_INTERFACE = "cli"
|
||||
SCRIPT_INTERFACE_REASON = "Scans an explicit local source file and summarizes redacted repeated user preference signals."
|
||||
|
||||
TEXT_FIELDS = ("text", "message", "content", "excerpt", "prompt", "note", "body")
|
||||
HISTORY_FILENAMES = {".zsh_history", ".bash_history", ".fish_history", "History"}
|
||||
SECRET_PATTERNS = [
|
||||
re.compile(r"sk-[A-Za-z0-9_-]{12,}"),
|
||||
re.compile(r"AKIA[0-9A-Z]{12,}"),
|
||||
re.compile(r"(?i)\b(api[_-]?key|token|password|secret)\b\s*[:=]\s*[^\s,;]+"),
|
||||
]
|
||||
EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
|
||||
LOCAL_PATH_RE = re.compile(r"/Users/[^\s'\"<>]+")
|
||||
|
||||
PATTERN_RULES = [
|
||||
{
|
||||
"pattern_id": "language_default",
|
||||
"label": "Default language preference",
|
||||
"signal_type": "report-language",
|
||||
"keywords": ["中文", "简体", "默认中文", "英文", "双语", "language", "bilingual", "chinese", "english"],
|
||||
"recommended_action": "Keep generated reports Chinese-first with an English switch where user-facing.",
|
||||
},
|
||||
{
|
||||
"pattern_id": "report_ui",
|
||||
"label": "Report UI and visualization preference",
|
||||
"signal_type": "artifact-design",
|
||||
"keywords": ["报告", "html", "图表", "排版", "ui", "kami", "白底", "模块", "导航", "report", "chart", "layout"],
|
||||
"recommended_action": "Prioritize white-background Kami-style reports with readable charts and stable navigation.",
|
||||
},
|
||||
{
|
||||
"pattern_id": "approval_safety",
|
||||
"label": "Approval and privacy boundary",
|
||||
"signal_type": "governance",
|
||||
"keywords": ["审批", "授权", "不要扫描", "隐私", "私人", "日志", "明确路径", "回滚", "提案", "批准", "approval", "privacy", "private", "proposal", "rollback"],
|
||||
"recommended_action": "Keep adaptive work proposal-only until a reviewer approves an allowlisted patch path.",
|
||||
},
|
||||
{
|
||||
"pattern_id": "delivery_format",
|
||||
"label": "Delivery format preference",
|
||||
"signal_type": "artifact-format",
|
||||
"keywords": ["markdown", "pdf", "word", "docx", "html", "地址", "路径", "打开", "输出", "交付"],
|
||||
"recommended_action": "Surface stable artifact paths and formats in CLI output and generated summaries.",
|
||||
},
|
||||
{
|
||||
"pattern_id": "evidence_testing",
|
||||
"label": "Evidence and testing preference",
|
||||
"signal_type": "quality-gate",
|
||||
"keywords": ["测试", "验证", "ci", "证据", "覆盖", "github", "push", "evidence", "review"],
|
||||
"recommended_action": "Attach focused tests and refreshed evidence reports to every non-trivial skill upgrade.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def display_path(path: Path, skill_dir: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
return f"[external-explicit-source]/{path.name}"
|
||||
|
||||
|
||||
def resolve_output(skill_dir: Path, value: str) -> Path:
|
||||
path = Path(value)
|
||||
return path if path.is_absolute() else skill_dir / path
|
||||
|
||||
|
||||
def source_fingerprint(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def redact_text(text: str) -> str:
|
||||
redacted = text
|
||||
for pattern in SECRET_PATTERNS:
|
||||
redacted = pattern.sub("[REDACTED_SECRET]", redacted)
|
||||
redacted = EMAIL_RE.sub("[REDACTED_EMAIL]", redacted)
|
||||
redacted = LOCAL_PATH_RE.sub("[LOCAL_PATH]", redacted)
|
||||
redacted = re.sub(r"\s+", " ", redacted).strip()
|
||||
if len(redacted) > 240:
|
||||
return redacted[:237].rstrip() + "..."
|
||||
return redacted
|
||||
|
||||
|
||||
def extract_text(raw: Any) -> str:
|
||||
if isinstance(raw, str):
|
||||
return raw
|
||||
if not isinstance(raw, dict):
|
||||
return ""
|
||||
for field in TEXT_FIELDS:
|
||||
value = raw.get(field)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value
|
||||
messages = raw.get("messages")
|
||||
if isinstance(messages, list):
|
||||
parts = []
|
||||
for item in messages:
|
||||
if isinstance(item, dict):
|
||||
content = item.get("content")
|
||||
if isinstance(content, str):
|
||||
parts.append(content)
|
||||
elif isinstance(item, str):
|
||||
parts.append(item)
|
||||
return "\n".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def load_records(source: Path) -> tuple[list[dict[str, str]], list[str]]:
|
||||
records: list[dict[str, str]] = []
|
||||
failures: list[str] = []
|
||||
text = source.read_text(encoding="utf-8", errors="replace")
|
||||
if source.suffix.lower() == ".jsonl":
|
||||
for index, line in enumerate(text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"line {index}: invalid JSONL source: {exc.msg}")
|
||||
continue
|
||||
extracted = extract_text(raw)
|
||||
if not extracted.strip():
|
||||
failures.append(f"line {index}: no supported text field found")
|
||||
continue
|
||||
records.append({"record_id": f"line-{index}", "excerpt": redact_text(extracted)})
|
||||
else:
|
||||
for index, line in enumerate(text.splitlines(), start=1):
|
||||
if line.strip():
|
||||
records.append({"record_id": f"line-{index}", "excerpt": redact_text(line)})
|
||||
return records, failures
|
||||
|
||||
|
||||
def classify_patterns(records: list[dict[str, str]], min_support: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
patterns: list[dict[str, Any]] = []
|
||||
discarded: list[dict[str, Any]] = []
|
||||
for rule in PATTERN_RULES:
|
||||
matches = []
|
||||
for record in records:
|
||||
lowered = record["excerpt"].lower()
|
||||
if any(keyword.lower() in lowered for keyword in rule["keywords"]):
|
||||
matches.append(record)
|
||||
if not matches:
|
||||
continue
|
||||
item = {
|
||||
"pattern_id": rule["pattern_id"],
|
||||
"label": rule["label"],
|
||||
"signal_type": rule["signal_type"],
|
||||
"support_count": len(matches),
|
||||
"confidence": min(0.95, round(0.55 + (0.12 * len(matches)), 2)),
|
||||
"reason": f"{len(matches)} redacted records matched repeated {rule['signal_type']} signals.",
|
||||
"recommended_action": rule["recommended_action"],
|
||||
"evidence": matches[:3],
|
||||
}
|
||||
if len(matches) >= min_support:
|
||||
patterns.append(item)
|
||||
else:
|
||||
discarded.append({**item, "discard_reason": f"support_count below min_support {min_support}"})
|
||||
return patterns, discarded
|
||||
|
||||
|
||||
def build_report(
|
||||
skill_dir: Path,
|
||||
source: Path,
|
||||
min_support: int,
|
||||
generated_at: str,
|
||||
allow_history_source: bool,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
source = source.resolve()
|
||||
failures: list[str] = []
|
||||
records: list[dict[str, str]] = []
|
||||
fingerprint = ""
|
||||
if not source.exists():
|
||||
failures.append(f"Explicit source does not exist: {display_path(source, skill_dir)}")
|
||||
elif not source.is_file():
|
||||
failures.append(f"Explicit source must be a file: {display_path(source, skill_dir)}")
|
||||
elif source.name in HISTORY_FILENAMES and not allow_history_source:
|
||||
failures.append(f"Refusing private history source by default: {source.name}")
|
||||
else:
|
||||
fingerprint = source_fingerprint(source)
|
||||
records, load_failures = load_records(source)
|
||||
failures.extend(load_failures)
|
||||
patterns, discarded = classify_patterns(records, min_support) if not failures else ([], [])
|
||||
return {
|
||||
"ok": not failures,
|
||||
"schema_version": "1.0",
|
||||
"generated_at": generated_at,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
"source": {
|
||||
"label": source.name,
|
||||
"path": display_path(source, skill_dir),
|
||||
"fingerprint_sha256": fingerprint,
|
||||
"explicit_source": True,
|
||||
"record_count": len(records),
|
||||
},
|
||||
"privacy_contract": {
|
||||
"local_only": True,
|
||||
"explicit_source_required": True,
|
||||
"implicit_private_log_scan": False,
|
||||
"raw_content_stored": False,
|
||||
"redacted_excerpts_only": True,
|
||||
"redacted_excerpt_limit": 240,
|
||||
"writes_repository_files": False,
|
||||
},
|
||||
"summary": {
|
||||
"record_count": len(records),
|
||||
"pattern_count": len(patterns),
|
||||
"discarded_signal_count": len(discarded),
|
||||
"min_support": min_support,
|
||||
"failure_count": len(failures),
|
||||
},
|
||||
"patterns": patterns,
|
||||
"discarded_signals": discarded,
|
||||
"failures": failures,
|
||||
"artifacts": {
|
||||
"json": "reports/user_patterns.json",
|
||||
"markdown": "reports/user_patterns.md",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(report: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# User Pattern Summary",
|
||||
"",
|
||||
f"- Generated at: `{report['generated_at']}`",
|
||||
f"- Local only: `{str(report['privacy_contract']['local_only']).lower()}`",
|
||||
f"- Explicit source: `{report['source']['path']}`",
|
||||
f"- Records: `{report['summary']['record_count']}`",
|
||||
f"- Patterns: `{report['summary']['pattern_count']}`",
|
||||
f"- Discarded signals: `{report['summary']['discarded_signal_count']}`",
|
||||
"",
|
||||
"## Privacy Contract",
|
||||
"",
|
||||
"- No implicit private log scan.",
|
||||
"- No unredacted raw content stored.",
|
||||
"- Scan and proposal stages do not write source files.",
|
||||
"",
|
||||
"## Patterns",
|
||||
"",
|
||||
]
|
||||
if not report["patterns"]:
|
||||
lines.append("- No repeated pattern met the support threshold.")
|
||||
for pattern in report["patterns"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {pattern['label']}",
|
||||
"",
|
||||
f"- Pattern: `{pattern['pattern_id']}`",
|
||||
f"- Support: `{pattern['support_count']}`",
|
||||
f"- Confidence: `{pattern['confidence']}`",
|
||||
f"- Reason: {pattern['reason']}",
|
||||
f"- Recommended action: {pattern['recommended_action']}",
|
||||
"- Redacted evidence:",
|
||||
]
|
||||
)
|
||||
for item in pattern["evidence"]:
|
||||
lines.append(f" - `{item['record_id']}`: {item['excerpt']}")
|
||||
lines.append("")
|
||||
if report["discarded_signals"]:
|
||||
lines.extend(["## Discarded Signals", ""])
|
||||
for item in report["discarded_signals"]:
|
||||
lines.append(f"- `{item['pattern_id']}`: {item['discard_reason']}")
|
||||
if report["failures"]:
|
||||
lines.extend(["", "## Failures", ""])
|
||||
lines.extend(f"- {failure}" for failure in report["failures"])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Summarize repeated user preference signals from one explicit local source file.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--source", required=True)
|
||||
parser.add_argument("--output-json", default="reports/user_patterns.json")
|
||||
parser.add_argument("--output-md", default="reports/user_patterns.md")
|
||||
parser.add_argument("--min-support", type=int, default=2)
|
||||
parser.add_argument("--generated-at", default=utc_now())
|
||||
parser.add_argument("--allow-history-source", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
report = build_report(
|
||||
skill_dir,
|
||||
Path(args.source),
|
||||
min_support=max(2, args.min_support),
|
||||
generated_at=args.generated_at,
|
||||
allow_history_source=args.allow_history_source,
|
||||
)
|
||||
if report["ok"]:
|
||||
output_json = resolve_output(skill_dir, args.output_json)
|
||||
output_md = resolve_output(skill_dir, args.output_md)
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
report["artifacts"] = {
|
||||
"json": display_path(output_json, skill_dir),
|
||||
"markdown": display_path(output_md, skill_dir),
|
||||
}
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(report), encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -3,14 +3,19 @@ import argparse
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from simulate_install import simulate_install
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SKILL_NAME = "yao-meta-skill"
|
||||
SENTINEL_NAME = ".yao-local-install.json"
|
||||
DEFAULT_INSTALL_DIR = Path.home() / ".agents" / "skills.disabled" / SKILL_NAME
|
||||
ACTIVE_INSTALL_DIR = Path.home() / ".agents" / "skills" / SKILL_NAME
|
||||
DEFAULT_PACKAGE_DIR = ROOT / "dist"
|
||||
ALLOW_UNTRACKED_PREFIXES = {
|
||||
".github",
|
||||
"agents",
|
||||
@@ -133,6 +138,31 @@ def write_sentinel(root: Path, install_dir: Path, dry_run: bool) -> None:
|
||||
(install_dir / SENTINEL_NAME).write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def install_preflight(root: Path, package_dir: Path, generated_at: str) -> dict:
|
||||
package_dir = package_dir.resolve()
|
||||
with tempfile.TemporaryDirectory(prefix="yao-local-install-preflight-") as temp_dir:
|
||||
report = simulate_install(root, package_dir, Path(temp_dir), generated_at)
|
||||
summary = report.get("summary", {}) if isinstance(report.get("summary"), dict) else {}
|
||||
permission_failures = int(summary.get("installer_permission_failure_count", 0) or 0)
|
||||
if not report.get("ok") or permission_failures:
|
||||
failure_preview = "; ".join(str(item) for item in report.get("failures", [])[:3])
|
||||
if permission_failures and not failure_preview:
|
||||
failure_preview = f"{permission_failures} installer permission failures"
|
||||
raise ValueError(f"Install preflight failed for {package_dir}: {failure_preview or 'unknown failure'}")
|
||||
return {
|
||||
"ok": True,
|
||||
"package_dir": str(package_dir),
|
||||
"archive_extracted": bool(summary.get("archive_extracted")),
|
||||
"adapter_count": int(summary.get("adapter_count", 0) or 0),
|
||||
"installer_permission_enforced_count": int(summary.get("installer_permission_enforced_count", 0) or 0),
|
||||
"installer_permission_failure_count": permission_failures,
|
||||
"permission_target_count": int(summary.get("permission_target_count", 0) or 0),
|
||||
"permission_capability_count": int(summary.get("permission_capability_count", 0) or 0),
|
||||
"failure_count": int(summary.get("failure_count", 0) or 0),
|
||||
"warning_count": int(summary.get("warning_count", 0) or 0),
|
||||
}
|
||||
|
||||
|
||||
def remove_stale_files(install_dir: Path, desired_files: set[Path], dry_run: bool) -> list[str]:
|
||||
removed = []
|
||||
if not install_dir.exists():
|
||||
@@ -176,10 +206,21 @@ def copy_files(root: Path, install_dir: Path, files: list[Path], dry_run: bool)
|
||||
return copied, skipped
|
||||
|
||||
|
||||
def sync_local_install(root: Path, install_dir: Path, dry_run: bool = False) -> dict:
|
||||
def sync_local_install(
|
||||
root: Path,
|
||||
install_dir: Path,
|
||||
dry_run: bool = False,
|
||||
package_dir: Path | None = None,
|
||||
generated_at: str | None = None,
|
||||
skip_install_preflight: bool = False,
|
||||
) -> dict:
|
||||
root = root.resolve()
|
||||
install_dir = install_dir.resolve()
|
||||
validate_install_dir(root, install_dir)
|
||||
preflight = {"ok": True, "skipped": True}
|
||||
if not skip_install_preflight:
|
||||
resolved_package_dir = (package_dir or DEFAULT_PACKAGE_DIR).resolve()
|
||||
preflight = install_preflight(root, resolved_package_dir, generated_at or str(date.today()))
|
||||
files, skipped_untracked = candidate_files(root)
|
||||
desired_files = set(files)
|
||||
desired_files.add(Path(SENTINEL_NAME))
|
||||
@@ -197,6 +238,7 @@ def sync_local_install(root: Path, install_dir: Path, dry_run: bool = False) ->
|
||||
"removed_count": len(removed),
|
||||
"skipped_source_count": len(skipped_sources),
|
||||
"skipped_untracked_count": len(skipped_untracked),
|
||||
"install_preflight": preflight,
|
||||
"copied_samples": copied[:10],
|
||||
"removed_samples": removed[:10],
|
||||
"skipped_source_samples": skipped_sources[:10],
|
||||
@@ -208,13 +250,23 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Sync the current yao-meta-skill source into a managed local skill mirror.")
|
||||
parser.add_argument("--root", default=str(ROOT))
|
||||
parser.add_argument("--install-dir", default=str(DEFAULT_INSTALL_DIR))
|
||||
parser.add_argument("--package-dir", default=str(DEFAULT_PACKAGE_DIR))
|
||||
parser.add_argument("--generated-at", default=str(date.today()))
|
||||
parser.add_argument("--skip-install-preflight", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
root = Path(args.root).resolve()
|
||||
package_dir = Path(args.package_dir)
|
||||
if not package_dir.is_absolute():
|
||||
package_dir = root / package_dir
|
||||
try:
|
||||
result = sync_local_install(
|
||||
Path(args.root),
|
||||
root,
|
||||
resolve_install_dir(args.install_dir),
|
||||
dry_run=args.dry_run,
|
||||
package_dir=package_dir,
|
||||
generated_at=args.generated_at,
|
||||
skip_install_preflight=args.skip_install_preflight,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError, ValueError) as exc:
|
||||
result = {"ok": False, "failures": [str(exc)]}
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import stat
|
||||
import struct
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
from emit_telemetry_event import append_event, default_spool_path
|
||||
from render_adoption_drift_report import display_path, normalize_event, skill_defaults
|
||||
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
DEFAULT_HOST_NAME = "com.yao.meta_skill.telemetry"
|
||||
DEFAULT_DESCRIPTION = "Yao metadata-only telemetry native messaging host"
|
||||
SKILL_DIR_ENV = "YAO_TELEMETRY_SKILL_DIR"
|
||||
EVENTS_ENV = "YAO_TELEMETRY_EVENTS"
|
||||
MAX_MESSAGE_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
def read_native_message(stream: BinaryIO) -> dict[str, Any] | None:
|
||||
header = stream.read(4)
|
||||
if not header:
|
||||
return None
|
||||
if len(header) != 4:
|
||||
raise ValueError("native message header must be exactly 4 bytes")
|
||||
size = struct.unpack("<I", header)[0]
|
||||
if size > MAX_MESSAGE_BYTES:
|
||||
raise ValueError(f"native message is too large: {size} bytes")
|
||||
payload = stream.read(size)
|
||||
if len(payload) != size:
|
||||
raise ValueError("native message payload ended before declared length")
|
||||
message = json.loads(payload.decode("utf-8"))
|
||||
if not isinstance(message, dict):
|
||||
raise ValueError("native message must be a JSON object")
|
||||
return message
|
||||
|
||||
|
||||
def write_native_message(stream: BinaryIO, payload: dict[str, Any]) -> None:
|
||||
data = json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
||||
stream.write(struct.pack("<I", len(data)))
|
||||
stream.write(data)
|
||||
stream.flush()
|
||||
|
||||
|
||||
def emit_native_message(
|
||||
skill_dir: Path,
|
||||
output_jsonl: Path,
|
||||
raw_message: dict[str, Any],
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
raw = dict(raw_message)
|
||||
raw.setdefault("source", "external")
|
||||
raw.setdefault("command", "native-messaging-host")
|
||||
event, failures = normalize_event(raw, skill_defaults(skill_dir), "native-host-message")
|
||||
if event and not failures and not dry_run:
|
||||
append_event(output_jsonl, event)
|
||||
return {
|
||||
"ok": not failures,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": "native-messaging-host",
|
||||
"skill_dir": display_path(skill_dir),
|
||||
"output_jsonl": display_path(output_jsonl),
|
||||
"dry_run": dry_run,
|
||||
"emitted": bool(event and not failures and not dry_run),
|
||||
"event": event or {},
|
||||
"failures": failures,
|
||||
}
|
||||
|
||||
|
||||
def write_launcher(path: Path, host_script: Path, skill_dir: Path, output_jsonl: Path) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
script = "\n".join(
|
||||
[
|
||||
"#!/bin/sh",
|
||||
"set -eu",
|
||||
(
|
||||
"exec python3 "
|
||||
f"{shlex.quote(str(host_script.resolve()))} "
|
||||
f"{shlex.quote(str(skill_dir.resolve()))} "
|
||||
"--stdio "
|
||||
f"--output-jsonl {shlex.quote(str(output_jsonl.resolve()))}"
|
||||
),
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text(script, encoding="utf-8")
|
||||
current = path.stat().st_mode
|
||||
path.chmod(current | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
|
||||
|
||||
|
||||
def write_manifest(
|
||||
manifest_path: Path,
|
||||
host_name: str,
|
||||
description: str,
|
||||
host_path: Path,
|
||||
allowed_origins: list[str],
|
||||
) -> dict[str, Any]:
|
||||
failures: list[str] = []
|
||||
if not allowed_origins:
|
||||
failures.append("At least one --allowed-origin is required for a Chrome native messaging manifest.")
|
||||
for origin in allowed_origins:
|
||||
if not origin.startswith("chrome-extension://") or not origin.endswith("/"):
|
||||
failures.append(f"Unsupported Chrome extension origin: {origin}")
|
||||
if not host_path.exists():
|
||||
failures.append(f"Native host path does not exist: {host_path}")
|
||||
elif not os.access(host_path, os.X_OK):
|
||||
failures.append(f"Native host path must be executable: {host_path}")
|
||||
manifest = {
|
||||
"name": host_name,
|
||||
"description": description,
|
||||
"path": str(host_path.resolve()),
|
||||
"type": "stdio",
|
||||
"allowed_origins": allowed_origins,
|
||||
}
|
||||
if not failures:
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return {
|
||||
"ok": not failures,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"manifest": manifest,
|
||||
"manifest_path": display_path(manifest_path),
|
||||
"failures": failures,
|
||||
}
|
||||
|
||||
|
||||
def run_stdio(skill_dir: Path, output_jsonl: Path, dry_run: bool) -> int:
|
||||
while True:
|
||||
raw = read_native_message(sys.stdin.buffer)
|
||||
if raw is None:
|
||||
return 0
|
||||
response = emit_native_message(skill_dir, output_jsonl, raw, dry_run=dry_run)
|
||||
write_native_message(sys.stdout.buffer, response)
|
||||
|
||||
|
||||
def env_default(name: str, fallback: str) -> str:
|
||||
return os.environ.get(name) or fallback
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run a metadata-only telemetry native messaging host.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=env_default(SKILL_DIR_ENV, "."))
|
||||
parser.add_argument("--output-jsonl", default=os.environ.get(EVENTS_ENV))
|
||||
parser.add_argument("--message-json", help="Emit one JSON object directly, useful for host smoke tests.")
|
||||
parser.add_argument("--stdio", action="store_true", help="Use Chrome/Browser Native Messaging length-prefixed stdio.")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--write-manifest")
|
||||
parser.add_argument("--write-launcher")
|
||||
parser.add_argument("--host-name", default=DEFAULT_HOST_NAME)
|
||||
parser.add_argument("--description", default=DEFAULT_DESCRIPTION)
|
||||
parser.add_argument("--allowed-origin", action="append", default=[])
|
||||
parser.add_argument("--host-path")
|
||||
args = parser.parse_args()
|
||||
|
||||
skill_dir = Path(args.skill_dir).resolve()
|
||||
output_jsonl = Path(args.output_jsonl).resolve() if args.output_jsonl else default_spool_path(skill_dir).resolve()
|
||||
host_script = Path(__file__).resolve()
|
||||
|
||||
launcher_path = Path(args.write_launcher).resolve() if args.write_launcher else None
|
||||
if launcher_path:
|
||||
write_launcher(launcher_path, host_script, skill_dir, output_jsonl)
|
||||
|
||||
if args.write_manifest:
|
||||
host_path = Path(args.host_path).resolve() if args.host_path else launcher_path or host_script
|
||||
report = write_manifest(
|
||||
Path(args.write_manifest).resolve(),
|
||||
args.host_name,
|
||||
args.description,
|
||||
host_path,
|
||||
args.allowed_origin,
|
||||
)
|
||||
if launcher_path:
|
||||
report["launcher_path"] = display_path(launcher_path)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
return
|
||||
|
||||
if args.stdio:
|
||||
try:
|
||||
raise SystemExit(run_stdio(skill_dir, output_jsonl, args.dry_run))
|
||||
except Exception as exc:
|
||||
write_native_message(
|
||||
sys.stdout.buffer,
|
||||
{
|
||||
"ok": False,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"mode": "native-messaging-host",
|
||||
"emitted": False,
|
||||
"failures": [str(exc)],
|
||||
},
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
if args.message_json:
|
||||
try:
|
||||
message = json.loads(args.message_json)
|
||||
except json.JSONDecodeError as exc:
|
||||
print(json.dumps({"ok": False, "failures": [f"Invalid --message-json: {exc.msg}"]}, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(2)
|
||||
if not isinstance(message, dict):
|
||||
print(json.dumps({"ok": False, "failures": ["--message-json must decode to a JSON object"]}, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(2)
|
||||
report = emit_native_message(skill_dir, output_jsonl, message, dry_run=args.dry_run)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
return
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"ok": False,
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"failures": ["Use --message-json, --stdio, or --write-manifest."],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+5
-137
@@ -1,6 +1,5 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import ast
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
@@ -9,25 +8,25 @@ import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
from trust_check_scripts import INTERNAL_SCRIPT_INTERFACE, script_inventory
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCAN_DIRS = ["agents", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"]
|
||||
SCAN_DIRS = ["agents", "assets", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"]
|
||||
ROOT_FILES = ["SKILL.md", "README.md", "manifest.json", "requirements-ci.txt", "Makefile"]
|
||||
TEXT_SUFFIXES = {".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"}
|
||||
TEXT_SUFFIXES = {".css", ".js", ".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"}
|
||||
PACKAGE_HASH_SCOPE = "source-contract-without-generated-reports"
|
||||
INTERNAL_SCRIPT_INTERFACE = "internal-module"
|
||||
NETWORK_POLICY_REL_PATH = "security/network_policy.json"
|
||||
PERMISSION_POLICY_REL_PATH = "security/permission_policy.json"
|
||||
HELP_SMOKE_TIMEOUT_SECONDS = 5.0
|
||||
PERMISSION_CAPABILITIES = ("network", "file_write", "subprocess", "interactive")
|
||||
PERMISSION_TARGETS = ("openai", "claude", "generic")
|
||||
PERMISSION_TARGETS = ("openai", "claude", "generic", "vscode")
|
||||
SECRET_PATTERNS = [
|
||||
("private_key", re.compile(r"-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")),
|
||||
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")),
|
||||
@@ -94,137 +93,6 @@ def scan_secrets(skill_dir: Path, files: list[Path]) -> list[dict[str, Any]]:
|
||||
return findings
|
||||
|
||||
|
||||
def script_inventory(skill_dir: Path) -> list[dict[str, Any]]:
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
return []
|
||||
inventory = []
|
||||
for path in sorted(scripts_dir.glob("*.py")):
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
flags = script_flags(text)
|
||||
interface = script_interface(text)
|
||||
urls = extract_url_literals(text)
|
||||
inventory.append(
|
||||
{
|
||||
"path": relpath(skill_dir, path),
|
||||
"interface": interface["name"],
|
||||
"interface_declared": interface["declared"],
|
||||
"interface_reason": interface["reason"],
|
||||
"has_argparse": "argparse" in text,
|
||||
"has_main_guard": 'if __name__ == "__main__"' in text,
|
||||
"uses_input": flags["uses_input"],
|
||||
"uses_network": flags["uses_network"],
|
||||
"uses_file_write": flags["uses_file_write"],
|
||||
"uses_subprocess": flags["uses_subprocess"],
|
||||
"network_urls": urls,
|
||||
"network_hosts": sorted({urlparse(url).hostname or "" for url in urls if urlparse(url).hostname}),
|
||||
}
|
||||
)
|
||||
return inventory
|
||||
|
||||
|
||||
def string_assignment(tree: ast.Module, variable_name: str) -> str:
|
||||
for node in tree.body:
|
||||
value_node = None
|
||||
if isinstance(node, ast.Assign):
|
||||
if any(isinstance(target, ast.Name) and target.id == variable_name for target in node.targets):
|
||||
value_node = node.value
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
target = node.target
|
||||
if isinstance(target, ast.Name) and target.id == variable_name:
|
||||
value_node = node.value
|
||||
if isinstance(value_node, ast.Constant) and isinstance(value_node.value, str):
|
||||
return value_node.value
|
||||
return ""
|
||||
|
||||
|
||||
def script_interface(text: str) -> dict[str, Any]:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
match = re.search(r"SCRIPT_INTERFACE\s*=\s*['\"]([^'\"]+)['\"]", text)
|
||||
reason_match = re.search(r"SCRIPT_INTERFACE_REASON\s*=\s*['\"]([^'\"]+)['\"]", text)
|
||||
name = match.group(1) if match else "cli"
|
||||
reason = reason_match.group(1) if reason_match else ""
|
||||
return {"name": name, "declared": bool(match), "reason": reason}
|
||||
|
||||
name = string_assignment(tree, "SCRIPT_INTERFACE")
|
||||
reason = string_assignment(tree, "SCRIPT_INTERFACE_REASON")
|
||||
if name:
|
||||
return {"name": name, "declared": True, "reason": reason}
|
||||
return {"name": "cli", "declared": False, "reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules."}
|
||||
|
||||
|
||||
def extract_url_literals(text: str) -> list[str]:
|
||||
values: list[str] = []
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
values.append(node.value)
|
||||
except SyntaxError:
|
||||
values = re.findall(r"['\"]([^'\"]+)['\"]", text)
|
||||
|
||||
urls = []
|
||||
seen = set()
|
||||
for value in values:
|
||||
for match in re.finditer(r"https?://[^\s'\"<>]+", value):
|
||||
url = match.group(0).rstrip(").,]")
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
urls.append(url)
|
||||
return urls
|
||||
|
||||
|
||||
def call_name(node: ast.AST) -> str:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id
|
||||
if isinstance(node, ast.Attribute):
|
||||
base = call_name(node.value)
|
||||
return f"{base}.{node.attr}" if base else node.attr
|
||||
return ""
|
||||
|
||||
|
||||
def script_flags(text: str) -> dict[str, bool]:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return {
|
||||
"uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)),
|
||||
"uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)),
|
||||
"uses_file_write": bool(
|
||||
re.search(r"\.(?:write_text|write_bytes|mkdir|unlink)\s*\(", text)
|
||||
or re.search(r"\b(?:open)\s*\([^)]*['\"][wa+x]", text)
|
||||
or "shutil.rmtree" in text
|
||||
),
|
||||
"uses_subprocess": "subprocess." in text,
|
||||
}
|
||||
call_nodes = [node for node in ast.walk(tree) if isinstance(node, ast.Call)]
|
||||
calls = [call_name(node.func) for node in call_nodes]
|
||||
return {
|
||||
"uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls),
|
||||
"uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls),
|
||||
"uses_file_write": any(is_file_write_call(node, call_name(node.func)) for node in call_nodes),
|
||||
"uses_subprocess": any(name.startswith("subprocess.") for name in calls),
|
||||
}
|
||||
|
||||
|
||||
def is_file_write_call(node: ast.Call, name: str) -> bool:
|
||||
if name.endswith((".write_text", ".write_bytes", ".mkdir", ".unlink", ".rmdir")):
|
||||
return True
|
||||
if name in {"shutil.copy", "shutil.copy2", "shutil.copytree", "shutil.move", "shutil.rmtree"}:
|
||||
return True
|
||||
if name == "zipfile.ZipFile":
|
||||
return True
|
||||
if name in {"open", "Path.open"} or name.endswith(".open"):
|
||||
args = list(node.args)
|
||||
keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg}
|
||||
mode_node = args[1] if len(args) > 1 else keywords.get("mode")
|
||||
if isinstance(mode_node, ast.Constant) and isinstance(mode_node.value, str):
|
||||
return any(flag in mode_node.value for flag in ("w", "a", "x", "+"))
|
||||
return False
|
||||
|
||||
|
||||
def dependency_status(skill_dir: Path) -> dict[str, Any]:
|
||||
candidates = ["requirements-ci.txt", "requirements.txt", "pyproject.toml", "package-lock.json", "uv.lock", "poetry.lock"]
|
||||
present = [name for name in candidates if (skill_dir / name).exists()]
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env python3
|
||||
import ast
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Static script inventory helpers imported by trust_check.py."
|
||||
INTERNAL_SCRIPT_INTERFACE = "internal-module"
|
||||
|
||||
|
||||
def relpath(skill_dir: Path, path: Path) -> str:
|
||||
return str(path.relative_to(skill_dir))
|
||||
|
||||
|
||||
def script_inventory(skill_dir: Path) -> list[dict[str, Any]]:
|
||||
scripts_dir = skill_dir / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
return []
|
||||
inventory = []
|
||||
for path in sorted(scripts_dir.glob("*.py")):
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
flags = script_flags(text)
|
||||
interface = script_interface(text)
|
||||
urls = extract_url_literals(text)
|
||||
inventory.append(
|
||||
{
|
||||
"path": relpath(skill_dir, path),
|
||||
"interface": interface["name"],
|
||||
"interface_declared": interface["declared"],
|
||||
"interface_reason": interface["reason"],
|
||||
"has_argparse": "argparse" in text,
|
||||
"has_main_guard": 'if __name__ == "__main__"' in text,
|
||||
"uses_input": flags["uses_input"],
|
||||
"uses_network": flags["uses_network"],
|
||||
"uses_file_write": flags["uses_file_write"],
|
||||
"uses_subprocess": flags["uses_subprocess"],
|
||||
"network_urls": urls,
|
||||
"network_hosts": sorted({urlparse(url).hostname or "" for url in urls if urlparse(url).hostname}),
|
||||
}
|
||||
)
|
||||
return inventory
|
||||
|
||||
|
||||
def string_assignment(tree: ast.Module, variable_name: str) -> str:
|
||||
for node in tree.body:
|
||||
value_node = None
|
||||
if isinstance(node, ast.Assign):
|
||||
if any(isinstance(target, ast.Name) and target.id == variable_name for target in node.targets):
|
||||
value_node = node.value
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
target = node.target
|
||||
if isinstance(target, ast.Name) and target.id == variable_name:
|
||||
value_node = node.value
|
||||
if isinstance(value_node, ast.Constant) and isinstance(value_node.value, str):
|
||||
return value_node.value
|
||||
return ""
|
||||
|
||||
|
||||
def script_interface(text: str) -> dict[str, Any]:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
match = re.search(r"SCRIPT_INTERFACE\s*=\s*['\"]([^'\"]+)['\"]", text)
|
||||
reason_match = re.search(r"SCRIPT_INTERFACE_REASON\s*=\s*['\"]([^'\"]+)['\"]", text)
|
||||
name = match.group(1) if match else "cli"
|
||||
reason = reason_match.group(1) if reason_match else ""
|
||||
return {"name": name, "declared": bool(match), "reason": reason}
|
||||
|
||||
name = string_assignment(tree, "SCRIPT_INTERFACE")
|
||||
reason = string_assignment(tree, "SCRIPT_INTERFACE_REASON")
|
||||
if name:
|
||||
return {"name": name, "declared": True, "reason": reason}
|
||||
return {"name": "cli", "declared": False, "reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules."}
|
||||
|
||||
|
||||
def extract_url_literals(text: str) -> list[str]:
|
||||
values: list[str] = []
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
values.append(node.value)
|
||||
except SyntaxError:
|
||||
values = re.findall(r"['\"]([^'\"]+)['\"]", text)
|
||||
|
||||
urls = []
|
||||
seen = set()
|
||||
for value in values:
|
||||
for match in re.finditer(r"https?://[^\s'\"<>]+", value):
|
||||
url = match.group(0).rstrip(").,]")
|
||||
if url not in seen:
|
||||
seen.add(url)
|
||||
urls.append(url)
|
||||
return urls
|
||||
|
||||
|
||||
def call_name(node: ast.AST) -> str:
|
||||
if isinstance(node, ast.Name):
|
||||
return node.id
|
||||
if isinstance(node, ast.Attribute):
|
||||
base = call_name(node.value)
|
||||
return f"{base}.{node.attr}" if base else node.attr
|
||||
return ""
|
||||
|
||||
|
||||
def script_flags(text: str) -> dict[str, bool]:
|
||||
try:
|
||||
tree = ast.parse(text)
|
||||
except SyntaxError:
|
||||
return {
|
||||
"uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)),
|
||||
"uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)),
|
||||
"uses_file_write": bool(
|
||||
re.search(r"\.(?:write_text|write_bytes|mkdir|unlink)\s*\(", text)
|
||||
or re.search(r"\b(?:open)\s*\([^)]*['\"][wa+x]", text)
|
||||
or "shutil.rmtree" in text
|
||||
),
|
||||
"uses_subprocess": "subprocess." in text,
|
||||
}
|
||||
call_nodes = [node for node in ast.walk(tree) if isinstance(node, ast.Call)]
|
||||
calls = [call_name(node.func) for node in call_nodes]
|
||||
return {
|
||||
"uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls),
|
||||
"uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls),
|
||||
"uses_file_write": any(is_file_write_call(node, call_name(node.func)) for node in call_nodes),
|
||||
"uses_subprocess": any(name.startswith("subprocess.") for name in calls),
|
||||
}
|
||||
|
||||
|
||||
def is_file_write_call(node: ast.Call, name: str) -> bool:
|
||||
if name.endswith((".write_text", ".write_bytes", ".mkdir", ".unlink", ".rmdir")):
|
||||
return True
|
||||
if name in {"shutil.copy", "shutil.copy2", "shutil.copytree", "shutil.move", "shutil.rmtree"}:
|
||||
return True
|
||||
if name == "zipfile.ZipFile":
|
||||
return True
|
||||
if name in {"open", "Path.open"} or name.endswith(".open"):
|
||||
args = list(node.args)
|
||||
keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg}
|
||||
mode_node = args[1] if len(args) > 1 else keywords.get("mode")
|
||||
if isinstance(mode_node, ast.Constant) and isinstance(mode_node.value, str):
|
||||
return any(flag in mode_node.value for flag in ("w", "a", "x", "+"))
|
||||
return False
|
||||
+18
-12
@@ -54,7 +54,7 @@ def generated_zip_entries(names: list[str]) -> list[str]:
|
||||
generated = []
|
||||
for name in names:
|
||||
parts = PurePosixPath(name).parts
|
||||
if "dist" in parts or (len(parts) > 2 and parts[1] == "tests" and any(part.startswith("tmp") for part in parts[2:])):
|
||||
if ".previews" in parts or "dist" in parts or (len(parts) > 2 and parts[1] == "tests" and any(part.startswith("tmp") for part in parts[2:])):
|
||||
generated.append(name)
|
||||
return generated
|
||||
|
||||
@@ -75,6 +75,10 @@ def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str,
|
||||
failures.append(detail)
|
||||
|
||||
|
||||
def package_name(manifest: dict[str, Any], skill_dir: Path) -> str:
|
||||
return str(manifest.get("name") or skill_dir.name)
|
||||
|
||||
|
||||
def verify_package(
|
||||
skill_dir: Path,
|
||||
package_dir: Path,
|
||||
@@ -91,6 +95,7 @@ def verify_package(
|
||||
|
||||
manifest_path = package_dir / "manifest.json"
|
||||
manifest = load_json(manifest_path)
|
||||
package_root = package_name(manifest, skill_dir)
|
||||
add_check(checks, failures, "package-manifest", bool(manifest), f"Package manifest exists: {display_path(manifest_path)}")
|
||||
|
||||
targets = required_targets(expectations, package_dir)
|
||||
@@ -108,15 +113,16 @@ def verify_package(
|
||||
field in adapter,
|
||||
f"{target} adapter includes field: {field}",
|
||||
)
|
||||
for target, key in (
|
||||
("openai", "openai_required_files"),
|
||||
("claude", "claude_required_files"),
|
||||
("generic", "generic_required_files"),
|
||||
):
|
||||
for rel in expectations.get(key, []):
|
||||
required_files_by_target = {
|
||||
key[: -len("_required_files")]: value
|
||||
for key, value in expectations.items()
|
||||
if key.endswith("_required_files")
|
||||
}
|
||||
for target, required_files in required_files_by_target.items():
|
||||
for rel in required_files:
|
||||
add_check(checks, failures, f"{target}-file-{rel}", (package_dir / rel).exists(), f"Package contains {rel}")
|
||||
|
||||
archive_path = package_dir / f"{skill_dir.name}.zip"
|
||||
archive_path = package_dir / f"{package_root}.zip"
|
||||
archive_sha = ""
|
||||
archive_entries: list[str] = []
|
||||
if archive_path.exists():
|
||||
@@ -128,15 +134,15 @@ def verify_package(
|
||||
else:
|
||||
unsafe_entries = unsafe_zip_entries(archive_entries)
|
||||
required_entries = [
|
||||
f"{skill_dir.name}/SKILL.md",
|
||||
f"{skill_dir.name}/manifest.json",
|
||||
f"{skill_dir.name}/agents/interface.yaml",
|
||||
f"{package_root}/SKILL.md",
|
||||
f"{package_root}/manifest.json",
|
||||
f"{package_root}/agents/interface.yaml",
|
||||
]
|
||||
add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries")
|
||||
for entry in required_entries:
|
||||
add_check(checks, failures, f"archive-entry-{entry}", entry in archive_entries, f"Archive contains {entry}")
|
||||
generated_entries = generated_zip_entries(archive_entries)
|
||||
add_check(checks, failures, "archive-excludes-generated", not generated_entries, "Archive excludes generated dist/ and tests/tmp* contents")
|
||||
add_check(checks, failures, "archive-excludes-generated", not generated_entries, "Archive excludes generated dist/, .previews/, and tests/tmp* contents")
|
||||
elif require_zip:
|
||||
add_check(checks, failures, "archive-present", False, f"Missing required package archive: {display_path(archive_path)}")
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,553 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from output_review_privacy import BLOCKED_DECISION_FIELDS
|
||||
from world_class_human_evidence import validate_human_adjudication_report
|
||||
from world_class_native_permission_evidence import validate_native_permission_report
|
||||
from world_class_native_telemetry_evidence import validate_native_telemetry_report
|
||||
from world_class_provider_evidence import validate_provider_execution_report
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks."
|
||||
|
||||
REQUIRED_TOP_LEVEL = [
|
||||
"schema_version",
|
||||
"evidence_key",
|
||||
"template_only",
|
||||
"category",
|
||||
"source_type",
|
||||
"submitted_by",
|
||||
"submitted_at",
|
||||
"summary",
|
||||
"artifact_refs",
|
||||
"provenance",
|
||||
"privacy",
|
||||
"anti_overclaim",
|
||||
"attestation",
|
||||
]
|
||||
REQUIRED_PRIVACY_FALSE = [
|
||||
"raw_user_content_included",
|
||||
"raw_provider_prompt_included",
|
||||
"credentials_included",
|
||||
"secrets_included",
|
||||
]
|
||||
REQUIRED_ANTI_OVERCLAIM_FALSE = [
|
||||
"planned_work_counts_as_evidence",
|
||||
"metadata_fallback_counts_as_native_enforcement",
|
||||
"pending_review_counts_as_human_decision",
|
||||
"local_command_runner_counts_as_provider_model",
|
||||
]
|
||||
REQUIRED_ATTESTATION_TRUE = [
|
||||
"real_external_or_human_evidence",
|
||||
"reviewer_or_operator_identity_present",
|
||||
"artifact_refs_reviewed",
|
||||
"privacy_contract_satisfied",
|
||||
"ledger_reviewer_approved",
|
||||
]
|
||||
EXPECTED_SOURCE_TYPES = {
|
||||
"provider-holdout": "provider-output-eval",
|
||||
"human-adjudication": "blind-ab-review",
|
||||
"native-permission-enforcement": "runtime-permission-guard",
|
||||
"native-client-telemetry": "native-client-telemetry",
|
||||
}
|
||||
SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
|
||||
SUBMITTED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$")
|
||||
DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"}
|
||||
REQUIRED_REAL_ARTIFACT_PATHS = {
|
||||
"provider-holdout": {"reports/output_execution_runs.json"},
|
||||
"human-adjudication": {
|
||||
"reports/output_review_adjudication.json",
|
||||
"reports/output_review_decisions.json",
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"reports/runtime_permission_probes.json",
|
||||
"reports/install_simulation.json",
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"reports/adoption_drift_report.json",
|
||||
"reports/telemetry_hook_recipes.json",
|
||||
},
|
||||
}
|
||||
EXPECTED_REAL_ARTIFACT_KINDS = {
|
||||
"provider-holdout": {
|
||||
"reports/output_execution_runs.json": "aggregate-report",
|
||||
},
|
||||
"human-adjudication": {
|
||||
"reports/output_review_adjudication.json": "adjudication-report",
|
||||
"reports/output_review_decisions.json": "review-decisions",
|
||||
},
|
||||
"native-permission-enforcement": {
|
||||
"reports/runtime_permission_probes.json": "runtime-probe-report",
|
||||
"reports/install_simulation.json": "installer-enforcement-report",
|
||||
},
|
||||
"native-client-telemetry": {
|
||||
"reports/adoption_drift_report.json": "adoption-drift-report",
|
||||
"reports/telemetry_hook_recipes.json": "hook-recipes",
|
||||
},
|
||||
}
|
||||
PLACEHOLDER_FRAGMENTS = (
|
||||
"YYYY-MM-DD",
|
||||
"name or team handle",
|
||||
"operator with provider credentials",
|
||||
"human reviewer",
|
||||
"target client or installer integrator",
|
||||
"Browser/Chrome/IDE/provider client integrator",
|
||||
"Browser/Chrome/IDE/provider client",
|
||||
"openai|claude|generic|vscode|other",
|
||||
"client or installer component",
|
||||
"/local/path/not/committed",
|
||||
)
|
||||
FORBIDDEN_REAL_SUBMISSION_FIELDS = BLOCKED_DECISION_FIELDS
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]:
|
||||
if not path.exists():
|
||||
return {}, "missing"
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}, "invalid-json"
|
||||
if not isinstance(payload, dict):
|
||||
return {}, "invalid-json"
|
||||
return payload, "present"
|
||||
|
||||
|
||||
def rel_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def add_error(errors: list[str], condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def has_placeholder_text(value: Any) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
lowered = text.lower()
|
||||
return any(fragment.lower() in lowered for fragment in PLACEHOLDER_FRAGMENTS)
|
||||
|
||||
|
||||
def require_real_text(errors: list[str], value: Any, field: str) -> None:
|
||||
text = str(value or "").strip()
|
||||
add_error(errors, bool(text), f"{field} is required")
|
||||
add_error(errors, not has_placeholder_text(text), f"{field} must not use template placeholder text")
|
||||
|
||||
|
||||
def parse_audit_timestamp(value: Any) -> datetime | None:
|
||||
text = str(value or "").strip()
|
||||
if not SUBMITTED_AT_RE.match(text):
|
||||
return None
|
||||
if "T" in text:
|
||||
return datetime.fromisoformat(text.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
return datetime.fromisoformat(text).replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def forbidden_submission_field_paths(value: Any, prefix: str = "$") -> list[str]:
|
||||
found: list[str] = []
|
||||
if isinstance(value, dict):
|
||||
for key, child in value.items():
|
||||
key_text = str(key)
|
||||
child_path = f"{prefix}.{key_text}"
|
||||
if key_text.strip().lower() in FORBIDDEN_REAL_SUBMISSION_FIELDS:
|
||||
found.append(child_path)
|
||||
found.extend(forbidden_submission_field_paths(child, child_path))
|
||||
elif isinstance(value, list):
|
||||
for index, child in enumerate(value):
|
||||
found.extend(forbidden_submission_field_paths(child, f"{prefix}[{index}]"))
|
||||
return found
|
||||
|
||||
|
||||
def validate_real_submission_file_identity(
|
||||
payload: dict[str, Any],
|
||||
errors: list[str],
|
||||
*,
|
||||
path: Path,
|
||||
template_expected: bool,
|
||||
) -> None:
|
||||
if template_expected:
|
||||
return
|
||||
evidence_key = str(payload.get("evidence_key", "")).strip()
|
||||
expected_name = f"{evidence_key}.json"
|
||||
add_error(
|
||||
errors,
|
||||
path.name == expected_name,
|
||||
f"real submission filename must be {expected_name}",
|
||||
)
|
||||
|
||||
|
||||
def validate_real_submission_privacy_fields(
|
||||
payload: dict[str, Any],
|
||||
errors: list[str],
|
||||
*,
|
||||
template_expected: bool,
|
||||
) -> None:
|
||||
if template_expected:
|
||||
return
|
||||
blocked_paths = forbidden_submission_field_paths(payload)
|
||||
add_error(
|
||||
errors,
|
||||
not blocked_paths,
|
||||
"real submission must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: "
|
||||
+ ", ".join(blocked_paths[:8]),
|
||||
)
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]:
|
||||
text = raw_path.strip()
|
||||
if any(token in text for token in ("<", ">", "*", "?")):
|
||||
return None, "artifact_refs path must be concrete, not a placeholder or glob"
|
||||
candidate = Path(text)
|
||||
if candidate.is_absolute():
|
||||
return None, "artifact_refs path must be relative to the skill directory"
|
||||
if ".." in candidate.parts:
|
||||
return None, "artifact_refs path must not escape the skill directory"
|
||||
resolved = (root / candidate).resolve()
|
||||
try:
|
||||
resolved.relative_to(root.resolve())
|
||||
except ValueError:
|
||||
return None, "artifact_refs path must stay inside the skill directory"
|
||||
return resolved, None
|
||||
|
||||
|
||||
def validate_artifact_refs(
|
||||
payload: dict[str, Any],
|
||||
errors: list[str],
|
||||
*,
|
||||
root: Path,
|
||||
template_expected: bool,
|
||||
) -> dict[str, int]:
|
||||
refs = payload.get("artifact_refs")
|
||||
add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference")
|
||||
evidence_key = str(payload.get("evidence_key", ""))
|
||||
required_paths = REQUIRED_REAL_ARTIFACT_PATHS.get(evidence_key, set())
|
||||
expected_kinds = EXPECTED_REAL_ARTIFACT_KINDS.get(evidence_key, {})
|
||||
observed_paths: set[str] = set()
|
||||
seen_artifact_paths: set[str] = set()
|
||||
stats = {
|
||||
"artifact_ref_count": len(refs) if isinstance(refs, list) else 0,
|
||||
"artifact_existing_count": 0,
|
||||
"artifact_sha256_verified_count": 0,
|
||||
"required_artifact_count": len(required_paths) if not template_expected else 0,
|
||||
"required_artifact_verified_count": 0,
|
||||
}
|
||||
if not isinstance(refs, list):
|
||||
return stats
|
||||
for index, ref in enumerate(refs):
|
||||
if not isinstance(ref, dict):
|
||||
errors.append(f"artifact_refs[{index}] must be an object")
|
||||
continue
|
||||
path_text = str(ref.get("path", "")).strip()
|
||||
add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required")
|
||||
add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required")
|
||||
add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content")
|
||||
if template_expected or not path_text:
|
||||
continue
|
||||
candidate = Path(path_text)
|
||||
if not candidate.is_absolute() and ".." not in candidate.parts and not any(
|
||||
token in path_text for token in ("<", ">", "*", "?")
|
||||
):
|
||||
observed_paths.add(candidate.as_posix())
|
||||
resolved, path_error = resolve_artifact_path(path_text, root)
|
||||
if path_error:
|
||||
errors.append(f"artifact_refs[{index}].path {path_error}")
|
||||
continue
|
||||
rel = rel_path(resolved, root)
|
||||
add_error(
|
||||
errors,
|
||||
rel not in seen_artifact_paths,
|
||||
f"artifact_refs[{index}].path must not duplicate another artifact reference",
|
||||
)
|
||||
seen_artifact_paths.add(rel)
|
||||
if rel in DISALLOWED_REAL_ARTIFACTS:
|
||||
errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs")
|
||||
expected_kind = expected_kinds.get(rel)
|
||||
if expected_kind:
|
||||
observed_kind = str(ref.get("kind", "")).strip()
|
||||
add_error(
|
||||
errors,
|
||||
observed_kind == expected_kind,
|
||||
f"artifact_refs[{index}].kind must be {expected_kind} for {rel}",
|
||||
)
|
||||
if not resolved.exists() or not resolved.is_file():
|
||||
errors.append(f"artifact_refs[{index}].path does not exist as a local file")
|
||||
continue
|
||||
stats["artifact_existing_count"] += 1
|
||||
declared = str(ref.get("sha256", "")).strip()
|
||||
if not declared:
|
||||
errors.append(f"artifact_refs[{index}].sha256 is required for a real submission")
|
||||
continue
|
||||
if not SHA256_RE.match(declared):
|
||||
errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest")
|
||||
continue
|
||||
actual = sha256_file(resolved)
|
||||
if actual.lower() != declared.lower():
|
||||
errors.append(f"artifact_refs[{index}].sha256 does not match local artifact")
|
||||
continue
|
||||
stats["artifact_sha256_verified_count"] += 1
|
||||
if rel in required_paths:
|
||||
stats["required_artifact_verified_count"] += 1
|
||||
if not template_expected and required_paths:
|
||||
missing_required = sorted(required_paths - observed_paths)
|
||||
for path in missing_required:
|
||||
errors.append(f"artifact_refs must include required evidence artifact {path}")
|
||||
if not missing_required and stats["required_artifact_verified_count"] < len(required_paths):
|
||||
errors.append("all required evidence artifacts must have verified sha256 digests")
|
||||
return stats
|
||||
|
||||
|
||||
def artifact_ref_path_map(payload: dict[str, Any], root: Path) -> dict[str, Path]:
|
||||
refs = payload.get("artifact_refs", [])
|
||||
if not isinstance(refs, list):
|
||||
return {}
|
||||
paths: dict[str, Path] = {}
|
||||
for ref in refs:
|
||||
if not isinstance(ref, dict):
|
||||
continue
|
||||
path_text = str(ref.get("path", "")).strip()
|
||||
if not path_text:
|
||||
continue
|
||||
resolved, path_error = resolve_artifact_path(path_text, root)
|
||||
if path_error or resolved is None:
|
||||
continue
|
||||
paths[rel_path(resolved, root)] = resolved
|
||||
return paths
|
||||
|
||||
|
||||
def validate_provider_holdout_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
|
||||
paths = artifact_ref_path_map(payload, root)
|
||||
execution = load_json(paths.get("reports/output_execution_runs.json", root / "__missing__"))
|
||||
if not execution:
|
||||
return
|
||||
provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
|
||||
validate_provider_execution_report(execution, provenance, errors)
|
||||
|
||||
|
||||
def validate_human_adjudication_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
|
||||
paths = artifact_ref_path_map(payload, root)
|
||||
adjudication = load_json(paths.get("reports/output_review_adjudication.json", root / "__missing__"))
|
||||
decisions = load_json(paths.get("reports/output_review_decisions.json", root / "__missing__"))
|
||||
if not adjudication or not decisions:
|
||||
return
|
||||
provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
|
||||
validate_human_adjudication_report(adjudication, decisions, provenance, errors)
|
||||
|
||||
|
||||
def validate_native_permission_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
|
||||
paths = artifact_ref_path_map(payload, root)
|
||||
probes = load_json(paths.get("reports/runtime_permission_probes.json", root / "__missing__"))
|
||||
install = load_json(paths.get("reports/install_simulation.json", root / "__missing__"))
|
||||
if probes:
|
||||
validate_native_permission_report(probes, install, errors)
|
||||
|
||||
|
||||
def validate_native_client_telemetry_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None:
|
||||
paths = artifact_ref_path_map(payload, root)
|
||||
adoption = load_json(paths.get("reports/adoption_drift_report.json", root / "__missing__"))
|
||||
recipes = load_json(paths.get("reports/telemetry_hook_recipes.json", root / "__missing__"))
|
||||
validate_native_telemetry_report(adoption, recipes, errors)
|
||||
|
||||
|
||||
def validate_real_artifact_payloads(payload: dict[str, Any], errors: list[str], root: Path, template_expected: bool) -> None:
|
||||
if template_expected:
|
||||
return
|
||||
evidence_key = str(payload.get("evidence_key", ""))
|
||||
if evidence_key == "provider-holdout":
|
||||
validate_provider_holdout_artifacts(payload, errors, root)
|
||||
elif evidence_key == "human-adjudication":
|
||||
validate_human_adjudication_artifacts(payload, errors, root)
|
||||
elif evidence_key == "native-permission-enforcement":
|
||||
validate_native_permission_artifacts(payload, errors, root)
|
||||
elif evidence_key == "native-client-telemetry":
|
||||
validate_native_client_telemetry_artifacts(payload, errors, root)
|
||||
|
||||
|
||||
def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None:
|
||||
key = str(payload.get("evidence_key", ""))
|
||||
template_expected = payload.get("template_only") is True
|
||||
provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {}
|
||||
if key == "provider-holdout":
|
||||
add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required")
|
||||
add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required")
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("credential_material_committed") is False,
|
||||
"provider-holdout must attest credential_material_committed is false",
|
||||
)
|
||||
elif key == "human-adjudication":
|
||||
add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required")
|
||||
add_error(errors, bool(str(provenance.get("reviewed_at", "")).strip()), "human-adjudication provenance.reviewed_at is required")
|
||||
if not template_expected:
|
||||
require_real_text(errors, provenance.get("reviewer", ""), "human-adjudication provenance.reviewer")
|
||||
add_error(
|
||||
errors,
|
||||
bool(SUBMITTED_AT_RE.match(str(provenance.get("reviewed_at", "")).strip())),
|
||||
"human-adjudication provenance.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("answer_key_opened_after_decisions") is True,
|
||||
"human-adjudication must attest answer_key_opened_after_decisions is true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("blind_review_completed_before_answer_key") is True,
|
||||
"human-adjudication must attest blind_review_completed_before_answer_key is true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
bool(str(provenance.get("blind_pack_sha256", "")).strip()),
|
||||
"human-adjudication provenance.blind_pack_sha256 is required",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("reviewer_reason_required") is True,
|
||||
"human-adjudication must attest reviewer_reason_required is true",
|
||||
)
|
||||
elif key == "native-permission-enforcement":
|
||||
add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required")
|
||||
if not template_expected:
|
||||
require_real_text(errors, provenance.get("target", ""), "native-permission-enforcement provenance.target")
|
||||
add_error(
|
||||
errors,
|
||||
bool(str(provenance.get("guard_location", "")).strip()),
|
||||
"native-permission-enforcement provenance.guard_location is required",
|
||||
)
|
||||
if not template_expected:
|
||||
require_real_text(
|
||||
errors,
|
||||
provenance.get("guard_location", ""),
|
||||
"native-permission-enforcement provenance.guard_location",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
str(provenance.get("guard_scope", "")).strip()
|
||||
in {"target-client-native", "external-installer-runtime-guard"},
|
||||
"native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("guard_blocks_undeclared_capability") is True,
|
||||
"native-permission-enforcement must attest guard_blocks_undeclared_capability is true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("metadata_fallback_retained_for_other_targets") is True,
|
||||
"native-permission-enforcement must retain metadata fallback for non-native targets",
|
||||
)
|
||||
elif key == "native-client-telemetry":
|
||||
add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required")
|
||||
if not template_expected:
|
||||
require_real_text(errors, provenance.get("client", ""), "native-client-telemetry provenance.client")
|
||||
if str(provenance.get("native_host_manifest", "")).strip():
|
||||
require_real_text(
|
||||
errors,
|
||||
provenance.get("native_host_manifest", ""),
|
||||
"native-client-telemetry provenance.native_host_manifest",
|
||||
)
|
||||
add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external")
|
||||
add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only")
|
||||
|
||||
|
||||
def validate_payload(
|
||||
payload: dict[str, Any],
|
||||
entry: dict[str, Any],
|
||||
*,
|
||||
path: Path,
|
||||
root: Path,
|
||||
template_expected: bool,
|
||||
) -> dict[str, Any]:
|
||||
errors: list[str] = []
|
||||
for key in REQUIRED_TOP_LEVEL:
|
||||
add_error(errors, key in payload, f"missing {key}")
|
||||
evidence_key = str(entry.get("key", ""))
|
||||
add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0")
|
||||
add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}")
|
||||
add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}")
|
||||
add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}")
|
||||
add_error(
|
||||
errors,
|
||||
payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key),
|
||||
f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}",
|
||||
)
|
||||
add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required")
|
||||
add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required")
|
||||
add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required")
|
||||
if not template_expected:
|
||||
validate_real_submission_file_identity(payload, errors, path=path, template_expected=template_expected)
|
||||
validate_real_submission_privacy_fields(payload, errors, template_expected=template_expected)
|
||||
require_real_text(errors, payload.get("submitted_by", ""), "submitted_by")
|
||||
add_error(
|
||||
errors,
|
||||
bool(SUBMITTED_AT_RE.match(str(payload.get("submitted_at", "")).strip())),
|
||||
"submitted_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
|
||||
)
|
||||
artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected)
|
||||
privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {}
|
||||
for key in REQUIRED_PRIVACY_FALSE:
|
||||
add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false")
|
||||
anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {}
|
||||
for key in REQUIRED_ANTI_OVERCLAIM_FALSE:
|
||||
add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false")
|
||||
validate_evidence_specific(payload, errors)
|
||||
attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {}
|
||||
if not template_expected:
|
||||
for key in REQUIRED_ATTESTATION_TRUE:
|
||||
add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission")
|
||||
ledger_reviewer = str(attestation.get("ledger_reviewer", "")).strip()
|
||||
submitted_by = str(payload.get("submitted_by", "")).strip()
|
||||
require_real_text(errors, ledger_reviewer, "attestation.ledger_reviewer")
|
||||
add_error(
|
||||
errors,
|
||||
bool(SUBMITTED_AT_RE.match(str(attestation.get("ledger_reviewed_at", "")).strip())),
|
||||
"attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
bool(ledger_reviewer and submitted_by and ledger_reviewer.casefold() != submitted_by.casefold()),
|
||||
"attestation.ledger_reviewer must be different from submitted_by",
|
||||
)
|
||||
submitted_at = parse_audit_timestamp(payload.get("submitted_at"))
|
||||
ledger_reviewed_at = parse_audit_timestamp(attestation.get("ledger_reviewed_at"))
|
||||
add_error(
|
||||
errors,
|
||||
bool(submitted_at and ledger_reviewed_at and ledger_reviewed_at >= submitted_at),
|
||||
"attestation.ledger_reviewed_at must be at or after submitted_at",
|
||||
)
|
||||
validate_real_artifact_payloads(payload, errors, root, template_expected)
|
||||
return {
|
||||
"path": rel_path(path, root),
|
||||
"evidence_key": evidence_key,
|
||||
"status": "pass" if not errors else "fail",
|
||||
"template_only": template_expected,
|
||||
"artifact_integrity": artifact_integrity,
|
||||
"errors": errors,
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from output_review_privacy import forbidden_decision_field_paths, forbidden_raw_content_field_paths
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate blind A/B human adjudication evidence from decision rows."
|
||||
|
||||
REVIEWED_AT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$")
|
||||
|
||||
|
||||
def add_error(errors: list[str], condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def real_int(value: Any) -> int | None:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
||||
|
||||
|
||||
def require_real_text(errors: list[str], value: Any, field: str) -> None:
|
||||
add_error(errors, bool(str(value or "").strip()), f"{field} is required")
|
||||
|
||||
|
||||
def object_list(value: Any) -> list[dict[str, Any]]:
|
||||
return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else []
|
||||
|
||||
|
||||
def case_ids(items: list[dict[str, Any]]) -> list[str]:
|
||||
return [str(item.get("case_id", "")).strip() for item in items]
|
||||
|
||||
|
||||
def duplicate_case_ids(ids: list[str]) -> list[str]:
|
||||
seen: set[str] = set()
|
||||
duplicates: list[str] = []
|
||||
for case_id in ids:
|
||||
if case_id in seen and case_id not in duplicates:
|
||||
duplicates.append(case_id)
|
||||
seen.add(case_id)
|
||||
return duplicates
|
||||
|
||||
|
||||
def confidence_valid(value: Any) -> bool:
|
||||
if value is None or value == "":
|
||||
return True
|
||||
if isinstance(value, bool):
|
||||
return False
|
||||
try:
|
||||
parsed = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return 0 <= parsed <= 1
|
||||
|
||||
|
||||
def validate_adjudication_raw_content(adjudication: dict[str, Any], errors: list[str]) -> None:
|
||||
blocked = forbidden_raw_content_field_paths(adjudication, "output_review_adjudication")
|
||||
add_error(
|
||||
errors,
|
||||
not blocked,
|
||||
"human-adjudication adjudication report must not include raw content, credential, secret, token, prompt, output, transcript, or message fields: "
|
||||
+ ", ".join(blocked[:8]),
|
||||
)
|
||||
|
||||
|
||||
def validate_decision_rows(
|
||||
decisions: list[dict[str, Any]],
|
||||
expected_case_ids: set[str],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
decision_ids = case_ids(decisions)
|
||||
add_error(errors, not any(not case_id for case_id in decision_ids), "human-adjudication decisions must include case_id for every row")
|
||||
duplicates = duplicate_case_ids([case_id for case_id in decision_ids if case_id])
|
||||
add_error(errors, not duplicates, "human-adjudication decisions must not contain duplicate case_id values")
|
||||
add_error(
|
||||
errors,
|
||||
set(decision_ids) == expected_case_ids,
|
||||
"human-adjudication decisions case_id set must match adjudication pairs",
|
||||
)
|
||||
for index, item in enumerate(decisions, start=1):
|
||||
blocked = forbidden_decision_field_paths(item, f"decisions[{index}]")
|
||||
add_error(
|
||||
errors,
|
||||
not blocked,
|
||||
"human-adjudication decisions must not include raw content or answer-key fields: "
|
||||
+ ", ".join(blocked[:8]),
|
||||
)
|
||||
winner = str(item.get("winner_variant", "")).strip().upper()
|
||||
add_error(errors, winner in {"A", "B"}, "human-adjudication decisions must include A/B winner_variant for every case")
|
||||
add_error(errors, confidence_valid(item.get("confidence")), "human-adjudication decisions confidence must be between 0 and 1")
|
||||
add_error(errors, bool(str(item.get("reason", "")).strip()), "human-adjudication decisions must include reviewer reason for every case")
|
||||
|
||||
|
||||
def validate_review_integrity(
|
||||
adjudication: dict[str, Any],
|
||||
decisions: dict[str, Any],
|
||||
expected_case_ids: set[str],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
adjudication_integrity = adjudication.get("review_integrity", {})
|
||||
decisions_integrity = decisions.get("review_integrity", {})
|
||||
add_error(
|
||||
errors,
|
||||
isinstance(adjudication_integrity, dict) and bool(adjudication_integrity.get("blind_pack_sha256")),
|
||||
"human-adjudication adjudication review_integrity.blind_pack_sha256 is required",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
isinstance(decisions_integrity, dict) and bool(decisions_integrity.get("blind_pack_sha256")),
|
||||
"human-adjudication decisions review_integrity.blind_pack_sha256 is required",
|
||||
)
|
||||
if isinstance(adjudication_integrity, dict) and isinstance(decisions_integrity, dict):
|
||||
add_error(
|
||||
errors,
|
||||
bool(adjudication_integrity.get("blind_pack_sha256"))
|
||||
and adjudication_integrity.get("blind_pack_sha256") == decisions_integrity.get("blind_pack_sha256"),
|
||||
"human-adjudication review_integrity.blind_pack_sha256 must match between decisions and adjudication",
|
||||
)
|
||||
decision_case_ids = {str(item) for item in decisions_integrity.get("case_ids", []) if str(item).strip()}
|
||||
adjudication_case_ids = {str(item) for item in adjudication_integrity.get("case_ids", []) if str(item).strip()}
|
||||
add_error(
|
||||
errors,
|
||||
decision_case_ids == expected_case_ids and adjudication_case_ids == expected_case_ids,
|
||||
"human-adjudication review_integrity.case_ids must match adjudicated case ids",
|
||||
)
|
||||
|
||||
|
||||
def validate_reviewer_attestation(
|
||||
adjudication: dict[str, Any],
|
||||
decisions: dict[str, Any],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {}
|
||||
attestation = decisions.get("reviewer_attestation", {})
|
||||
add_error(errors, isinstance(attestation, dict), "human-adjudication decisions.reviewer_attestation must be an object")
|
||||
if not isinstance(attestation, dict):
|
||||
attestation = {}
|
||||
for key in [
|
||||
"blind_review_completed_before_answer_key",
|
||||
"answer_key_not_opened_before_decisions",
|
||||
"raw_content_excluded",
|
||||
"reviewer_reason_required",
|
||||
]:
|
||||
add_error(errors, attestation.get(key) is True, f"human-adjudication decisions.reviewer_attestation.{key} must be true")
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("blind_review_attested") is True,
|
||||
"human-adjudication adjudication summary.blind_review_attested must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("raw_content_excluded_attested") is True,
|
||||
"human-adjudication adjudication summary.raw_content_excluded_attested must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("reviewer_reason_required_attested") is True,
|
||||
"human-adjudication adjudication summary.reviewer_reason_required_attested must be true",
|
||||
)
|
||||
|
||||
|
||||
def validate_adjudicated_pairs(
|
||||
pairs: list[dict[str, Any]],
|
||||
expected_count: int,
|
||||
errors: list[str],
|
||||
) -> set[str]:
|
||||
pair_ids = case_ids(pairs)
|
||||
add_error(errors, len(pairs) == expected_count, "human-adjudication adjudication pairs length must equal summary.pair_count")
|
||||
add_error(errors, not any(not case_id for case_id in pair_ids), "human-adjudication adjudication pairs must include case_id")
|
||||
duplicates = duplicate_case_ids([case_id for case_id in pair_ids if case_id])
|
||||
add_error(errors, not duplicates, "human-adjudication adjudication pairs must not contain duplicate case_id values")
|
||||
for item in pairs:
|
||||
case_id = str(item.get("case_id", "")).strip()
|
||||
status = str(item.get("status", "")).strip()
|
||||
add_error(
|
||||
errors,
|
||||
status in {"match", "disagree"},
|
||||
f"human-adjudication adjudication pair {case_id or '<missing>'} must be match or disagree",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
item.get("expected_revealed") is True,
|
||||
f"human-adjudication adjudication pair {case_id or '<missing>'} must reveal expected winner only after valid decision",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
str(item.get("reviewer_winner_variant", "")).strip().upper() in {"A", "B"},
|
||||
f"human-adjudication adjudication pair {case_id or '<missing>'} must include reviewer_winner_variant",
|
||||
)
|
||||
return {case_id for case_id in pair_ids if case_id}
|
||||
|
||||
|
||||
def validate_human_adjudication_report(
|
||||
adjudication: dict[str, Any],
|
||||
decisions: dict[str, Any],
|
||||
provenance: dict[str, Any],
|
||||
errors: list[str],
|
||||
) -> None:
|
||||
summary = adjudication.get("summary", {}) if isinstance(adjudication.get("summary", {}), dict) else {}
|
||||
pair_count = real_int(summary.get("pair_count"))
|
||||
judgment_count = real_int(summary.get("judgment_count"))
|
||||
pending_count = real_int(summary.get("pending_count"))
|
||||
invalid_decision_count = real_int(summary.get("invalid_decision_count"))
|
||||
answer_revealed_count = real_int(summary.get("answer_revealed_count"))
|
||||
pending_answer_hidden_count = real_int(summary.get("pending_answer_hidden_count"))
|
||||
checklist_ready_count = real_int(summary.get("reviewer_checklist_ready_count"))
|
||||
checklist_count = real_int(summary.get("reviewer_checklist_count"))
|
||||
|
||||
validate_adjudication_raw_content(adjudication, errors)
|
||||
add_error(errors, adjudication.get("ok") is True, "human-adjudication adjudication report ok must be true")
|
||||
add_error(errors, bool(pair_count and pair_count > 0), "human-adjudication adjudication summary.pair_count must be >0")
|
||||
add_error(
|
||||
errors,
|
||||
bool(pair_count and judgment_count == pair_count),
|
||||
"human-adjudication adjudication summary.judgment_count must equal summary.pair_count",
|
||||
)
|
||||
add_error(errors, pending_count == 0, "human-adjudication adjudication summary.pending_count must be 0")
|
||||
add_error(errors, invalid_decision_count == 0, "human-adjudication adjudication summary.invalid_decision_count must be 0")
|
||||
add_error(
|
||||
errors,
|
||||
bool(pair_count and answer_revealed_count == pair_count),
|
||||
"human-adjudication adjudication summary.answer_revealed_count must equal summary.pair_count",
|
||||
)
|
||||
add_error(errors, pending_answer_hidden_count == 0, "human-adjudication adjudication summary.pending_answer_hidden_count must be 0")
|
||||
add_error(errors, summary.get("needs_review") is False, "human-adjudication adjudication summary.needs_review must be false")
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("reviewer_metadata_present") is True,
|
||||
"human-adjudication adjudication summary.reviewer_metadata_present must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("reason_required") is True,
|
||||
"human-adjudication adjudication summary.reason_required must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("ready_for_human_evidence") is True,
|
||||
"human-adjudication adjudication summary.ready_for_human_evidence must be true",
|
||||
)
|
||||
if checklist_count is not None or checklist_ready_count is not None:
|
||||
add_error(
|
||||
errors,
|
||||
bool(pair_count and checklist_count == pair_count and checklist_ready_count == pair_count),
|
||||
"human-adjudication reviewer checklist must be ready for every pair",
|
||||
)
|
||||
add_error(errors, not adjudication.get("failures"), "human-adjudication adjudication failures must be empty")
|
||||
|
||||
decision_rows = object_list(decisions.get("decisions", []))
|
||||
add_error(errors, decisions.get("schema_version") == "1.0", "human-adjudication decisions.schema_version must be 1.0")
|
||||
reviewer = str(decisions.get("reviewer", "")).strip()
|
||||
reviewed_at = str(decisions.get("reviewed_at", "")).strip()
|
||||
require_real_text(errors, reviewer, "human-adjudication decisions.reviewer")
|
||||
add_error(
|
||||
errors,
|
||||
bool(REVIEWED_AT_RE.match(reviewed_at)),
|
||||
"human-adjudication decisions.reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
bool(pair_count and len(decision_rows) == pair_count),
|
||||
"human-adjudication decisions count must equal adjudication summary.pair_count",
|
||||
)
|
||||
|
||||
expected_case_ids = validate_adjudicated_pairs(object_list(adjudication.get("pairs", [])), int(pair_count or 0), errors)
|
||||
validate_decision_rows(decision_rows, expected_case_ids, errors)
|
||||
validate_review_integrity(adjudication, decisions, expected_case_ids, errors)
|
||||
validate_reviewer_attestation(adjudication, decisions, errors)
|
||||
|
||||
provenance_reviewer = str(provenance.get("reviewer", "")).strip()
|
||||
provenance_reviewed_at = str(provenance.get("reviewed_at", "")).strip()
|
||||
add_error(
|
||||
errors,
|
||||
bool(reviewer and provenance_reviewer and reviewer == provenance_reviewer),
|
||||
"human-adjudication provenance.reviewer must match decisions.reviewer",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
bool(reviewed_at and provenance_reviewed_at and provenance_reviewed_at == reviewed_at),
|
||||
"human-adjudication provenance.reviewed_at must match decisions.reviewed_at",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("answer_key_opened_after_decisions") is True,
|
||||
"human-adjudication provenance.answer_key_opened_after_decisions must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("blind_review_completed_before_answer_key") is True,
|
||||
"human-adjudication provenance.blind_review_completed_before_answer_key must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
provenance.get("reviewer_reason_required") is True,
|
||||
"human-adjudication provenance.reviewer_reason_required must be true",
|
||||
)
|
||||
integrity = adjudication.get("review_integrity", {}) if isinstance(adjudication.get("review_integrity", {}), dict) else {}
|
||||
add_error(
|
||||
errors,
|
||||
bool(provenance.get("blind_pack_sha256"))
|
||||
and provenance.get("blind_pack_sha256") == integrity.get("blind_pack_sha256"),
|
||||
"human-adjudication provenance.blind_pack_sha256 must match adjudication review_integrity.blind_pack_sha256",
|
||||
)
|
||||
decision_fields = provenance.get("decision_fields", [])
|
||||
required_decision_fields = {"case_id", "winner_variant", "reason"}
|
||||
add_error(
|
||||
errors,
|
||||
isinstance(decision_fields, list) and required_decision_fields <= {str(item) for item in decision_fields},
|
||||
"human-adjudication provenance.decision_fields must include case_id, winner_variant, and reason",
|
||||
)
|
||||
add_error(errors, adjudication.get("reviewer") == reviewer, "human-adjudication adjudication reviewer must match decisions.reviewer")
|
||||
add_error(
|
||||
errors,
|
||||
adjudication.get("reviewed_at") == reviewed_at,
|
||||
"human-adjudication adjudication reviewed_at must match decisions.reviewed_at",
|
||||
)
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
from typing import Any
|
||||
|
||||
from output_review_privacy import forbidden_decision_field_paths
|
||||
|
||||
|
||||
SCRIPT_INTERFACE = "internal-module"
|
||||
SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate runtime permission evidence from target-level probe rows."
|
||||
|
||||
|
||||
def add_error(errors: list[str], condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
errors.append(message)
|
||||
|
||||
|
||||
def real_int(value: Any) -> int | None:
|
||||
return value if isinstance(value, int) and not isinstance(value, bool) else None
|
||||
|
||||
|
||||
def object_list(value: Any) -> list[dict[str, Any]]:
|
||||
return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else []
|
||||
|
||||
|
||||
def sorted_strings(value: Any) -> list[str]:
|
||||
return sorted(str(item) for item in value) if isinstance(value, list) else []
|
||||
|
||||
|
||||
def validate_no_raw_fields(payload: dict[str, Any], label: str, errors: list[str]) -> None:
|
||||
blocked = forbidden_decision_field_paths(payload, label)
|
||||
add_error(
|
||||
errors,
|
||||
not blocked,
|
||||
f"native-permission-enforcement {label} must not include raw content, credential, secret, token, prompt, output, transcript, message, or answer-key fields: "
|
||||
+ ", ".join(blocked[:8]),
|
||||
)
|
||||
|
||||
|
||||
def validate_target_rows(probes: dict[str, Any], summary: dict[str, Any], errors: list[str]) -> None:
|
||||
targets = object_list(probes.get("targets", []))
|
||||
target_count = real_int(summary.get("target_count"))
|
||||
native_count = real_int(summary.get("native_enforcement_count"))
|
||||
fail_count = real_int(summary.get("fail_count"))
|
||||
expected_capabilities = sorted_strings(probes.get("expected_capabilities"))
|
||||
|
||||
add_error(errors, bool(targets), "native-permission-enforcement runtime probe targets must not be empty")
|
||||
if target_count is not None:
|
||||
add_error(errors, len(targets) == target_count, "native-permission-enforcement runtime probe targets length must equal summary.target_count")
|
||||
add_error(errors, fail_count == 0, "native-permission-enforcement runtime probe summary.fail_count must be 0")
|
||||
|
||||
observed_native = 0
|
||||
for index, target in enumerate(targets, start=1):
|
||||
target_name = str(target.get("target", "")).strip() or str(index)
|
||||
add_error(errors, target.get("status") == "pass", f"native-permission-enforcement target {target_name} status must be pass")
|
||||
add_error(errors, not target.get("failures"), f"native-permission-enforcement target {target_name} failures must be empty")
|
||||
checks = object_list(target.get("checks", []))
|
||||
add_error(errors, bool(checks), f"native-permission-enforcement target {target_name} checks must not be empty")
|
||||
add_error(
|
||||
errors,
|
||||
all(item.get("passed") is True for item in checks),
|
||||
f"native-permission-enforcement target {target_name} checks must all pass",
|
||||
)
|
||||
native = target.get("native_enforcement") is True
|
||||
if native:
|
||||
observed_native += 1
|
||||
add_error(
|
||||
errors,
|
||||
target.get("assurance") == "native-enforced",
|
||||
f"native-permission-enforcement target {target_name} assurance must be native-enforced",
|
||||
)
|
||||
else:
|
||||
add_error(
|
||||
errors,
|
||||
target.get("metadata_fallback_explicit") is True,
|
||||
f"native-permission-enforcement fallback target {target_name} must keep metadata_fallback_explicit true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
bool(target.get("residual_risks")),
|
||||
f"native-permission-enforcement fallback target {target_name} must retain residual risks",
|
||||
)
|
||||
if expected_capabilities:
|
||||
add_error(
|
||||
errors,
|
||||
sorted_strings(target.get("declared_capabilities")) == expected_capabilities,
|
||||
f"native-permission-enforcement target {target_name} declared_capabilities must match expected_capabilities",
|
||||
)
|
||||
installer = target.get("installer_enforcement", {}) if isinstance(target.get("installer_enforcement"), dict) else {}
|
||||
add_error(
|
||||
errors,
|
||||
installer.get("enforced") is True,
|
||||
f"native-permission-enforcement target {target_name} installer_enforcement.enforced must be true",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
not installer.get("missing_capabilities"),
|
||||
f"native-permission-enforcement target {target_name} installer_enforcement missing_capabilities must be empty",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
not installer.get("failure_details"),
|
||||
f"native-permission-enforcement target {target_name} installer_enforcement failure_details must be empty",
|
||||
)
|
||||
|
||||
add_error(errors, bool(native_count and observed_native == native_count), "native-permission-enforcement native target rows must match summary.native_enforcement_count")
|
||||
add_error(errors, observed_native > 0, "native-permission-enforcement must include at least one native-enforced target row")
|
||||
|
||||
|
||||
def validate_install_simulation_report(install: dict[str, Any], errors: list[str]) -> None:
|
||||
summary = install.get("summary", {}) if isinstance(install.get("summary", {}), dict) else {}
|
||||
validate_no_raw_fields(install, "install_simulation", errors)
|
||||
add_error(errors, install.get("ok") is True, "native-permission-enforcement install simulation report ok must be true")
|
||||
add_error(
|
||||
errors,
|
||||
bool(real_int(summary.get("installer_permission_enforced_count")) and summary["installer_permission_enforced_count"] > 0),
|
||||
"native-permission-enforcement install simulation summary.installer_permission_enforced_count must be >0",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("installer_permission_failure_count") == 0,
|
||||
"native-permission-enforcement install simulation summary.installer_permission_failure_count must be 0",
|
||||
)
|
||||
add_error(errors, summary.get("failure_count") == 0, "native-permission-enforcement install simulation summary.failure_count must be 0")
|
||||
permission_checks = [
|
||||
item
|
||||
for item in object_list(install.get("checks", []))
|
||||
if str(item.get("id", "")).startswith("permission-") and str(item.get("id", "")).endswith(("-approved", "-target-enforcement"))
|
||||
]
|
||||
if permission_checks:
|
||||
add_error(
|
||||
errors,
|
||||
all(item.get("status") == "pass" for item in permission_checks),
|
||||
"native-permission-enforcement install simulation permission checks must all pass",
|
||||
)
|
||||
|
||||
|
||||
def validate_native_permission_report(probes: dict[str, Any], install: dict[str, Any], errors: list[str]) -> None:
|
||||
summary = probes.get("summary", {}) if isinstance(probes.get("summary", {}), dict) else {}
|
||||
validate_no_raw_fields(probes, "runtime_permission_probes", errors)
|
||||
add_error(errors, probes.get("ok") is True, "native-permission-enforcement runtime probe report ok must be true")
|
||||
add_error(
|
||||
errors,
|
||||
bool(real_int(summary.get("native_enforcement_count")) and summary["native_enforcement_count"] > 0),
|
||||
"native-permission-enforcement runtime probe summary.native_enforcement_count must be >0",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("failure_count") == 0,
|
||||
"native-permission-enforcement runtime probe summary.failure_count must be 0",
|
||||
)
|
||||
add_error(
|
||||
errors,
|
||||
summary.get("installer_enforcement_ready") is True,
|
||||
"native-permission-enforcement runtime probe summary.installer_enforcement_ready must be true",
|
||||
)
|
||||
validate_target_rows(probes, summary, errors)
|
||||
if install:
|
||||
validate_install_simulation_report(install, errors)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user