yao-meta-skill/scripts/render_intent_confidence.py

#!/usr/bin/env python3
import argparse
import json
import re
from pathlib import Path
from typing import Any

try:
    import yaml
except ImportError:  # pragma: no cover
    yaml = None


GENERIC_PHRASES = {
    "turn a repeated workflow into a reusable skill",
    "a reusable skill package",
    "describe what the skill does and when to use it",
    "turn rough requests into a compact reusable demo skill",
}

GENERIC_TOKENS = {
    "workflow",
    "skill",
    "package",
    "reusable",
    "repeated",
    "request",
    "requests",
    "task",
    "tasks",
    "work",
    "job",
}

FOLLOW_UP_LIBRARY = {
    "job_specificity": {
        "slot": "job",
        "question": "If you say it plainly, what concrete repeated task should this skill own every time?",
        "why": "A skill needs a real recurring job, not only a generic packaging goal.",
        "list": False,
    },
    "real_inputs": {
        "slot": "real_inputs",
        "question": "What material will people actually hand to this skill in practice?",
        "why": "Real input shape decides whether references, scripts, or examples are needed.",
        "list": True,
    },
    "primary_output": {
        "slot": "primary_output",
        "question": "What finished hand-back should this skill return so the next person can keep moving?",
        "why": "The output is the anchor for package design and review.",
        "list": False,
    },
    "exclusions": {
        "slot": "exclusions",
        "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?",
        "why": "Exclusions are the fastest route to better trigger quality.",
        "list": True,
    },
    "constraints": {
        "slot": "constraints",
        "question": "What constraints matter most here: privacy, naming, compatibility, portability, governance, or speed?",
        "why": "Constraints decide how much structure and validation this skill really needs.",
        "list": True,
    },
    "standards": {
        "slot": "standards",
        "question": "What quality bar matters most here: consistency, auditability, tone, or delivery speed?",
        "why": "Standards explain how to choose the first evaluation gate.",
        "list": True,
    },
}


def parse_frontmatter(text: str) -> tuple[dict, str]:
    lines = text.splitlines()
    if not lines or lines[0].strip() != "---":
        return {}, text
    try:
        end_index = lines[1:].index("---") + 1
    except ValueError:
        return {}, text
    frontmatter_text = "\n".join(lines[1:end_index])
    body = "\n".join(lines[end_index + 1 :]).lstrip()
    if yaml is not None:
        payload = yaml.safe_load(frontmatter_text) or {}
        return payload if isinstance(payload, dict) else {}, body
    data = {}
    for line in frontmatter_text.splitlines():
        if ":" not in line:
            continue
        key, value = line.split(":", 1)
        data[key.strip()] = value.strip().strip('"')
    return data, body


def normalized_list(value: list[str] | str | None) -> list[str]:
    if value is None:
        return []
    if isinstance(value, str):
        parts = [item.strip() for item in value.split(",")]
        return [item for item in parts if item]
    return [str(item).strip() for item in value if str(item).strip()]


def tokenize(text: str) -> list[str]:
    return re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", text.lower())


def is_generic(text: str) -> bool:
    compact = " ".join(text.lower().split())
    if not compact:
        return True
    if compact in GENERIC_PHRASES:
        return True
    tokens = tokenize(compact)
    if len(tokens) <= 3:
        return True
    content_tokens = [token for token in tokens if token not in GENERIC_TOKENS]
    return len(content_tokens) < 2


def build_context_from_skill(skill_dir: Path) -> dict[str, Any]:
    skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8")
    frontmatter, _ = parse_frontmatter(skill_text)
    payload = load_json(skill_dir / "reports" / "intent-context.json")
    if payload:
        return payload
    return {
        "job": frontmatter.get("description", ""),
        "real_inputs": [],
        "primary_output": "",
        "description": frontmatter.get("description", ""),
        "exclusions": [],
        "constraints": [],
        "standards": [],
        "correction": "",
        "user_references": [],
    }


def load_json(path: Path) -> dict[str, Any]:
    if not path.exists():
        return {}
    payload = json.loads(path.read_text(encoding="utf-8"))
    return payload if isinstance(payload, dict) else {}


def assess_intent_confidence(context: dict[str, Any]) -> dict[str, Any]:
    job = str(context.get("job", "")).strip()
    primary_output = str(context.get("primary_output", "")).strip()
    description = str(context.get("description", "")).strip()
    real_inputs = normalized_list(context.get("real_inputs"))
    exclusions = normalized_list(context.get("exclusions"))
    constraints = normalized_list(context.get("constraints"))
    standards = normalized_list(context.get("standards"))
    user_references = normalized_list(context.get("user_references"))
    correction = str(context.get("correction", "")).strip()

    score = 0
    strengths = []
    gaps = []

    def add_gap(key: str, label: str, reason: str, severity: str = "high") -> None:
        gaps.append({"key": key, "label": label, "reason": reason, "severity": severity})

    if job and not is_generic(job):
        score += 25
        strengths.append("The recurring job is concrete enough to anchor the package.")
    elif job:
        score += 10
        add_gap(
            "job_specificity",
            "Recurring job is still generic",
            "The current job statement sounds more like a packaging goal than a concrete repeated task.",
        )
    else:
        add_gap("job_specificity", "Recurring job is missing", "The package has no clear job-to-be-done anchor yet.")

    if real_inputs:
        score += 15
        strengths.append("Real input shape is explicit.")
    else:
        add_gap("real_inputs", "Real inputs are missing", "Without real inputs, it is hard to choose assets, scripts, or examples.")

    if primary_output and not is_generic(primary_output):
        score += 20
        strengths.append("The hand-back output is concrete.")
    elif primary_output:
        score += 8
        add_gap(
            "primary_output",
            "Primary output is still generic",
            "The current output does not yet say what a useful finished deliverable looks like.",
        )
    else:
        add_gap("primary_output", "Primary output is missing", "The package does not yet know what it must hand back.")

    if exclusions:
        score += 15
        strengths.append("Boundary exclusions are already explicit.")
    else:
        add_gap("exclusions", "Near-neighbor exclusions are missing", "The route may blur into nearby requests without an exclusion list.")

    if constraints:
        score += 10
        strengths.append("Operational constraints are visible.")
    else:
        add_gap("constraints", "Constraints are missing", "The package does not yet know which tradeoffs matter most.")

    if standards:
        score += 5
        strengths.append("Quality standards are visible.")
    else:
        add_gap("standards", "Quality bar is implied, not explicit", "The first evaluation target is still underspecified.", "medium")

    if correction:
        score += 5
        strengths.append("A correction loop already tightened the first reading.")

    if user_references:
        score += 5
        strengths.append("Reference preferences are already available.")

    if description and not is_generic(description):
        score += 5

    score = min(score, 100)
    if score >= 85:
        band = "high"
    elif score >= 70:
        band = "medium"
    else:
        band = "low"

    gate_passed = score >= 70 and not any(
        gap["key"] in {"job_specificity", "real_inputs", "primary_output"} and gap["severity"] == "high"
        for gap in gaps
    )
    follow_up_questions = [
        {
            **FOLLOW_UP_LIBRARY[gap["key"]],
            "label": gap["label"],
            "severity": gap["severity"],
        }
        for gap in gaps
        if gap["key"] in FOLLOW_UP_LIBRARY
    ][:3]

    anchor_sentence = " ".join(
        item
        for item in [
            job or "Unclear recurring job.",
            f"Primary output: {primary_output}." if primary_output else "",
            f"Exclusions: {', '.join(exclusions)}." if exclusions else "",
        ]
        if item
    ).strip()

    return {
        "score": score,
        "band": band,
        "gate_passed": gate_passed,
        "strengths": strengths[:5],
        "gaps": gaps,
        "follow_up_questions": follow_up_questions,
        "anchor_sentence": anchor_sentence,
        "recommended_action": (
            "Intent is clear enough to package the first routeable version."
            if gate_passed
            else "Pause before deep authoring and close the highest-leverage gaps first."
        ),
        "context": {
            "job": job,
            "real_inputs": real_inputs,
            "primary_output": primary_output,
            "description": description,
            "exclusions": exclusions,
            "constraints": constraints,
            "standards": standards,
            "correction": correction,
            "user_references": user_references,
        },
    }


def render_markdown(summary: dict[str, Any]) -> str:
    lines = [
        "# Intent Confidence",
        "",
        f"- Confidence score: `{summary['score']}/100`",
        f"- Confidence band: `{summary['band']}`",
        f"- Gate passed: `{summary['gate_passed']}`",
        f"- Recommended action: {summary['recommended_action']}",
        "",
        "## Current Reading",
        "",
        summary["anchor_sentence"] or "No clear anchor sentence yet.",
        "",
        "## Strong Signals",
        "",
    ]
    if summary["strengths"]:
        for item in summary["strengths"]:
            lines.append(f"- {item}")
    else:
        lines.append("- No strong signals yet.")

    lines.extend(["", "## Gaps To Close", ""])
    if summary["gaps"]:
        for gap in summary["gaps"]:
            lines.append(f"- **{gap['label']}** (`{gap['severity']}`): {gap['reason']}")
    else:
        lines.append("- No major intent gaps detected.")

    lines.extend(["", "## Follow-Up Questions", ""])
    if summary["follow_up_questions"]:
        for item in summary["follow_up_questions"]:
            lines.append(f"- **{item['question']}**")
            lines.append(f"  - Why: {item['why']}")
    else:
        lines.append("- No extra follow-up questions required before the first package.")
    return "\n".join(lines).strip() + "\n"


def render_intent_confidence(
    skill_dir: Path,
    context: dict[str, Any] | None = None,
    output_md: Path | None = None,
    output_json: Path | None = None,
    context_json: Path | None = None,
) -> dict[str, Any]:
    skill_dir = skill_dir.resolve()
    reports_dir = skill_dir / "reports"
    reports_dir.mkdir(parents=True, exist_ok=True)
    output_md = output_md or reports_dir / "intent-confidence.md"
    output_json = output_json or reports_dir / "intent-confidence.json"
    context_json = context_json or reports_dir / "intent-context.json"

    context_payload = context or build_context_from_skill(skill_dir)
    summary = assess_intent_confidence(context_payload)
    output_md.write_text(render_markdown(summary), encoding="utf-8")
    output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    context_json.write_text(json.dumps(summary["context"], ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return {
        "ok": True,
        "skill_dir": str(skill_dir),
        "artifacts": {
            "markdown": str(output_md),
            "json": str(output_json),
            "context_json": str(context_json),
        },
        "summary": summary,
    }


def main() -> None:
    parser = argparse.ArgumentParser(description="Render an intent confidence report for a skill package.")
    parser.add_argument("skill_dir", nargs="?", default=".")
    parser.add_argument("--context-json")
    parser.add_argument("--output-md")
    parser.add_argument("--output-json")
    args = parser.parse_args()

    context = None
    if args.context_json:
        context = load_json(Path(args.context_json).resolve())

    result = render_intent_confidence(
        Path(args.skill_dir),
        context=context,
        output_md=Path(args.output_md).resolve() if args.output_md else None,
        output_json=Path(args.output_json).resolve() if args.output_json else None,
    )
    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()