From 7126bcb2e58547ef2ab0d76049297cd1f5857e29 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Thu, 23 Apr 2026 14:41:37 +0800 Subject: [PATCH] feat: surface only real reference conflicts --- reports/context_budget.json | 4 +- reports/reference-synthesis.json | 5 +- reports/reference-synthesis.md | 8 +- scripts/render_reference_synthesis.py | 162 ++++++++++++++++++++++++-- scripts/yao.py | 15 ++- tests/verify_reference_synthesis.py | 9 +- tests/verify_yao_cli.py | 24 ++++ 7 files changed, 202 insertions(+), 25 deletions(-) diff --git a/reports/context_budget.json b/reports/context_budget.json index 4bbba639..d6c7821f 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,9 +6,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 818, - "other_text_tokens": 293209, + "other_text_tokens": 295144, "estimated_initial_load_tokens": 994, - "estimated_total_text_tokens": 294027, + "estimated_total_text_tokens": 295962, "relevant_file_count": 184, "unused_resource_dirs": [], "quality_signal_points": 130, diff --git a/reports/reference-synthesis.json b/reports/reference-synthesis.json index a3a0d256..55ff51bf 100644 --- a/reports/reference-synthesis.json +++ b/reports/reference-synthesis.json @@ -59,6 +59,7 @@ "Use research tracks to justify the smallest evaluation loop that still catches regressions.", "Use principle tracks to keep the package small, boundary-aware, and outcome-driven." ], + "conflicts": [], "recommendation": { "summary": "Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.", "borrow_now": [ @@ -78,10 +79,10 @@ "reasons": [ "intent_uncertain" ], - "user_note": "Surface the recommendation because intent is still settling or a user reference needs to be reconciled.", + "user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.", "reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers." }, - "decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a user reference needs a deliberate call.", + "decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.", "source_mix": { "github_benchmarks": 0, "curated_tracks": 3, diff --git a/reports/reference-synthesis.md b/reports/reference-synthesis.md index a10c6297..5fe6ae16 100644 --- a/reports/reference-synthesis.md +++ b/reports/reference-synthesis.md @@ -53,9 +53,13 @@ Skill: `yao-meta-skill` - Mode: `explicit` - Reasons: intent_uncertain -- User note: Surface the recommendation because intent is still settling or a user reference needs to be reconciled. +- User note: Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call. - Reviewer note: Keep the full benchmark and synthesis evidence visible for authors and reviewers. +## Conflict Check + +- No material design conflict detected. Keep the synthesis silent for the user. + ## Quality Lift Thesis - Use GitHub repositories for concrete package and workflow patterns. @@ -65,4 +69,4 @@ Skill: `yao-meta-skill` ## Decision Prompt -Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a user reference needs a deliberate call. +Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call. diff --git a/scripts/render_reference_synthesis.py b/scripts/render_reference_synthesis.py index 018525c2..574c1516 100644 --- a/scripts/render_reference_synthesis.py +++ b/scripts/render_reference_synthesis.py @@ -63,6 +63,11 @@ CURATED_TRACKS = [ }, ] +LIGHTWEIGHT_KEYWORDS = ["light", "lean", "minimal", "small", "simple", "fast", "speed", "quick", "scaffold"] +GOVERNED_KEYWORDS = ["govern", "audit", "review", "approval", "compliance", "risk", "trust", "policy", "cadence"] +POLISH_KEYWORDS = ["polish", "beautiful", "ui", "ux", "experience", "operator flow", "product", "viewer", "ergonomic"] +EVAL_HEAVY_KEYWORDS = ["benchmark", "holdout", "regression", "evidence", "test", "compare", "review checkpoint"] + def load_json(path: Path) -> dict[str, Any]: if not path.exists(): @@ -71,6 +76,10 @@ def load_json(path: Path) -> dict[str, Any]: return payload if isinstance(payload, dict) else {} +def load_manifest(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "manifest.json") + + def parse_frontmatter(text: str) -> tuple[dict, str]: lines = text.splitlines() if not lines or lines[0].strip() != "---": @@ -160,19 +169,136 @@ def unique_items(items: list[str], limit: int) -> list[str]: return output -def build_visibility(intent_payload: dict[str, Any], user_refs: list[dict[str, Any]]) -> dict[str, Any]: +def join_text(parts: list[str]) -> str: + return " ".join(part for part in parts if part).lower() + + +def has_any(text: str, keywords: list[str]) -> bool: + normalized = re.sub(r"[^a-z0-9]+", " ", text.lower()).strip() + tokens = set(normalized.split()) + for keyword in keywords: + normalized_keyword = re.sub(r"[^a-z0-9]+", " ", keyword.lower()).strip() + if not normalized_keyword: + continue + if " " in normalized_keyword: + if normalized_keyword in normalized: + return True + elif normalized_keyword in tokens: + return True + return False + + +def detect_conflicts( + intent_payload: dict[str, Any], + user_refs: list[dict[str, Any]], + manifest: dict[str, Any], + source_tracks: list[dict[str, Any]], + borrow_now: list[str], + avoid_now: list[str], +) -> list[dict[str, Any]]: + context = intent_payload.get("context", {}) if isinstance(intent_payload, dict) else {} + preference_text = join_text( + [ + str(context.get("job", "")), + str(context.get("primary_output", "")), + str(context.get("description", "")), + " ".join(context.get("constraints", []) or []), + " ".join(context.get("standards", []) or []), + " ".join(context.get("exclusions", []) or []), + str(manifest.get("skill_archetype", "")), + str(manifest.get("maturity_tier", "")), + " ".join( + " ".join( + str(ref.get(key, "")) for key in ("name", "borrow", "avoid", "category") + ) + for ref in user_refs + ), + ] + ) + benchmark_text = join_text( + [ + *borrow_now, + *avoid_now, + *[ + " ".join( + [ + str(track.get("name", "")), + str(track.get("borrow", "")), + str(track.get("avoid", "")), + ] + ) + for track in source_tracks + ], + ] + ) + + conflicts = [] + wants_lightweight = has_any(preference_text, LIGHTWEIGHT_KEYWORDS) + wants_governed = has_any(preference_text, GOVERNED_KEYWORDS) + wants_polish = has_any(preference_text, POLISH_KEYWORDS) + benchmark_heavy = has_any(benchmark_text, GOVERNED_KEYWORDS + EVAL_HEAVY_KEYWORDS) + benchmark_minimal = has_any(benchmark_text, LIGHTWEIGHT_KEYWORDS) + benchmark_anti_polish = any("polish" in item.lower() or "ui bulk" in item.lower() for item in avoid_now) + + if wants_lightweight and benchmark_heavy: + conflicts.append( + { + "key": "lightweight_vs_governance", + "summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_preference": "lightweight or speed-first", + "benchmark_pressure": "governance or evaluation-heavy patterns", + } + ) + if wants_polish and benchmark_anti_polish: + conflicts.append( + { + "key": "polish_vs_minimal_structure", + "summary": "The stated preference leans toward product polish or richer operator experience, while the benchmark recommendation is trying to keep the first pass structurally minimal.", + "user_preference": "product polish or richer operator experience", + "benchmark_pressure": "minimum-structure first pass", + } + ) + if wants_governed and manifest.get("skill_archetype") == "scaffold": + conflicts.append( + { + "key": "governance_vs_scaffold", + "summary": "The stated preference leans toward governance or auditability, but the current archetype is still scaffold-level.", + "user_preference": "governance or auditability", + "benchmark_pressure": "scaffold-first package shape", + } + ) + if wants_governed and benchmark_minimal and not benchmark_heavy: + conflicts.append( + { + "key": "governance_vs_minimal_structure", + "summary": "The stated preference leans toward governance or auditability, while the benchmark recommendation is still biased toward a lightweight first pass.", + "user_preference": "governance or auditability", + "benchmark_pressure": "lightweight first-pass structure", + } + ) + seen = set() + deduped = [] + for conflict in conflicts: + if conflict["key"] in seen: + continue + seen.add(conflict["key"]) + deduped.append(conflict) + return deduped + + +def build_visibility(intent_payload: dict[str, Any], conflicts: list[dict[str, Any]]) -> dict[str, Any]: reasons = [] if not intent_payload.get("gate_passed", False): reasons.append("intent_uncertain") - if user_refs: - reasons.append("user_reference_alignment") + if conflicts: + reasons.append("design_conflict") mode = "explicit" if reasons else "silent" return { "mode": mode, "user_decision_required": mode == "explicit", "reasons": reasons, "user_note": ( - "Surface the recommendation because intent is still settling or a user reference needs to be reconciled." + "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call." if mode == "explicit" else "Apply the synthesis quietly unless uncertainty or a real design conflict appears." ), @@ -185,14 +311,16 @@ def build_recommendation( avoid_now: list[str], intent_payload: dict[str, Any], visibility: dict[str, Any], + conflicts: list[dict[str, Any]], ) -> dict[str, Any]: primary_borrow = borrow_now[0] if borrow_now else "Keep the entrypoint lean and boundary-first." primary_avoid = avoid_now[0] if avoid_now else "Do not add weight that the first pass does not yet need." - why = ( - "Intent is clear enough, so the system should make the first pattern call quietly." - if intent_payload.get("gate_passed", False) - else "Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package." - ) + if conflicts: + why = f"There is a real design conflict to resolve: {conflicts[0]['summary']}" + elif intent_payload.get("gate_passed", False): + why = "Intent is clear enough, so the system should make the first pattern call quietly." + else: + why = "Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package." return { "summary": f"Start by borrowing this pattern: {primary_borrow} Avoid this for the first pass: {primary_avoid}", "borrow_now": borrow_now[:2], @@ -208,6 +336,7 @@ def build_summary(skill_dir: Path) -> dict[str, Any]: benchmark = load_json(skill_dir / "reports" / "github-benchmark-scan.json") intent_payload = load_json(skill_dir / "reports" / "intent-confidence.json") reference_scan = load_json(skill_dir / "reports" / "reference-scan.json") + manifest = load_manifest(skill_dir) source_tracks = select_source_tracks(anchor_text(skill_dir, benchmark, intent_payload)) github_repos = benchmark.get("repositories", [])[:3] @@ -242,8 +371,9 @@ def build_summary(skill_dir: Path) -> dict[str, Any]: ], 4, ) - visibility = build_visibility(intent_payload, user_refs) - recommendation = build_recommendation(borrow_now, avoid_now, intent_payload, visibility) + conflicts = detect_conflicts(intent_payload, user_refs, manifest, source_tracks, borrow_now, avoid_now) + visibility = build_visibility(intent_payload, conflicts) + recommendation = build_recommendation(borrow_now, avoid_now, intent_payload, visibility, conflicts) return { "skill_name": frontmatter.get("name", skill_dir.name), @@ -267,10 +397,11 @@ def build_summary(skill_dir: Path) -> dict[str, Any]: "borrow_now": borrow_now, "avoid_now": avoid_now, "quality_risers": quality_risers, + "conflicts": conflicts, "recommendation": recommendation, "visibility": visibility, "decision_prompt": ( - "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a user reference needs a deliberate call." + "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call." ), "source_mix": { "github_benchmarks": len(github_repos), @@ -342,6 +473,13 @@ def render_markdown(summary: dict[str, Any]) -> str: lines.append(f"- User note: {summary['synthesis']['visibility']['user_note']}") lines.append(f"- Reviewer note: {summary['synthesis']['visibility']['reviewer_note']}") + lines.extend(["", "## Conflict Check", ""]) + if summary["synthesis"]["conflicts"]: + for conflict in summary["synthesis"]["conflicts"]: + lines.append(f"- **{conflict['key']}**: {conflict['summary']}") + else: + lines.append("- No material design conflict detected. Keep the synthesis silent for the user.") + lines.extend(["", "## Quality Lift Thesis", ""]) for item in summary["synthesis"]["quality_risers"]: lines.append(f"- {item}") diff --git a/scripts/yao.py b/scripts/yao.py index 9b2c749d..73af2b63 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -213,17 +213,16 @@ def discovery_summary(job: str, primary_output: str, archetype: str, guidance: d ) -def reference_visibility(reference_synthesis: dict, user_references: list[str]) -> dict: +def reference_visibility(reference_synthesis: dict) -> dict: synthesis = reference_synthesis.get("synthesis", {}) if isinstance(reference_synthesis, dict) else {} visibility = synthesis.get("visibility", {}) if isinstance(synthesis, dict) else {} reasons = list(visibility.get("reasons", [])) - if user_references and "user_reference_alignment" not in reasons: - reasons.append("user_reference_alignment") - mode = "explicit" if reasons else visibility.get("mode", "silent") + mode = visibility.get("mode", "explicit" if reasons else "silent") return { "mode": mode, "user_decision_required": mode == "explicit", "reasons": reasons, + "conflicts": synthesis.get("conflicts", []), } @@ -408,11 +407,14 @@ def command_quickstart(args: argparse.Namespace) -> int: result = run_script("init_skill.py", cmd) payload = result["payload"] if result["payload"] is not None else result reference_synthesis = payload.get("reference_synthesis") or {} - visibility = reference_visibility(reference_synthesis, user_references) + visibility = reference_visibility(reference_synthesis) recommendation = recommendation_from_synthesis(reference_synthesis, visibility) sys.stderr.write(f"\nRecommendation: {recommendation['summary']}\n") if visibility["user_decision_required"]: - sys.stderr.write("I am surfacing this because intent is still settling or a user reference needs a deliberate call.\n") + if visibility["conflicts"]: + sys.stderr.write(f"I am surfacing this because there is a real design conflict: {visibility['conflicts'][0]['summary']}\n") + else: + sys.stderr.write("I am surfacing this because intent is still settling and the package should not deepen on guesswork.\n") else: sys.stderr.write("I will keep the underlying benchmark evidence in the reviewer reports and move ahead with this recommendation.\n") @@ -470,6 +472,7 @@ def command_quickstart(args: argparse.Namespace) -> int: if visibility["user_decision_required"]: report["uncertainty_or_conflict"] = { "reasons": visibility["reasons"], + "conflicts": visibility["conflicts"], "note": "A design decision still needs your input before the package should be deepened.", } print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/tests/verify_reference_synthesis.py b/tests/verify_reference_synthesis.py index 76977352..3ec3a0f0 100644 --- a/tests/verify_reference_synthesis.py +++ b/tests/verify_reference_synthesis.py @@ -25,12 +25,16 @@ def main() -> None: "Turn repeated release notes into a portable governed release skill.", "--output-dir", str(tmp_root), + "--mode", + "governed", + "--archetype", + "governed", "--github-query", "release workflow evaluation portability", "--github-fixture-dir", str(BENCHMARK_FIXTURE_DIR), "--user-reference", - "A trusted workflow product::taste::Learn the calm operating shape.::Do not copy wording.", + "Minimal vibe helper::taste::Keep the first pass fast, minimal, and lightweight.::Do not add review, governance, or approval steps.", "--intent-job", "Turn repeated release notes into a portable governed release skill.", "--intent-real-input", @@ -61,10 +65,13 @@ def main() -> None: assert synthesis["synthesis"]["borrow_now"], synthesis assert synthesis["synthesis"]["recommendation"]["summary"], synthesis assert synthesis["synthesis"]["visibility"]["mode"] == "explicit", synthesis + assert "design_conflict" in synthesis["synthesis"]["visibility"]["reasons"], synthesis + assert synthesis["synthesis"]["conflicts"], synthesis markdown = (skill_dir / "reports" / "reference-synthesis.md").read_text(encoding="utf-8") assert "Curated World-Class Pattern Tracks" in markdown, markdown[:600] assert "Borrow Now" in markdown, markdown[:900] assert "Default Recommendation" in markdown, markdown[:1200] + assert "Conflict Check" in markdown, markdown[:1500] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 7ec817da..76cd42df 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -82,6 +82,30 @@ def main() -> None: ), quickstart_result assert "uncertainty_or_conflict" not in quickstart_result["payload"], quickstart_result + quickstart_conflict_result = run( + "quickstart", + "--output-dir", + str(tmp_root), + "--github-fixture-dir", + str(BENCHMARK_FIXTURE_DIR), + input_text=( + "quickstart-conflict-skill\n" + "Turn repeated release notes into a governed release command skill.\n" + "release notes, changelog snippets\n" + "A governed release packet.\n" + "looks right\n" + "It should not publish blog posts or send email.\n" + "auditability, portability\n" + "governed\n" + "governed\n" + "Minimal vibe helper::taste::Keep the first pass fast, minimal, and lightweight.::Do not add review, governance, or approval steps.\n" + "privacy and naming\n" + ), + ) + assert quickstart_conflict_result["ok"], quickstart_conflict_result + assert quickstart_conflict_result["payload"]["reference_mode"]["mode"] == "explicit", quickstart_conflict_result + assert quickstart_conflict_result["payload"]["uncertainty_or_conflict"]["conflicts"], quickstart_conflict_result + validate_result = run("validate", str(created)) assert validate_result["ok"], validate_result assert len(validate_result["payload"]["steps"]) == 4, validate_result