From 5b24cb2caed3a8634ef8dd078eed844955d7b774 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 14 Apr 2026 11:09:14 +0800 Subject: [PATCH] feat: add github benchmark scan to quickstart --- .gitignore | 1 + Makefile | 9 +- README.md | 5 +- docs/README.fr-FR.md | 4 +- docs/README.ja-JP.md | 4 +- docs/README.ru-RU.md | 4 +- docs/README.zh-CN.md | 4 +- reports/context_budget.json | 6 +- scripts/github_benchmark_scan.py | 400 ++++++++++++++++++ scripts/init_skill.py | 77 +++- scripts/yao.py | 69 ++- .../github_benchmark_scan/bundle.json | 39 ++ tests/verify_github_benchmark_scan.py | 55 +++ tests/verify_yao_cli.py | 16 + 14 files changed, 658 insertions(+), 35 deletions(-) create mode 100644 scripts/github_benchmark_scan.py create mode 100644 tests/fixtures/github_benchmark_scan/bundle.json create mode 100644 tests/verify_github_benchmark_scan.py diff --git a/.gitignore b/.gitignore index 624225fb..f003499d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ tests/tmp_reference_scan/ tests/tmp_iteration_directions/ tests/tmp_review_viewer/ tests/tmp_feedback/ +tests/tmp_github_benchmark_scan/ reports/release_snapshots/ diff --git a/Makefile b/Makefile index 96baa5a0..b421d29b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -44,6 +44,9 @@ baseline-compare-check: reference-scan-check: $(PYTHON) tests/verify_reference_scan.py +github-benchmark-scan-check: + $(PYTHON) tests/verify_github_benchmark_scan.py + iteration-directions-check: $(PYTHON) tests/verify_iteration_directions.py @@ -95,7 +98,7 @@ resource-boundary-check: quality-check: $(PYTHON) tests/verify_quality_checks.py -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md diff --git a/README.md b/README.md index 8a0b0f02..cc4de1bb 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a lean `SKILL.md` - optional references, scripts, and evals - a front-loaded intent dialogue that starts from the user's real work, desired outcome, and quality bar -- a controlled benchmark scan that prefers high-quality public references first and asks whether the user has examples worth learning from +- an automatic GitHub benchmark scan that pulls the top three public repositories first, extracts borrow or avoid patterns, and then asks whether the user has references worth learning from - a generated visual HTML overview for each newly initialized skill - a compact HTML review viewer for first-pass human review - three high-value next iteration directions after the first package is created @@ -80,7 +80,7 @@ This is a scenario-oriented benchmark shared with the project. It is most useful 1. Describe the workflow, prompt set, or repeated task you want to turn into a skill. 2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit. -3. Run a short reference scan with high-quality public benchmarks first, then ask whether the user has examples worth learning from; use local files only for fit and privacy checks. +3. Let `quickstart` run a short GitHub benchmark scan first, study the top three public repositories it retrieves, then ask whether the user has examples worth learning from; use local files only for fit and privacy checks. 4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode. 5. Review the generated `reports/intent-dialogue.md`, `reports/skill-overview.html`, and `reports/iteration-directions.md` before adding more structure. @@ -88,6 +88,7 @@ Or use the unified authoring CLI: ```bash python3 scripts/yao.py quickstart --output-dir . +python3 scripts/yao.py github-benchmark-scan my-skill --query "release workflow portability" python3 scripts/yao.py reference-scan my-skill \ --external-reference "World Class Method::method::Borrow a tight evaluation loop.::Do not copy heavy process." \ --user-reference "A product or repo I admire::taste::Learn the clarity and operating standard.::Do not copy wording." \ diff --git a/docs/README.fr-FR.md b/docs/README.fr-FR.md index 8d99f3a3..486a2b90 100644 --- a/docs/README.fr-FR.md +++ b/docs/README.fr-FR.md @@ -10,7 +10,7 @@ Il transforme des workflows bruts, des transcripts, des prompts, des notes et de - un `SKILL.md` léger - des references, scripts et evals optionnels - un dialogue d'intention plus humain avant l'authoring approfondi, pour comprendre le vrai travail, la sortie attendue, les frontières et le niveau d'exigence -- un benchmark/reference scan contrôlé avant l'authoring profond, en donnant la priorité aux dépôts GitHub très reconnus, aux documentations officielles et aux références publiques de niveau mondial +- un benchmark scan GitHub automatique avant l'authoring profond, qui récupère d'abord trois dépôts publics très reconnus puis extrait les patterns à reprendre ou à éviter - une demande explicite de références fournies par l'utilisateur quand elles existent, afin d'apprendre des modèles, pas de copier le texte ni du contenu privé - un rapport HTML minimaliste en fond blanc généré automatiquement pour chaque nouveau skill - trois directions d'itération à plus forte valeur après la première création @@ -71,7 +71,7 @@ Le tableau ci-dessous est un comparatif orienté scénario. Il aide à choisir l 1. Décrivez le workflow, l'ensemble de prompts ou la tâche répétée que vous voulez transformer en skill. 2. Commencez par un court dialogue d'intention plus humain pour clarifier le vrai travail, les sorties attendues, les exclusions, les contraintes et les standards qui comptent pour vous. -3. Lancez ensuite un court reference scan en privilégiant les dépôts GitHub à forte reconnaissance, les documentations officielles et les références publiques de haut niveau ; le système vous demandera aussi si vous avez des exemples à faire étudier. Les fichiers locaux ne servent ensuite qu'à l'ajustement, à la confidentialité et à la compatibilité. +3. Laissez d'abord `quickstart` lancer un benchmark scan GitHub pour récupérer trois bons dépôts publics et en extraire les patterns réutilisables ; ajoutez ensuite vos propres références si besoin. Les fichiers locaux ne servent ensuite qu'à l'ajustement, à la confidentialité et à la compatibilité. 4. Utilisez le `quickstart` sensible aux archetypes ou le flux complet d'authoring pour générer ou améliorer le paquet en mode scaffold, production, library ou governed. 5. Chaque nouveau skill reçoit aussi `reports/intent-dialogue.md`, `reports/skill-overview.html`, `reports/review-viewer.html`, `reports/reference-scan.md` et `reports/iteration-directions.md`. Ensuite, le feedback log et le baseline compare permettent de boucler rapidement sans lancer tout le flux de promotion. diff --git a/docs/README.ja-JP.md b/docs/README.ja-JP.md index cb3ed5fa..a538a8a4 100644 --- a/docs/README.ja-JP.md +++ b/docs/README.ja-JP.md @@ -10,7 +10,7 @@ - 軽量な `SKILL.md` - 必要に応じた references、scripts、evals - 深い authoring の前に、人間味のある短い intent dialogue を行い、本当の仕事、成果物、境界、重視する基準をつかむ -- 深い authoring の前に、GitHub の高評価リポジトリ、公式ドキュメント、世界水準の公開事例を優先する制御された benchmark/reference scan を 1 回行う +- 深い authoring の前に、自動 GitHub benchmark scan を 1 回実行し、高評価の公開リポジトリを 3 件取得して borrow / avoid パターンを整理する - ユーザー自身の参考例があれば、それも取り込み、文章ではなくパターン・構造・品質基準だけを学ぶ - 新しい skill ごとに、白背景の簡潔な HTML overview を自動生成 - 初回作成後に自動で提示される 3 つの高価値な次の iteration direction @@ -71,7 +71,7 @@ flowchart LR 1. skill 化したい workflow、prompt 集合、または反復タスクを説明します。 2. まず短いが人間味のある intent dialogue で、実際の job、outputs、boundary、constraints、重視する品質基準を明確にします。 -3. 次に短い reference scan を行い、GitHub の高評価プロジェクト、公式ドキュメント、世界トップ級の公開対象を優先し、必要ならユーザーの参考例も加えます。ローカル資産は適合確認と privacy 調整だけに使います。 +3. まず `quickstart` に GitHub benchmark scan を走らせ、公開リポジトリ上位 3 件から借りるべきパターンを抽出します。その後で必要ならユーザーの参考例を加えます。ローカル資産は適合確認と privacy 調整だけに使います。 4. archetype-aware な `quickstart` か完全な authoring flow を使い、scaffold、production、library、governed のいずれかでパッケージを生成または改善します。 5. 新しく作成した skill には `reports/intent-dialogue.md`、`reports/skill-overview.html`、`reports/review-viewer.html`、`reports/reference-scan.md`、`reports/iteration-directions.md` が付きます。さらに feedback log と baseline compare を使えば、毎回フル promotion flow を回さずに短い改善ループを回せます。 diff --git a/docs/README.ru-RU.md b/docs/README.ru-RU.md index 3ea30a9b..e9fab5ad 100644 --- a/docs/README.ru-RU.md +++ b/docs/README.ru-RU.md @@ -10,7 +10,7 @@ - компактным `SKILL.md` - необязательными references, scripts и evals - более человечным коротким intent dialogue перед глубокой разработкой, чтобы сначала понять реальную работу, ожидаемый результат, границы и важные для пользователя стандарты -- контролируемым benchmark/reference scan до глубокого authoring, где приоритет получают сильные GitHub-проекты, официальная документация и публичные эталонные системы мирового уровня +- автоматическим GitHub benchmark scan до глубокого authoring: сначала он подбирает три сильных публичных репозитория и извлекает patterns, которые стоит заимствовать или избегать - явным запросом пользовательских референсов, если они есть, чтобы перенимать паттерны, структуру и стандарты качества, а не копировать формулировки или приватный материал - автоматически создаваемым минималистичным HTML-обзором на белом фоне для каждого нового skill - тремя наиболее ценными направлениями следующей итерации после первого создания @@ -71,7 +71,7 @@ flowchart LR 1. Опишите workflow, набор prompts или повторяющуюся задачу, которую хотите превратить в skill. 2. Сначала проведите короткий, но более человечный intent dialogue, чтобы уточнить реальную job-to-be-done, outputs, exclusions, constraints и те стандарты качества, которые для вас важны. -3. Затем выполните короткий reference scan, где основными источниками будут сильные GitHub-проекты, официальные документы и публичные эталоны; при необходимости система также спросит о ваших собственных референсах. Локальные файлы используются только для адаптации, приватности и совместимости. +3. Сначала позвольте `quickstart` выполнить GitHub benchmark scan: он найдёт три сильных публичных репозитория и выделит полезные patterns. После этого при необходимости добавьте собственные референсы. Локальные файлы используются только для адаптации, приватности и совместимости. 4. Используйте archetype-aware `quickstart` или полный authoring flow, чтобы сгенерировать или улучшить пакет в режиме scaffold, production, library или governed. 5. Каждый новый skill также получает `reports/intent-dialogue.md`, `reports/skill-overview.html`, `reports/review-viewer.html`, `reports/reference-scan.md` и `reports/iteration-directions.md`. После этого feedback log и baseline compare позволяют запускать короткий цикл улучшений без полного promotion flow. diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 05025f52..6248edf1 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -10,7 +10,7 @@ - 精简的 `SKILL.md` - 可选的 references、scripts 和 evals - 深度起草前先做一轮更有人味的意图对话，先理解真实任务、输出物、边界和用户在意的标准 -- 深度起草前先做一轮受控的 benchmark/reference scan，以 GitHub 高星项目、官方文档和世界级公开对象为优先参考源 +- 深度起草前先做一轮自动 GitHub benchmark scan，先拉取 3 个高星公开项目，提炼 borrow / avoid 模式，再结合用户参考与本地约束 - 会主动询问用户是否有希望借鉴的参考对象，只学习其中的模式抽象、结构和标准，不复制原文或私密内容 - 新建 skill 时自动生成一份极简白底 HTML 可视化说明 - 首次建包后会自动给出 3 个最有价值的下一步迭代方向 @@ -71,7 +71,7 @@ flowchart LR 1. 先描述你想沉淀成 skill 的 workflow、prompt 集合或重复任务。 2. 先做一轮简短但更有人味的意图对话，把真实任务、输出物、边界、约束和你在意的质量标准说清楚。 -3. 再做一轮参考扫描，优先参考 GitHub 高星项目、官方文档和世界级公开对象，同时也会问你有没有想借鉴的参考；本地文件只做适配和隐私校准。 +3. 先让 `quickstart` 自动跑一轮 GitHub benchmark scan，读取 3 个高质量公开项目并提炼可借鉴模式；之后再结合你自己的参考对象。本地文件只做适配和隐私校准。 4. 使用 archetype-aware 的 `quickstart` 或完整作者流，在 scaffold、production、library 或 governed 模式下生成或改进 skill 包。 5. 新建 skill 后，会默认附带 `reports/intent-dialogue.md`、`reports/skill-overview.html`、`reports/review-viewer.html`、`reports/reference-scan.md` 和 `reports/iteration-directions.md`；后续还可以通过 feedback log 和 baseline compare 快速收集意见、查看增量收益，而不必每次都走完整 promotion 流程。 diff --git a/reports/context_budget.json b/reports/context_budget.json index 8f5c8f84..e408ca1c 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 824, - "other_text_tokens": 266035, + "other_text_tokens": 271901, "estimated_initial_load_tokens": 1000, - "estimated_total_text_tokens": 266859, - "relevant_file_count": 161, + "estimated_total_text_tokens": 272725, + "relevant_file_count": 165, "unused_resource_dirs": [], "quality_signal_points": 140, "quality_density": 140.0 diff --git a/scripts/github_benchmark_scan.py b/scripts/github_benchmark_scan.py new file mode 100644 index 00000000..da9dcd4e --- /dev/null +++ b/scripts/github_benchmark_scan.py @@ -0,0 +1,400 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +from pathlib import Path +from typing import Any +from urllib.parse import quote +from urllib.request import Request, urlopen + + +API_ROOT = "https://api.github.com" +API_VERSION = "2022-11-28" +USER_AGENT = "yao-meta-skill/1.0" +DEFAULT_TOP_N = 3 +DEFAULT_PER_PAGE = 8 +MIN_STARS = 50 + +STOPWORDS = { + "the", + "and", + "for", + "with", + "into", + "from", + "this", + "that", + "your", + "will", + "make", + "turn", + "build", + "create", + "reusable", + "skill", + "package", + "agent", + "workflow", + "prompt", + "notes", + "docs", + "internal", + "team", + "real", + "output", + "outputs", + "input", + "inputs", +} + +PATTERN_RULES = [ + { + "category": "method", + "keywords": ["workflow", "process", "pipeline", "orchestrat", "runbook", "playbook"], + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + }, + { + "category": "execution", + "keywords": ["cli", "command", "shell", "script", "automation", "toolchain"], + "borrow": "Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not copy repo-specific commands or environment assumptions verbatim.", + }, + { + "category": "evaluation", + "keywords": ["test", "benchmark", "eval", "assert", "validation", "quality"], + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + }, + { + "category": "structure", + "keywords": ["template", "example", "guide", "docs", "reference", "starter"], + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability.", + }, + { + "category": "portability", + "keywords": ["adapter", "portable", "plugin", "integration", "export", "compatibility"], + "borrow": "Borrow how it keeps core semantics stable across environments or integrations.", + "avoid": "Do not inherit platform lock-in or vendor-specific branching unless truly required.", + }, + { + "category": "governance", + "keywords": ["govern", "policy", "security", "audit", "compliance", "review"], + "borrow": "Borrow the explicit review, safety, or operational trust boundaries.", + "avoid": "Do not import compliance-heavy process where the new skill does not need it.", + }, +] + + +def github_headers() -> dict[str, str]: + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": API_VERSION, + "User-Agent": USER_AGENT, + } + token = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + return headers + + +def api_get(url: str) -> dict[str, Any]: + request = Request(url, headers=github_headers()) + with urlopen(request, timeout=15) as response: # nosec B310 - GitHub API fetch + data = response.read().decode("utf-8") + return json.loads(data) + + +def load_fixture_bundle(fixture_dir: Path | None) -> dict[str, Any] | None: + if not fixture_dir: + return None + bundle_path = fixture_dir / "bundle.json" + if not bundle_path.exists(): + raise FileNotFoundError(f"Missing fixture bundle: {bundle_path}") + return json.loads(bundle_path.read_text(encoding="utf-8")) + + +def build_query(seed_text: str) -> str: + words = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", seed_text.lower()) + keywords = [] + seen = set() + for word in words: + if word in STOPWORDS or word in seen: + continue + seen.add(word) + keywords.append(word) + if len(keywords) == 4: + break + if keywords: + return " ".join(keywords) + compact = re.sub(r"\s+", " ", seed_text.strip()) + return compact[:64] or "agent workflow automation" + + +def search_repositories(query: str, top_n: int, fixture_bundle: dict[str, Any] | None = None) -> list[dict[str, Any]]: + if fixture_bundle is not None: + items = fixture_bundle.get("search_items", []) + return items[:top_n] + + github_query = f"{query} fork:false archived:false stars:>={MIN_STARS}" + url = ( + f"{API_ROOT}/search/repositories?q={quote(github_query)}" + f"&sort=stars&order=desc&per_page={DEFAULT_PER_PAGE}" + ) + payload = api_get(url) + items = payload.get("items", []) + filtered = [item for item in items if not item.get("fork") and not item.get("archived")] + return filtered[:top_n] + + +def fetch_readme(full_name: str, fixture_bundle: dict[str, Any] | None = None) -> str: + if fixture_bundle is not None: + return fixture_bundle.get("readmes", {}).get(full_name, "") + + url = f"{API_ROOT}/repos/{full_name}/readme" + request = Request(url, headers={**github_headers(), "Accept": "application/vnd.github.raw+json"}) + with urlopen(request, timeout=15) as response: # nosec B310 - GitHub API fetch + return response.read().decode("utf-8", errors="replace") + + +def detect_patterns(text: str) -> list[dict[str, str]]: + lowered = text.lower() + hits = [] + for rule in PATTERN_RULES: + matched = [token for token in rule["keywords"] if token in lowered] + if matched: + hits.append( + { + "category": rule["category"], + "matched": ", ".join(matched[:4]), + "borrow": rule["borrow"], + "avoid": rule["avoid"], + } + ) + return hits[:4] + + +def repo_summary(repo: dict[str, Any], readme_text: str) -> dict[str, Any]: + combined = " ".join( + filter( + None, + [ + repo.get("name", ""), + repo.get("description", "") or "", + " ".join(repo.get("topics") or []), + readme_text[:4000], + ], + ) + ) + patterns = detect_patterns(combined) + if not patterns: + patterns = [ + { + "category": "general", + "matched": "high-level fit", + "borrow": "Borrow only the parts that make the new skill clearer, safer, or easier to reuse.", + "avoid": "Do not import source-specific scope, branding, or excess weight.", + } + ] + + borrow = [item["borrow"] for item in patterns[:3]] + avoid = [item["avoid"] for item in patterns[:3]] + return { + "full_name": repo.get("full_name"), + "html_url": repo.get("html_url"), + "description": repo.get("description"), + "stars": repo.get("stargazers_count", 0), + "topics": repo.get("topics") or [], + "primary_language": repo.get("language"), + "patterns": patterns, + "borrow": borrow, + "avoid": avoid, + "readme_excerpt": readme_text[:1200].strip(), + } + + +def cross_repo_insights(repos: list[dict[str, Any]]) -> dict[str, list[str]]: + borrow = [] + avoid = [] + seen_borrow = set() + seen_avoid = set() + for repo in repos: + for item in repo.get("borrow", []): + if item not in seen_borrow: + seen_borrow.add(item) + borrow.append(item) + for item in repo.get("avoid", []): + if item not in seen_avoid: + seen_avoid.add(item) + avoid.append(item) + borrow.append("Ask the user which of these patterns feels most worth borrowing before freezing the package shape.") + return {"borrow": borrow[:6], "avoid": avoid[:6]} + + +def repo_to_external_reference(repo: dict[str, Any]) -> dict[str, str]: + primary = repo.get("patterns", [{}])[0] + return { + "name": repo.get("full_name") or "Unnamed repo", + "category": primary.get("category", "general"), + "borrow": repo.get("borrow", ["Borrow the strongest reusable pattern."])[0], + "avoid": repo.get("avoid", ["Do not copy source-specific language or excess scope."])[0], + "source": "external", + } + + +def render_markdown(payload: dict[str, Any]) -> str: + lines = [ + "# GitHub Benchmark Scan", + "", + f"- Query: `{payload['query']}`", + f"- Source: `{payload['source']}`", + f"- Top repositories: `{len(payload['repositories'])}`", + "", + "## Suggested Next Step", + "", + "Review the three benchmark objects below, then decide whether any of their patterns are worth borrowing into the new skill.", + "", + ] + + if not payload["repositories"]: + lines.extend( + [ + "No benchmark repositories were collected.", + "", + "Possible reasons:", + "- the query was too weak or too local", + "- the network or API rate limit blocked the scan", + "- this skill idea may need a manual benchmark query", + ] + ) + return "\n".join(lines).strip() + "\n" + + lines.extend(["## Top 3 Benchmark Repositories", ""]) + for repo in payload["repositories"]: + lines.extend( + [ + f"### {repo['full_name']}", + f"- URL: {repo['html_url']}", + f"- Stars: `{repo['stars']}`", + f"- Description: {repo.get('description') or 'No public description provided.'}", + f"- Topics: {', '.join(repo.get('topics') or []) or 'None'}", + "", + "#### Patterns worth studying", + "", + ] + ) + for pattern in repo["patterns"]: + lines.append( + f"- **{pattern['category']}** (`{pattern['matched']}`): {pattern['borrow']}" + ) + lines.extend(["", "#### Borrow", ""]) + for item in repo["borrow"]: + lines.append(f"- {item}") + lines.extend(["", "#### Avoid", ""]) + for item in repo["avoid"]: + lines.append(f"- {item}") + lines.extend(["", "#### README glimpse", "", repo["readme_excerpt"] or "No README excerpt collected.", ""]) + + lines.extend(["## Cross-Repo Borrow Recommendations", ""]) + for item in payload["cross_repo"]["borrow"]: + lines.append(f"- {item}") + + lines.extend(["", "## Cross-Repo Avoid Recommendations", ""]) + for item in payload["cross_repo"]["avoid"]: + lines.append(f"- {item}") + + lines.extend( + [ + "", + "## Borrow Prompt", + "", + payload["borrow_prompt"], + "", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def run_github_benchmark_scan( + skill_dir: Path, + query: str, + top_n: int = DEFAULT_TOP_N, + fixture_dir: Path | None = None, + output_md: Path | None = None, + output_json: Path | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + output_md = output_md or reports_dir / "github-benchmark-scan.md" + output_json = output_json or reports_dir / "github-benchmark-scan.json" + fixture_bundle = load_fixture_bundle(fixture_dir) + + warnings = [] + try: + repos = search_repositories(query, top_n=top_n, fixture_bundle=fixture_bundle) + repo_summaries = [] + for repo in repos: + readme_text = fetch_readme(repo["full_name"], fixture_bundle=fixture_bundle) + repo_summaries.append(repo_summary(repo, readme_text)) + cross_repo = cross_repo_insights(repo_summaries) + payload = { + "ok": True, + "query": query, + "source": "fixture" if fixture_bundle is not None else "github-api", + "repositories": repo_summaries, + "cross_repo": cross_repo, + "external_references": [repo_to_external_reference(repo) for repo in repo_summaries], + "borrow_prompt": "I found three public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability?", + "warnings": warnings, + } + except Exception as exc: # pragma: no cover - network failures are environment-dependent + payload = { + "ok": False, + "query": query, + "source": "fixture" if fixture_bundle is not None else "github-api", + "repositories": [], + "cross_repo": { + "borrow": [], + "avoid": [], + }, + "external_references": [], + "borrow_prompt": "No benchmark suggestions were collected yet. You can retry with a stronger query or add manual references.", + "warnings": [f"GitHub benchmark scan failed: {exc}"], + } + + output_md.write_text(render_markdown(payload), encoding="utf-8") + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + payload["artifacts"] = { + "markdown": str(output_md), + "json": str(output_json), + } + payload["skill_dir"] = str(skill_dir) + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Search GitHub for top benchmark repositories and extract reusable patterns.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--query", required=True) + parser.add_argument("--top-n", type=int, default=DEFAULT_TOP_N) + parser.add_argument("--fixture-dir") + parser.add_argument("--output-md") + parser.add_argument("--output-json") + args = parser.parse_args() + + result = run_github_benchmark_scan( + Path(args.skill_dir), + query=args.query, + top_n=args.top_n, + fixture_dir=Path(args.fixture_dir).resolve() if args.fixture_dir else None, + output_md=Path(args.output_md).resolve() if args.output_md else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_skill.py b/scripts/init_skill.py index ad4ff6ab..1f5b6342 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -4,6 +4,7 @@ import json from datetime import date from pathlib import Path +from github_benchmark_scan import run_github_benchmark_scan from render_intent_dialogue import render_intent_dialogue from render_iteration_directions import render_iteration_directions from render_reference_scan import parse_reference, render_reference_scan @@ -50,6 +51,7 @@ README_TEMPLATE = """# {title} - `agents/interface.yaml`: portable interface metadata - `manifest.json`: lifecycle and packaging metadata - `reports/intent-dialogue.md`: front-loaded discovery questions for better boundary design and clearer human alignment +- `reports/github-benchmark-scan.md`: top public benchmark repositories, extracted patterns, and borrow or avoid notes - `reports/reference-scan.md`: benchmark notes from public references, user references, and local constraints - `reports/skill-overview.html`: visual overview report - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps @@ -139,6 +141,24 @@ def build_manifest(name: str, mode: str, archetype: str) -> dict: } +def dedupe_references(references: list[dict]) -> list[dict]: + seen = set() + deduped = [] + for reference in references: + key = ( + reference.get("source"), + reference.get("name"), + reference.get("category"), + reference.get("borrow"), + reference.get("avoid"), + ) + if key in seen: + continue + seen.add(key) + deduped.append(reference) + return deduped + + def initialize_skill( name: str, description: str, @@ -149,6 +169,9 @@ def initialize_skill( external_references: list[dict] | None = None, user_references: list[dict] | None = None, local_constraints: list[dict] | None = None, + github_query: str | None = None, + github_top_n: int = 3, + github_fixture_dir: str | None = None, ) -> dict: title = title or name.replace("-", " ").title() root = Path(output_dir).resolve() / name @@ -179,28 +202,46 @@ def initialize_skill( ) overview = render_skill_overview(root) intent_dialogue = render_intent_dialogue(root) - reference_scan = render_reference_scan(root, [*(external_references or []), *(user_references or []), *(local_constraints or [])]) + benchmark_scan = None + combined_external_references = [*(external_references or [])] + if github_query: + benchmark_scan = run_github_benchmark_scan( + root, + query=github_query, + top_n=github_top_n, + fixture_dir=Path(github_fixture_dir).resolve() if github_fixture_dir else None, + ) + combined_external_references = [*benchmark_scan.get("external_references", []), *combined_external_references] + reference_scan = render_reference_scan( + root, + dedupe_references([*combined_external_references, *(user_references or []), *(local_constraints or [])]), + ) iteration_directions = render_iteration_directions(root) review_viewer = render_review_viewer(root) + artifacts = { + "readme": str(root / "README.md"), + "manifest": str(root / "manifest.json"), + "skill_overview_html": overview["artifacts"]["html"], + "skill_overview_json": overview["artifacts"]["json"], + "intent_dialogue_md": intent_dialogue["artifacts"]["markdown"], + "intent_dialogue_json": intent_dialogue["artifacts"]["json"], + "reference_scan_md": reference_scan["artifacts"]["markdown"], + "reference_scan_json": reference_scan["artifacts"]["json"], + "iteration_directions_md": iteration_directions["artifacts"]["markdown"], + "iteration_directions_json": iteration_directions["artifacts"]["json"], + "review_viewer_html": review_viewer["artifacts"]["html"], + "review_viewer_json": review_viewer["artifacts"]["json"], + } + if benchmark_scan is not None: + artifacts["github_benchmark_scan_md"] = benchmark_scan["artifacts"]["markdown"] + artifacts["github_benchmark_scan_json"] = benchmark_scan["artifacts"]["json"] return { "ok": True, "root": str(root), "mode": mode, "archetype": archetype, - "artifacts": { - "readme": str(root / "README.md"), - "manifest": str(root / "manifest.json"), - "skill_overview_html": overview["artifacts"]["html"], - "skill_overview_json": overview["artifacts"]["json"], - "intent_dialogue_md": intent_dialogue["artifacts"]["markdown"], - "intent_dialogue_json": intent_dialogue["artifacts"]["json"], - "reference_scan_md": reference_scan["artifacts"]["markdown"], - "reference_scan_json": reference_scan["artifacts"]["json"], - "iteration_directions_md": iteration_directions["artifacts"]["markdown"], - "iteration_directions_json": iteration_directions["artifacts"]["json"], - "review_viewer_html": review_viewer["artifacts"]["html"], - "review_viewer_json": review_viewer["artifacts"]["json"], - }, + "github_benchmark_scan": benchmark_scan, + "artifacts": artifacts, } @@ -215,6 +256,9 @@ def main() -> None: parser.add_argument("--external-reference", action="append", default=[]) parser.add_argument("--user-reference", action="append", default=[]) parser.add_argument("--local-constraint", action="append", default=[]) + parser.add_argument("--github-query") + parser.add_argument("--github-top-n", type=int, default=3) + parser.add_argument("--github-fixture-dir") args = parser.parse_args() result = initialize_skill( args.name, @@ -226,6 +270,9 @@ def main() -> None: external_references=[parse_reference(item, "external") for item in args.external_reference], user_references=[parse_reference(item, "user") for item in args.user_reference], local_constraints=[parse_reference(item, "local") for item in args.local_constraint], + github_query=args.github_query, + github_top_n=args.github_top_n, + github_fixture_dir=args.github_fixture_dir, ) print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/scripts/yao.py b/scripts/yao.py index 4a999a89..009919b2 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -5,6 +5,8 @@ import subprocess import sys from pathlib import Path +from github_benchmark_scan import build_query + ROOT = Path(__file__).resolve().parent.parent SCRIPTS = ROOT / "scripts" @@ -190,6 +192,11 @@ def command_init(args: argparse.Namespace) -> int: cmd.extend(["--user-reference", reference]) for constraint in args.local_constraint: cmd.extend(["--local-constraint", constraint]) + if args.github_query: + cmd.extend(["--github-query", args.github_query]) + cmd.extend(["--github-top-n", str(args.github_top_n)]) + if args.github_fixture_dir: + cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) result = run_script("init_skill.py", cmd) print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) return 0 if result["ok"] else 2 @@ -197,7 +204,7 @@ def command_init(args: argparse.Namespace) -> int: def command_quickstart(args: argparse.Namespace) -> int: sys.stderr.write("Let's shape this skill around the real work, the outcome, and the standards you care about before we add structure.\n") - sys.stderr.write("If you already have references you admire, bring them in. We'll learn the pattern, not copy the source.\n") + sys.stderr.write("I will also scan a few strong public GitHub references first, so we can borrow the right patterns without copying the source.\n") name = args.name or prompt_with_default("Skill name", "my-skill") job = args.job or prompt_with_default("What repeated work should this skill quietly take off your hands", "Turn a repeated workflow into a reusable skill.") primary_output = args.primary_output or prompt_with_default("What should it reliably hand back", "A reusable skill package.") @@ -209,14 +216,16 @@ def command_quickstart(args: argparse.Namespace) -> int: mode = args.mode or prompt_with_default("Mode (scaffold/production/library/governed)", default_mode) mode = mode if mode in ARCHETYPE_MODE.values() else default_mode user_references = args.user_reference or prompt_optional_entries( - "Reference examples you want it to learn from (repo, product, page, workflow; comma-separated)" + "Your own reference examples to learn from as pattern hints (repo, product, page, workflow; comma-separated)" ) external_references = args.external_reference or prompt_optional_entries( - "Public benchmark objects to scan first (high-star GitHub repo, official doc, product; comma-separated)" + "Additional public benchmark objects you already know you want to borrow from (comma-separated)" ) local_constraints = args.local_constraint or prompt_optional_entries( "Local constraints that must still be respected (privacy, naming, compatibility; comma-separated)" ) + github_query = args.github_query or build_query(" ".join(filter(None, [job, primary_output, description]))) + sys.stderr.write(f"GitHub benchmark query: {github_query}\n") title = args.title or name.replace("-", " ").title() guidance = archetype_guidance(archetype) cmd = [ @@ -231,7 +240,13 @@ def command_quickstart(args: argparse.Namespace) -> int: mode, "--archetype", archetype, + "--github-query", + github_query, + "--github-top-n", + str(args.github_top_n), ] + if args.github_fixture_dir: + cmd.extend(["--github-fixture-dir", args.github_fixture_dir]) for reference in external_references: cmd.extend(["--external-reference", reference]) for reference in user_references: @@ -240,17 +255,29 @@ def command_quickstart(args: argparse.Namespace) -> int: cmd.extend(["--local-constraint", constraint]) result = run_script("init_skill.py", cmd) payload = result["payload"] if result["payload"] is not None else result + benchmark_scan = payload.get("github_benchmark_scan") or {} + benchmark_repositories = [ + { + "name": repo.get("full_name"), + "stars": repo.get("stars"), + "url": repo.get("html_url"), + } + for repo in benchmark_scan.get("repositories", []) + ] report = { "ok": result["ok"], "root": payload.get("root"), "mode": mode, "archetype": archetype, "references": { + "github_query": github_query, + "benchmark_repositories": benchmark_repositories, "external_benchmarks": external_references, "user_references": user_references, "local_constraints": local_constraints, }, "artifacts": payload.get("artifacts", {}), + "github_benchmark_scan": benchmark_scan, "guidance": { "archetype_reason": archetype_reason, "why_this_mode": ( @@ -262,10 +289,12 @@ def command_quickstart(args: argparse.Namespace) -> int: "focus": guidance["focus"], "next_steps": [ "Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.", - "Open reports/reference-scan.md and decide which external patterns to borrow and which user references set the quality bar.", + "Open reports/github-benchmark-scan.md and review the top three public benchmark repositories plus their borrow and avoid notes.", + "Open reports/reference-scan.md and decide which benchmark patterns to borrow and which user references set the quality bar.", "Open reports/review-viewer.html to explain the package to a first-time reviewer.", "Use reports/iteration-directions.md to choose only one high-value next move before adding more files.", ], + "borrow_prompt": benchmark_scan.get("borrow_prompt"), }, } print(json.dumps(report, ensure_ascii=False, indent=2)) @@ -416,6 +445,20 @@ def command_reference_scan(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_github_benchmark_scan(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir, "--query", args.query, "--top-n", str(args.top_n)] + if args.fixture_dir: + cmd.extend(["--fixture-dir", args.fixture_dir]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + result = run_script("github_benchmark_scan.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_intent_dialogue(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -620,6 +663,9 @@ def build_parser() -> argparse.ArgumentParser: init_cmd.add_argument("--external-reference", action="append", default=[]) init_cmd.add_argument("--user-reference", action="append", default=[]) init_cmd.add_argument("--local-constraint", action="append", default=[]) + init_cmd.add_argument("--github-query") + init_cmd.add_argument("--github-top-n", type=int, default=3) + init_cmd.add_argument("--github-fixture-dir") init_cmd.set_defaults(func=command_init) quickstart_cmd = subparsers.add_parser( @@ -637,6 +683,9 @@ def build_parser() -> argparse.ArgumentParser: quickstart_cmd.add_argument("--external-reference", action="append", default=[]) quickstart_cmd.add_argument("--user-reference", action="append", default=[]) quickstart_cmd.add_argument("--local-constraint", action="append", default=[]) + quickstart_cmd.add_argument("--github-query") + quickstart_cmd.add_argument("--github-top-n", type=int, default=3) + quickstart_cmd.add_argument("--github-fixture-dir") quickstart_cmd.set_defaults(func=command_quickstart) validate_cmd = subparsers.add_parser("validate", help="Run validate, lint, governance, and resource checks.") @@ -714,6 +763,18 @@ def build_parser() -> argparse.ArgumentParser: reference_scan_cmd.add_argument("--output-json") reference_scan_cmd.set_defaults(func=command_reference_scan) + github_scan_cmd = subparsers.add_parser( + "github-benchmark-scan", + help="Search top public GitHub repositories and extract borrow or avoid patterns for a skill.", + ) + github_scan_cmd.add_argument("skill_dir", nargs="?", default=".") + github_scan_cmd.add_argument("--query", required=True) + github_scan_cmd.add_argument("--top-n", type=int, default=3) + github_scan_cmd.add_argument("--fixture-dir") + github_scan_cmd.add_argument("--output-md") + github_scan_cmd.add_argument("--output-json") + github_scan_cmd.set_defaults(func=command_github_benchmark_scan) + intent_dialogue_cmd = subparsers.add_parser( "intent-dialogue", help="Render a front-loaded intent dialogue guide for a skill package.", diff --git a/tests/fixtures/github_benchmark_scan/bundle.json b/tests/fixtures/github_benchmark_scan/bundle.json new file mode 100644 index 00000000..abbe9b99 --- /dev/null +++ b/tests/fixtures/github_benchmark_scan/bundle.json @@ -0,0 +1,39 @@ +{ + "search_items": [ + { + "full_name": "anthropics/skills", + "html_url": "https://github.com/anthropics/skills", + "description": "A collection of composable skills for agent workflows, evaluation, and reusable operating patterns.", + "stargazers_count": 12800, + "topics": ["skills", "agents", "workflow", "evaluation"], + "language": "Markdown", + "fork": false, + "archived": false + }, + { + "full_name": "openai/skills", + "html_url": "https://github.com/openai/skills", + "description": "Reference skills and patterns for creating portable, reusable agent capabilities.", + "stargazers_count": 9400, + "topics": ["skills", "portable", "agent", "toolchain"], + "language": "Markdown", + "fork": false, + "archived": false + }, + { + "full_name": "microsoft/autogen", + "html_url": "https://github.com/microsoft/autogen", + "description": "Agent orchestration framework with workflow patterns, evaluation, tooling, and portability guidance.", + "stargazers_count": 35500, + "topics": ["agents", "automation", "toolchain", "workflow"], + "language": "Python", + "fork": false, + "archived": false + } + ], + "readmes": { + "anthropics/skills": "# Skills\n\nThis repository documents workflow patterns, evaluation loops, benchmark practices, and review steps for reusable agent skills. It includes examples, references, and quality checks for operating a repeatable process.\n", + "openai/skills": "# Skills\n\nThis project emphasizes portable metadata, compatibility adapters, references, templates, and clear execution entrypoints. It keeps core semantics stable while allowing environment-specific packaging.\n", + "microsoft/autogen": "# AutoGen\n\nThis framework provides toolchain commands, automation flows, orchestration guidance, and validation practices. It highlights scripts, command entrypoints, and reusable execution structures for agent workflows.\n" + } +} diff --git a/tests/verify_github_benchmark_scan.py b/tests/verify_github_benchmark_scan.py new file mode 100644 index 00000000..85b7ef43 --- /dev/null +++ b/tests/verify_github_benchmark_scan.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "github_benchmark_scan.py" +FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_github_benchmark_scan" + if tmp_root.exists(): + subprocess.run(["rm", "-rf", str(tmp_root)], check=True) + skill_dir = tmp_root / "benchmark-scan-demo" + (skill_dir / "reports").mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text( + "---\nname: benchmark-scan-demo\ndescription: Turn rough release notes into a reusable governed skill.\n---\n\n# Benchmark Scan Demo\n", + encoding="utf-8", + ) + + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(skill_dir), + "--query", + "release workflow evaluation portability", + "--fixture-dir", + str(FIXTURE_DIR), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["source"] == "fixture", payload + assert len(payload["repositories"]) == 3, payload + assert len(payload["external_references"]) == 3, payload + assert payload["borrow_prompt"], payload + + markdown = Path(payload["artifacts"]["markdown"]).read_text(encoding="utf-8") + assert "## Top 3 Benchmark Repositories" in markdown, markdown[:400] + assert "anthropics/skills" in markdown, markdown[:800] + assert "Cross-Repo Borrow Recommendations" in markdown, markdown[:1400] + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 9093f574..54a3f437 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -7,6 +7,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent CLI = ROOT / "scripts" / "yao.py" +BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" def run(*args: str, input_text: str | None = None) -> dict: @@ -47,12 +48,16 @@ def main() -> None: "quickstart", "--output-dir", str(tmp_root), + "--github-fixture-dir", + str(BENCHMARK_FIXTURE_DIR), input_text="quickstart-skill\nTurn rough notes into a reusable package.\nA reusable markdown workflow.\nproduction\nproduction\nA top-tier internal workflow product\nhigh-star GitHub repo, official docs\nprivacy and naming\n", ) assert quickstart_result["ok"], quickstart_result quickstart_root = Path(quickstart_result["payload"]["root"]) assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root + assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root assert quickstart_result["payload"]["archetype"] == "production", quickstart_result + assert len(quickstart_result["payload"]["references"]["benchmark_repositories"]) == 3, quickstart_result assert quickstart_result["payload"]["references"]["user_references"] == ["A top-tier internal workflow product"], quickstart_result validate_result = run("validate", str(created)) @@ -81,6 +86,17 @@ def main() -> None: assert reference_scan_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-scan.md"), reference_scan_result assert len(reference_scan_result["payload"]["summary"]["user_references"]) == 1, reference_scan_result + github_benchmark_result = run( + "github-benchmark-scan", + str(created), + "--query", + "workflow evaluation portability", + "--fixture-dir", + str(BENCHMARK_FIXTURE_DIR), + ) + assert github_benchmark_result["ok"], github_benchmark_result + assert len(github_benchmark_result["payload"]["repositories"]) == 3, github_benchmark_result + intent_result = run("intent-dialogue", str(created)) assert intent_result["ok"], intent_result assert intent_result["payload"]["artifacts"]["markdown"].endswith("reports/intent-dialogue.md"), intent_result