feat: add github benchmark scan to quickstart
This commit is contained in:
@@ -11,4 +11,5 @@ tests/tmp_reference_scan/
|
||||
tests/tmp_iteration_directions/
|
||||
tests/tmp_review_viewer/
|
||||
tests/tmp_feedback/
|
||||
tests/tmp_github_benchmark_scan/
|
||||
reports/release_snapshots/
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
PYTHON ?= python3
|
||||
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean
|
||||
|
||||
eval:
|
||||
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
@@ -44,6 +44,9 @@ baseline-compare-check:
|
||||
reference-scan-check:
|
||||
$(PYTHON) tests/verify_reference_scan.py
|
||||
|
||||
github-benchmark-scan-check:
|
||||
$(PYTHON) tests/verify_github_benchmark_scan.py
|
||||
|
||||
iteration-directions-check:
|
||||
$(PYTHON) tests/verify_iteration_directions.py
|
||||
|
||||
@@ -95,7 +98,7 @@ resource-boundary-check:
|
||||
quality-check:
|
||||
$(PYTHON) tests/verify_quality_checks.py
|
||||
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
|
||||
clean:
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
|
||||
@@ -20,7 +20,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
- a lean `SKILL.md`
|
||||
- optional references, scripts, and evals
|
||||
- a front-loaded intent dialogue that starts from the user's real work, desired outcome, and quality bar
|
||||
- a controlled benchmark scan that prefers high-quality public references first and asks whether the user has examples worth learning from
|
||||
- an automatic GitHub benchmark scan that pulls the top three public repositories first, extracts borrow or avoid patterns, and then asks whether the user has references worth learning from
|
||||
- a generated visual HTML overview for each newly initialized skill
|
||||
- a compact HTML review viewer for first-pass human review
|
||||
- three high-value next iteration directions after the first package is created
|
||||
@@ -80,7 +80,7 @@ This is a scenario-oriented benchmark shared with the project. It is most useful
|
||||
|
||||
1. Describe the workflow, prompt set, or repeated task you want to turn into a skill.
|
||||
2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit.
|
||||
3. Run a short reference scan with high-quality public benchmarks first, then ask whether the user has examples worth learning from; use local files only for fit and privacy checks.
|
||||
3. Let `quickstart` run a short GitHub benchmark scan first, study the top three public repositories it retrieves, then ask whether the user has examples worth learning from; use local files only for fit and privacy checks.
|
||||
4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode.
|
||||
5. Review the generated `reports/intent-dialogue.md`, `reports/skill-overview.html`, and `reports/iteration-directions.md` before adding more structure.
|
||||
|
||||
@@ -88,6 +88,7 @@ Or use the unified authoring CLI:
|
||||
|
||||
```bash
|
||||
python3 scripts/yao.py quickstart --output-dir .
|
||||
python3 scripts/yao.py github-benchmark-scan my-skill --query "release workflow portability"
|
||||
python3 scripts/yao.py reference-scan my-skill \
|
||||
--external-reference "World Class Method::method::Borrow a tight evaluation loop.::Do not copy heavy process." \
|
||||
--user-reference "A product or repo I admire::taste::Learn the clarity and operating standard.::Do not copy wording." \
|
||||
|
||||
@@ -10,7 +10,7 @@ Il transforme des workflows bruts, des transcripts, des prompts, des notes et de
|
||||
- un `SKILL.md` léger
|
||||
- des references, scripts et evals optionnels
|
||||
- un dialogue d'intention plus humain avant l'authoring approfondi, pour comprendre le vrai travail, la sortie attendue, les frontières et le niveau d'exigence
|
||||
- un benchmark/reference scan contrôlé avant l'authoring profond, en donnant la priorité aux dépôts GitHub très reconnus, aux documentations officielles et aux références publiques de niveau mondial
|
||||
- un benchmark scan GitHub automatique avant l'authoring profond, qui récupère d'abord trois dépôts publics très reconnus puis extrait les patterns à reprendre ou à éviter
|
||||
- une demande explicite de références fournies par l'utilisateur quand elles existent, afin d'apprendre des modèles, pas de copier le texte ni du contenu privé
|
||||
- un rapport HTML minimaliste en fond blanc généré automatiquement pour chaque nouveau skill
|
||||
- trois directions d'itération à plus forte valeur après la première création
|
||||
@@ -71,7 +71,7 @@ Le tableau ci-dessous est un comparatif orienté scénario. Il aide à choisir l
|
||||
|
||||
1. Décrivez le workflow, l'ensemble de prompts ou la tâche répétée que vous voulez transformer en skill.
|
||||
2. Commencez par un court dialogue d'intention plus humain pour clarifier le vrai travail, les sorties attendues, les exclusions, les contraintes et les standards qui comptent pour vous.
|
||||
3. Lancez ensuite un court reference scan en privilégiant les dépôts GitHub à forte reconnaissance, les documentations officielles et les références publiques de haut niveau ; le système vous demandera aussi si vous avez des exemples à faire étudier. Les fichiers locaux ne servent ensuite qu'à l'ajustement, à la confidentialité et à la compatibilité.
|
||||
3. Laissez d'abord `quickstart` lancer un benchmark scan GitHub pour récupérer trois bons dépôts publics et en extraire les patterns réutilisables ; ajoutez ensuite vos propres références si besoin. Les fichiers locaux ne servent ensuite qu'à l'ajustement, à la confidentialité et à la compatibilité.
|
||||
4. Utilisez le `quickstart` sensible aux archetypes ou le flux complet d'authoring pour générer ou améliorer le paquet en mode scaffold, production, library ou governed.
|
||||
5. Chaque nouveau skill reçoit aussi `reports/intent-dialogue.md`, `reports/skill-overview.html`, `reports/review-viewer.html`, `reports/reference-scan.md` et `reports/iteration-directions.md`. Ensuite, le feedback log et le baseline compare permettent de boucler rapidement sans lancer tout le flux de promotion.
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
- 軽量な `SKILL.md`
|
||||
- 必要に応じた references、scripts、evals
|
||||
- 深い authoring の前に、人間味のある短い intent dialogue を行い、本当の仕事、成果物、境界、重視する基準をつかむ
|
||||
- 深い authoring の前に、GitHub の高評価リポジトリ、公式ドキュメント、世界水準の公開事例を優先する制御された benchmark/reference scan を 1 回行う
|
||||
- 深い authoring の前に、自動 GitHub benchmark scan を 1 回実行し、高評価の公開リポジトリを 3 件取得して borrow / avoid パターンを整理する
|
||||
- ユーザー自身の参考例があれば、それも取り込み、文章ではなくパターン・構造・品質基準だけを学ぶ
|
||||
- 新しい skill ごとに、白背景の簡潔な HTML overview を自動生成
|
||||
- 初回作成後に自動で提示される 3 つの高価値な次の iteration direction
|
||||
@@ -71,7 +71,7 @@ flowchart LR
|
||||
|
||||
1. skill 化したい workflow、prompt 集合、または反復タスクを説明します。
|
||||
2. まず短いが人間味のある intent dialogue で、実際の job、outputs、boundary、constraints、重視する品質基準を明確にします。
|
||||
3. 次に短い reference scan を行い、GitHub の高評価プロジェクト、公式ドキュメント、世界トップ級の公開対象を優先し、必要ならユーザーの参考例も加えます。ローカル資産は適合確認と privacy 調整だけに使います。
|
||||
3. まず `quickstart` に GitHub benchmark scan を走らせ、公開リポジトリ上位 3 件から借りるべきパターンを抽出します。その後で必要ならユーザーの参考例を加えます。ローカル資産は適合確認と privacy 調整だけに使います。
|
||||
4. archetype-aware な `quickstart` か完全な authoring flow を使い、scaffold、production、library、governed のいずれかでパッケージを生成または改善します。
|
||||
5. 新しく作成した skill には `reports/intent-dialogue.md`、`reports/skill-overview.html`、`reports/review-viewer.html`、`reports/reference-scan.md`、`reports/iteration-directions.md` が付きます。さらに feedback log と baseline compare を使えば、毎回フル promotion flow を回さずに短い改善ループを回せます。
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
- компактным `SKILL.md`
|
||||
- необязательными references, scripts и evals
|
||||
- более человечным коротким intent dialogue перед глубокой разработкой, чтобы сначала понять реальную работу, ожидаемый результат, границы и важные для пользователя стандарты
|
||||
- контролируемым benchmark/reference scan до глубокого authoring, где приоритет получают сильные GitHub-проекты, официальная документация и публичные эталонные системы мирового уровня
|
||||
- автоматическим GitHub benchmark scan до глубокого authoring: сначала он подбирает три сильных публичных репозитория и извлекает patterns, которые стоит заимствовать или избегать
|
||||
- явным запросом пользовательских референсов, если они есть, чтобы перенимать паттерны, структуру и стандарты качества, а не копировать формулировки или приватный материал
|
||||
- автоматически создаваемым минималистичным HTML-обзором на белом фоне для каждого нового skill
|
||||
- тремя наиболее ценными направлениями следующей итерации после первого создания
|
||||
@@ -71,7 +71,7 @@ flowchart LR
|
||||
|
||||
1. Опишите workflow, набор prompts или повторяющуюся задачу, которую хотите превратить в skill.
|
||||
2. Сначала проведите короткий, но более человечный intent dialogue, чтобы уточнить реальную job-to-be-done, outputs, exclusions, constraints и те стандарты качества, которые для вас важны.
|
||||
3. Затем выполните короткий reference scan, где основными источниками будут сильные GitHub-проекты, официальные документы и публичные эталоны; при необходимости система также спросит о ваших собственных референсах. Локальные файлы используются только для адаптации, приватности и совместимости.
|
||||
3. Сначала позвольте `quickstart` выполнить GitHub benchmark scan: он найдёт три сильных публичных репозитория и выделит полезные patterns. После этого при необходимости добавьте собственные референсы. Локальные файлы используются только для адаптации, приватности и совместимости.
|
||||
4. Используйте archetype-aware `quickstart` или полный authoring flow, чтобы сгенерировать или улучшить пакет в режиме scaffold, production, library или governed.
|
||||
5. Каждый новый skill также получает `reports/intent-dialogue.md`, `reports/skill-overview.html`, `reports/review-viewer.html`, `reports/reference-scan.md` и `reports/iteration-directions.md`. После этого feedback log и baseline compare позволяют запускать короткий цикл улучшений без полного promotion flow.
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
- 精简的 `SKILL.md`
|
||||
- 可选的 references、scripts 和 evals
|
||||
- 深度起草前先做一轮更有人味的意图对话,先理解真实任务、输出物、边界和用户在意的标准
|
||||
- 深度起草前先做一轮受控的 benchmark/reference scan,以 GitHub 高星项目、官方文档和世界级公开对象为优先参考源
|
||||
- 深度起草前先做一轮自动 GitHub benchmark scan,先拉取 3 个高星公开项目,提炼 borrow / avoid 模式,再结合用户参考与本地约束
|
||||
- 会主动询问用户是否有希望借鉴的参考对象,只学习其中的模式抽象、结构和标准,不复制原文或私密内容
|
||||
- 新建 skill 时自动生成一份极简白底 HTML 可视化说明
|
||||
- 首次建包后会自动给出 3 个最有价值的下一步迭代方向
|
||||
@@ -71,7 +71,7 @@ flowchart LR
|
||||
|
||||
1. 先描述你想沉淀成 skill 的 workflow、prompt 集合或重复任务。
|
||||
2. 先做一轮简短但更有人味的意图对话,把真实任务、输出物、边界、约束和你在意的质量标准说清楚。
|
||||
3. 再做一轮参考扫描,优先参考 GitHub 高星项目、官方文档和世界级公开对象,同时也会问你有没有想借鉴的参考;本地文件只做适配和隐私校准。
|
||||
3. 先让 `quickstart` 自动跑一轮 GitHub benchmark scan,读取 3 个高质量公开项目并提炼可借鉴模式;之后再结合你自己的参考对象。本地文件只做适配和隐私校准。
|
||||
4. 使用 archetype-aware 的 `quickstart` 或完整作者流,在 scaffold、production、library 或 governed 模式下生成或改进 skill 包。
|
||||
5. 新建 skill 后,会默认附带 `reports/intent-dialogue.md`、`reports/skill-overview.html`、`reports/review-viewer.html`、`reports/reference-scan.md` 和 `reports/iteration-directions.md`;后续还可以通过 feedback log 和 baseline compare 快速收集意见、查看增量收益,而不必每次都走完整 promotion 流程。
|
||||
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 824,
|
||||
"other_text_tokens": 266035,
|
||||
"other_text_tokens": 271901,
|
||||
"estimated_initial_load_tokens": 1000,
|
||||
"estimated_total_text_tokens": 266859,
|
||||
"relevant_file_count": 161,
|
||||
"estimated_total_text_tokens": 272725,
|
||||
"relevant_file_count": 165,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 140,
|
||||
"quality_density": 140.0
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
API_ROOT = "https://api.github.com"
|
||||
API_VERSION = "2022-11-28"
|
||||
USER_AGENT = "yao-meta-skill/1.0"
|
||||
DEFAULT_TOP_N = 3
|
||||
DEFAULT_PER_PAGE = 8
|
||||
MIN_STARS = 50
|
||||
|
||||
STOPWORDS = {
|
||||
"the",
|
||||
"and",
|
||||
"for",
|
||||
"with",
|
||||
"into",
|
||||
"from",
|
||||
"this",
|
||||
"that",
|
||||
"your",
|
||||
"will",
|
||||
"make",
|
||||
"turn",
|
||||
"build",
|
||||
"create",
|
||||
"reusable",
|
||||
"skill",
|
||||
"package",
|
||||
"agent",
|
||||
"workflow",
|
||||
"prompt",
|
||||
"notes",
|
||||
"docs",
|
||||
"internal",
|
||||
"team",
|
||||
"real",
|
||||
"output",
|
||||
"outputs",
|
||||
"input",
|
||||
"inputs",
|
||||
}
|
||||
|
||||
PATTERN_RULES = [
|
||||
{
|
||||
"category": "method",
|
||||
"keywords": ["workflow", "process", "pipeline", "orchestrat", "runbook", "playbook"],
|
||||
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.",
|
||||
"avoid": "Do not import process overhead that only exists for that project's scale.",
|
||||
},
|
||||
{
|
||||
"category": "execution",
|
||||
"keywords": ["cli", "command", "shell", "script", "automation", "toolchain"],
|
||||
"borrow": "Borrow the clear execution entrypoints and command structure.",
|
||||
"avoid": "Do not copy repo-specific commands or environment assumptions verbatim.",
|
||||
},
|
||||
{
|
||||
"category": "evaluation",
|
||||
"keywords": ["test", "benchmark", "eval", "assert", "validation", "quality"],
|
||||
"borrow": "Borrow explicit validation and quality gates that make iteration safer.",
|
||||
"avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.",
|
||||
},
|
||||
{
|
||||
"category": "structure",
|
||||
"keywords": ["template", "example", "guide", "docs", "reference", "starter"],
|
||||
"borrow": "Borrow the way it separates explanation, examples, and reusable structure.",
|
||||
"avoid": "Do not mirror documentation bulk that adds context cost without improving reliability.",
|
||||
},
|
||||
{
|
||||
"category": "portability",
|
||||
"keywords": ["adapter", "portable", "plugin", "integration", "export", "compatibility"],
|
||||
"borrow": "Borrow how it keeps core semantics stable across environments or integrations.",
|
||||
"avoid": "Do not inherit platform lock-in or vendor-specific branching unless truly required.",
|
||||
},
|
||||
{
|
||||
"category": "governance",
|
||||
"keywords": ["govern", "policy", "security", "audit", "compliance", "review"],
|
||||
"borrow": "Borrow the explicit review, safety, or operational trust boundaries.",
|
||||
"avoid": "Do not import compliance-heavy process where the new skill does not need it.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def github_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": API_VERSION,
|
||||
"User-Agent": USER_AGENT,
|
||||
}
|
||||
token = os.getenv("GITHUB_TOKEN") or os.getenv("GH_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def api_get(url: str) -> dict[str, Any]:
|
||||
request = Request(url, headers=github_headers())
|
||||
with urlopen(request, timeout=15) as response: # nosec B310 - GitHub API fetch
|
||||
data = response.read().decode("utf-8")
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
def load_fixture_bundle(fixture_dir: Path | None) -> dict[str, Any] | None:
|
||||
if not fixture_dir:
|
||||
return None
|
||||
bundle_path = fixture_dir / "bundle.json"
|
||||
if not bundle_path.exists():
|
||||
raise FileNotFoundError(f"Missing fixture bundle: {bundle_path}")
|
||||
return json.loads(bundle_path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def build_query(seed_text: str) -> str:
|
||||
words = re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", seed_text.lower())
|
||||
keywords = []
|
||||
seen = set()
|
||||
for word in words:
|
||||
if word in STOPWORDS or word in seen:
|
||||
continue
|
||||
seen.add(word)
|
||||
keywords.append(word)
|
||||
if len(keywords) == 4:
|
||||
break
|
||||
if keywords:
|
||||
return " ".join(keywords)
|
||||
compact = re.sub(r"\s+", " ", seed_text.strip())
|
||||
return compact[:64] or "agent workflow automation"
|
||||
|
||||
|
||||
def search_repositories(query: str, top_n: int, fixture_bundle: dict[str, Any] | None = None) -> list[dict[str, Any]]:
|
||||
if fixture_bundle is not None:
|
||||
items = fixture_bundle.get("search_items", [])
|
||||
return items[:top_n]
|
||||
|
||||
github_query = f"{query} fork:false archived:false stars:>={MIN_STARS}"
|
||||
url = (
|
||||
f"{API_ROOT}/search/repositories?q={quote(github_query)}"
|
||||
f"&sort=stars&order=desc&per_page={DEFAULT_PER_PAGE}"
|
||||
)
|
||||
payload = api_get(url)
|
||||
items = payload.get("items", [])
|
||||
filtered = [item for item in items if not item.get("fork") and not item.get("archived")]
|
||||
return filtered[:top_n]
|
||||
|
||||
|
||||
def fetch_readme(full_name: str, fixture_bundle: dict[str, Any] | None = None) -> str:
|
||||
if fixture_bundle is not None:
|
||||
return fixture_bundle.get("readmes", {}).get(full_name, "")
|
||||
|
||||
url = f"{API_ROOT}/repos/{full_name}/readme"
|
||||
request = Request(url, headers={**github_headers(), "Accept": "application/vnd.github.raw+json"})
|
||||
with urlopen(request, timeout=15) as response: # nosec B310 - GitHub API fetch
|
||||
return response.read().decode("utf-8", errors="replace")
|
||||
|
||||
|
||||
def detect_patterns(text: str) -> list[dict[str, str]]:
|
||||
lowered = text.lower()
|
||||
hits = []
|
||||
for rule in PATTERN_RULES:
|
||||
matched = [token for token in rule["keywords"] if token in lowered]
|
||||
if matched:
|
||||
hits.append(
|
||||
{
|
||||
"category": rule["category"],
|
||||
"matched": ", ".join(matched[:4]),
|
||||
"borrow": rule["borrow"],
|
||||
"avoid": rule["avoid"],
|
||||
}
|
||||
)
|
||||
return hits[:4]
|
||||
|
||||
|
||||
def repo_summary(repo: dict[str, Any], readme_text: str) -> dict[str, Any]:
|
||||
combined = " ".join(
|
||||
filter(
|
||||
None,
|
||||
[
|
||||
repo.get("name", ""),
|
||||
repo.get("description", "") or "",
|
||||
" ".join(repo.get("topics") or []),
|
||||
readme_text[:4000],
|
||||
],
|
||||
)
|
||||
)
|
||||
patterns = detect_patterns(combined)
|
||||
if not patterns:
|
||||
patterns = [
|
||||
{
|
||||
"category": "general",
|
||||
"matched": "high-level fit",
|
||||
"borrow": "Borrow only the parts that make the new skill clearer, safer, or easier to reuse.",
|
||||
"avoid": "Do not import source-specific scope, branding, or excess weight.",
|
||||
}
|
||||
]
|
||||
|
||||
borrow = [item["borrow"] for item in patterns[:3]]
|
||||
avoid = [item["avoid"] for item in patterns[:3]]
|
||||
return {
|
||||
"full_name": repo.get("full_name"),
|
||||
"html_url": repo.get("html_url"),
|
||||
"description": repo.get("description"),
|
||||
"stars": repo.get("stargazers_count", 0),
|
||||
"topics": repo.get("topics") or [],
|
||||
"primary_language": repo.get("language"),
|
||||
"patterns": patterns,
|
||||
"borrow": borrow,
|
||||
"avoid": avoid,
|
||||
"readme_excerpt": readme_text[:1200].strip(),
|
||||
}
|
||||
|
||||
|
||||
def cross_repo_insights(repos: list[dict[str, Any]]) -> dict[str, list[str]]:
|
||||
borrow = []
|
||||
avoid = []
|
||||
seen_borrow = set()
|
||||
seen_avoid = set()
|
||||
for repo in repos:
|
||||
for item in repo.get("borrow", []):
|
||||
if item not in seen_borrow:
|
||||
seen_borrow.add(item)
|
||||
borrow.append(item)
|
||||
for item in repo.get("avoid", []):
|
||||
if item not in seen_avoid:
|
||||
seen_avoid.add(item)
|
||||
avoid.append(item)
|
||||
borrow.append("Ask the user which of these patterns feels most worth borrowing before freezing the package shape.")
|
||||
return {"borrow": borrow[:6], "avoid": avoid[:6]}
|
||||
|
||||
|
||||
def repo_to_external_reference(repo: dict[str, Any]) -> dict[str, str]:
|
||||
primary = repo.get("patterns", [{}])[0]
|
||||
return {
|
||||
"name": repo.get("full_name") or "Unnamed repo",
|
||||
"category": primary.get("category", "general"),
|
||||
"borrow": repo.get("borrow", ["Borrow the strongest reusable pattern."])[0],
|
||||
"avoid": repo.get("avoid", ["Do not copy source-specific language or excess scope."])[0],
|
||||
"source": "external",
|
||||
}
|
||||
|
||||
|
||||
def render_markdown(payload: dict[str, Any]) -> str:
|
||||
lines = [
|
||||
"# GitHub Benchmark Scan",
|
||||
"",
|
||||
f"- Query: `{payload['query']}`",
|
||||
f"- Source: `{payload['source']}`",
|
||||
f"- Top repositories: `{len(payload['repositories'])}`",
|
||||
"",
|
||||
"## Suggested Next Step",
|
||||
"",
|
||||
"Review the three benchmark objects below, then decide whether any of their patterns are worth borrowing into the new skill.",
|
||||
"",
|
||||
]
|
||||
|
||||
if not payload["repositories"]:
|
||||
lines.extend(
|
||||
[
|
||||
"No benchmark repositories were collected.",
|
||||
"",
|
||||
"Possible reasons:",
|
||||
"- the query was too weak or too local",
|
||||
"- the network or API rate limit blocked the scan",
|
||||
"- this skill idea may need a manual benchmark query",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
lines.extend(["## Top 3 Benchmark Repositories", ""])
|
||||
for repo in payload["repositories"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### {repo['full_name']}",
|
||||
f"- URL: {repo['html_url']}",
|
||||
f"- Stars: `{repo['stars']}`",
|
||||
f"- Description: {repo.get('description') or 'No public description provided.'}",
|
||||
f"- Topics: {', '.join(repo.get('topics') or []) or 'None'}",
|
||||
"",
|
||||
"#### Patterns worth studying",
|
||||
"",
|
||||
]
|
||||
)
|
||||
for pattern in repo["patterns"]:
|
||||
lines.append(
|
||||
f"- **{pattern['category']}** (`{pattern['matched']}`): {pattern['borrow']}"
|
||||
)
|
||||
lines.extend(["", "#### Borrow", ""])
|
||||
for item in repo["borrow"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.extend(["", "#### Avoid", ""])
|
||||
for item in repo["avoid"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.extend(["", "#### README glimpse", "", repo["readme_excerpt"] or "No README excerpt collected.", ""])
|
||||
|
||||
lines.extend(["## Cross-Repo Borrow Recommendations", ""])
|
||||
for item in payload["cross_repo"]["borrow"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(["", "## Cross-Repo Avoid Recommendations", ""])
|
||||
for item in payload["cross_repo"]["avoid"]:
|
||||
lines.append(f"- {item}")
|
||||
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## Borrow Prompt",
|
||||
"",
|
||||
payload["borrow_prompt"],
|
||||
"",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def run_github_benchmark_scan(
|
||||
skill_dir: Path,
|
||||
query: str,
|
||||
top_n: int = DEFAULT_TOP_N,
|
||||
fixture_dir: Path | None = None,
|
||||
output_md: Path | None = None,
|
||||
output_json: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
reports_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_md = output_md or reports_dir / "github-benchmark-scan.md"
|
||||
output_json = output_json or reports_dir / "github-benchmark-scan.json"
|
||||
fixture_bundle = load_fixture_bundle(fixture_dir)
|
||||
|
||||
warnings = []
|
||||
try:
|
||||
repos = search_repositories(query, top_n=top_n, fixture_bundle=fixture_bundle)
|
||||
repo_summaries = []
|
||||
for repo in repos:
|
||||
readme_text = fetch_readme(repo["full_name"], fixture_bundle=fixture_bundle)
|
||||
repo_summaries.append(repo_summary(repo, readme_text))
|
||||
cross_repo = cross_repo_insights(repo_summaries)
|
||||
payload = {
|
||||
"ok": True,
|
||||
"query": query,
|
||||
"source": "fixture" if fixture_bundle is not None else "github-api",
|
||||
"repositories": repo_summaries,
|
||||
"cross_repo": cross_repo,
|
||||
"external_references": [repo_to_external_reference(repo) for repo in repo_summaries],
|
||||
"borrow_prompt": "I found three public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability?",
|
||||
"warnings": warnings,
|
||||
}
|
||||
except Exception as exc: # pragma: no cover - network failures are environment-dependent
|
||||
payload = {
|
||||
"ok": False,
|
||||
"query": query,
|
||||
"source": "fixture" if fixture_bundle is not None else "github-api",
|
||||
"repositories": [],
|
||||
"cross_repo": {
|
||||
"borrow": [],
|
||||
"avoid": [],
|
||||
},
|
||||
"external_references": [],
|
||||
"borrow_prompt": "No benchmark suggestions were collected yet. You can retry with a stronger query or add manual references.",
|
||||
"warnings": [f"GitHub benchmark scan failed: {exc}"],
|
||||
}
|
||||
|
||||
output_md.write_text(render_markdown(payload), encoding="utf-8")
|
||||
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
payload["artifacts"] = {
|
||||
"markdown": str(output_md),
|
||||
"json": str(output_json),
|
||||
}
|
||||
payload["skill_dir"] = str(skill_dir)
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Search GitHub for top benchmark repositories and extract reusable patterns.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--query", required=True)
|
||||
parser.add_argument("--top-n", type=int, default=DEFAULT_TOP_N)
|
||||
parser.add_argument("--fixture-dir")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--output-json")
|
||||
args = parser.parse_args()
|
||||
|
||||
result = run_github_benchmark_scan(
|
||||
Path(args.skill_dir),
|
||||
query=args.query,
|
||||
top_n=args.top_n,
|
||||
fixture_dir=Path(args.fixture_dir).resolve() if args.fixture_dir else None,
|
||||
output_md=Path(args.output_md).resolve() if args.output_md else None,
|
||||
output_json=Path(args.output_json).resolve() if args.output_json else None,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+62
-15
@@ -4,6 +4,7 @@ import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
from github_benchmark_scan import run_github_benchmark_scan
|
||||
from render_intent_dialogue import render_intent_dialogue
|
||||
from render_iteration_directions import render_iteration_directions
|
||||
from render_reference_scan import parse_reference, render_reference_scan
|
||||
@@ -50,6 +51,7 @@ README_TEMPLATE = """# {title}
|
||||
- `agents/interface.yaml`: portable interface metadata
|
||||
- `manifest.json`: lifecycle and packaging metadata
|
||||
- `reports/intent-dialogue.md`: front-loaded discovery questions for better boundary design and clearer human alignment
|
||||
- `reports/github-benchmark-scan.md`: top public benchmark repositories, extracted patterns, and borrow or avoid notes
|
||||
- `reports/reference-scan.md`: benchmark notes from public references, user references, and local constraints
|
||||
- `reports/skill-overview.html`: visual overview report
|
||||
- `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps
|
||||
@@ -139,6 +141,24 @@ def build_manifest(name: str, mode: str, archetype: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def dedupe_references(references: list[dict]) -> list[dict]:
|
||||
seen = set()
|
||||
deduped = []
|
||||
for reference in references:
|
||||
key = (
|
||||
reference.get("source"),
|
||||
reference.get("name"),
|
||||
reference.get("category"),
|
||||
reference.get("borrow"),
|
||||
reference.get("avoid"),
|
||||
)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(reference)
|
||||
return deduped
|
||||
|
||||
|
||||
def initialize_skill(
|
||||
name: str,
|
||||
description: str,
|
||||
@@ -149,6 +169,9 @@ def initialize_skill(
|
||||
external_references: list[dict] | None = None,
|
||||
user_references: list[dict] | None = None,
|
||||
local_constraints: list[dict] | None = None,
|
||||
github_query: str | None = None,
|
||||
github_top_n: int = 3,
|
||||
github_fixture_dir: str | None = None,
|
||||
) -> dict:
|
||||
title = title or name.replace("-", " ").title()
|
||||
root = Path(output_dir).resolve() / name
|
||||
@@ -179,28 +202,46 @@ def initialize_skill(
|
||||
)
|
||||
overview = render_skill_overview(root)
|
||||
intent_dialogue = render_intent_dialogue(root)
|
||||
reference_scan = render_reference_scan(root, [*(external_references or []), *(user_references or []), *(local_constraints or [])])
|
||||
benchmark_scan = None
|
||||
combined_external_references = [*(external_references or [])]
|
||||
if github_query:
|
||||
benchmark_scan = run_github_benchmark_scan(
|
||||
root,
|
||||
query=github_query,
|
||||
top_n=github_top_n,
|
||||
fixture_dir=Path(github_fixture_dir).resolve() if github_fixture_dir else None,
|
||||
)
|
||||
combined_external_references = [*benchmark_scan.get("external_references", []), *combined_external_references]
|
||||
reference_scan = render_reference_scan(
|
||||
root,
|
||||
dedupe_references([*combined_external_references, *(user_references or []), *(local_constraints or [])]),
|
||||
)
|
||||
iteration_directions = render_iteration_directions(root)
|
||||
review_viewer = render_review_viewer(root)
|
||||
artifacts = {
|
||||
"readme": str(root / "README.md"),
|
||||
"manifest": str(root / "manifest.json"),
|
||||
"skill_overview_html": overview["artifacts"]["html"],
|
||||
"skill_overview_json": overview["artifacts"]["json"],
|
||||
"intent_dialogue_md": intent_dialogue["artifacts"]["markdown"],
|
||||
"intent_dialogue_json": intent_dialogue["artifacts"]["json"],
|
||||
"reference_scan_md": reference_scan["artifacts"]["markdown"],
|
||||
"reference_scan_json": reference_scan["artifacts"]["json"],
|
||||
"iteration_directions_md": iteration_directions["artifacts"]["markdown"],
|
||||
"iteration_directions_json": iteration_directions["artifacts"]["json"],
|
||||
"review_viewer_html": review_viewer["artifacts"]["html"],
|
||||
"review_viewer_json": review_viewer["artifacts"]["json"],
|
||||
}
|
||||
if benchmark_scan is not None:
|
||||
artifacts["github_benchmark_scan_md"] = benchmark_scan["artifacts"]["markdown"]
|
||||
artifacts["github_benchmark_scan_json"] = benchmark_scan["artifacts"]["json"]
|
||||
return {
|
||||
"ok": True,
|
||||
"root": str(root),
|
||||
"mode": mode,
|
||||
"archetype": archetype,
|
||||
"artifacts": {
|
||||
"readme": str(root / "README.md"),
|
||||
"manifest": str(root / "manifest.json"),
|
||||
"skill_overview_html": overview["artifacts"]["html"],
|
||||
"skill_overview_json": overview["artifacts"]["json"],
|
||||
"intent_dialogue_md": intent_dialogue["artifacts"]["markdown"],
|
||||
"intent_dialogue_json": intent_dialogue["artifacts"]["json"],
|
||||
"reference_scan_md": reference_scan["artifacts"]["markdown"],
|
||||
"reference_scan_json": reference_scan["artifacts"]["json"],
|
||||
"iteration_directions_md": iteration_directions["artifacts"]["markdown"],
|
||||
"iteration_directions_json": iteration_directions["artifacts"]["json"],
|
||||
"review_viewer_html": review_viewer["artifacts"]["html"],
|
||||
"review_viewer_json": review_viewer["artifacts"]["json"],
|
||||
},
|
||||
"github_benchmark_scan": benchmark_scan,
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
|
||||
|
||||
@@ -215,6 +256,9 @@ def main() -> None:
|
||||
parser.add_argument("--external-reference", action="append", default=[])
|
||||
parser.add_argument("--user-reference", action="append", default=[])
|
||||
parser.add_argument("--local-constraint", action="append", default=[])
|
||||
parser.add_argument("--github-query")
|
||||
parser.add_argument("--github-top-n", type=int, default=3)
|
||||
parser.add_argument("--github-fixture-dir")
|
||||
args = parser.parse_args()
|
||||
result = initialize_skill(
|
||||
args.name,
|
||||
@@ -226,6 +270,9 @@ def main() -> None:
|
||||
external_references=[parse_reference(item, "external") for item in args.external_reference],
|
||||
user_references=[parse_reference(item, "user") for item in args.user_reference],
|
||||
local_constraints=[parse_reference(item, "local") for item in args.local_constraint],
|
||||
github_query=args.github_query,
|
||||
github_top_n=args.github_top_n,
|
||||
github_fixture_dir=args.github_fixture_dir,
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
+65
-4
@@ -5,6 +5,8 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from github_benchmark_scan import build_query
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
@@ -190,6 +192,11 @@ def command_init(args: argparse.Namespace) -> int:
|
||||
cmd.extend(["--user-reference", reference])
|
||||
for constraint in args.local_constraint:
|
||||
cmd.extend(["--local-constraint", constraint])
|
||||
if args.github_query:
|
||||
cmd.extend(["--github-query", args.github_query])
|
||||
cmd.extend(["--github-top-n", str(args.github_top_n)])
|
||||
if args.github_fixture_dir:
|
||||
cmd.extend(["--github-fixture-dir", args.github_fixture_dir])
|
||||
result = run_script("init_skill.py", cmd)
|
||||
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["ok"] else 2
|
||||
@@ -197,7 +204,7 @@ def command_init(args: argparse.Namespace) -> int:
|
||||
|
||||
def command_quickstart(args: argparse.Namespace) -> int:
|
||||
sys.stderr.write("Let's shape this skill around the real work, the outcome, and the standards you care about before we add structure.\n")
|
||||
sys.stderr.write("If you already have references you admire, bring them in. We'll learn the pattern, not copy the source.\n")
|
||||
sys.stderr.write("I will also scan a few strong public GitHub references first, so we can borrow the right patterns without copying the source.\n")
|
||||
name = args.name or prompt_with_default("Skill name", "my-skill")
|
||||
job = args.job or prompt_with_default("What repeated work should this skill quietly take off your hands", "Turn a repeated workflow into a reusable skill.")
|
||||
primary_output = args.primary_output or prompt_with_default("What should it reliably hand back", "A reusable skill package.")
|
||||
@@ -209,14 +216,16 @@ def command_quickstart(args: argparse.Namespace) -> int:
|
||||
mode = args.mode or prompt_with_default("Mode (scaffold/production/library/governed)", default_mode)
|
||||
mode = mode if mode in ARCHETYPE_MODE.values() else default_mode
|
||||
user_references = args.user_reference or prompt_optional_entries(
|
||||
"Reference examples you want it to learn from (repo, product, page, workflow; comma-separated)"
|
||||
"Your own reference examples to learn from as pattern hints (repo, product, page, workflow; comma-separated)"
|
||||
)
|
||||
external_references = args.external_reference or prompt_optional_entries(
|
||||
"Public benchmark objects to scan first (high-star GitHub repo, official doc, product; comma-separated)"
|
||||
"Additional public benchmark objects you already know you want to borrow from (comma-separated)"
|
||||
)
|
||||
local_constraints = args.local_constraint or prompt_optional_entries(
|
||||
"Local constraints that must still be respected (privacy, naming, compatibility; comma-separated)"
|
||||
)
|
||||
github_query = args.github_query or build_query(" ".join(filter(None, [job, primary_output, description])))
|
||||
sys.stderr.write(f"GitHub benchmark query: {github_query}\n")
|
||||
title = args.title or name.replace("-", " ").title()
|
||||
guidance = archetype_guidance(archetype)
|
||||
cmd = [
|
||||
@@ -231,7 +240,13 @@ def command_quickstart(args: argparse.Namespace) -> int:
|
||||
mode,
|
||||
"--archetype",
|
||||
archetype,
|
||||
"--github-query",
|
||||
github_query,
|
||||
"--github-top-n",
|
||||
str(args.github_top_n),
|
||||
]
|
||||
if args.github_fixture_dir:
|
||||
cmd.extend(["--github-fixture-dir", args.github_fixture_dir])
|
||||
for reference in external_references:
|
||||
cmd.extend(["--external-reference", reference])
|
||||
for reference in user_references:
|
||||
@@ -240,17 +255,29 @@ def command_quickstart(args: argparse.Namespace) -> int:
|
||||
cmd.extend(["--local-constraint", constraint])
|
||||
result = run_script("init_skill.py", cmd)
|
||||
payload = result["payload"] if result["payload"] is not None else result
|
||||
benchmark_scan = payload.get("github_benchmark_scan") or {}
|
||||
benchmark_repositories = [
|
||||
{
|
||||
"name": repo.get("full_name"),
|
||||
"stars": repo.get("stars"),
|
||||
"url": repo.get("html_url"),
|
||||
}
|
||||
for repo in benchmark_scan.get("repositories", [])
|
||||
]
|
||||
report = {
|
||||
"ok": result["ok"],
|
||||
"root": payload.get("root"),
|
||||
"mode": mode,
|
||||
"archetype": archetype,
|
||||
"references": {
|
||||
"github_query": github_query,
|
||||
"benchmark_repositories": benchmark_repositories,
|
||||
"external_benchmarks": external_references,
|
||||
"user_references": user_references,
|
||||
"local_constraints": local_constraints,
|
||||
},
|
||||
"artifacts": payload.get("artifacts", {}),
|
||||
"github_benchmark_scan": benchmark_scan,
|
||||
"guidance": {
|
||||
"archetype_reason": archetype_reason,
|
||||
"why_this_mode": (
|
||||
@@ -262,10 +289,12 @@ def command_quickstart(args: argparse.Namespace) -> int:
|
||||
"focus": guidance["focus"],
|
||||
"next_steps": [
|
||||
"Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.",
|
||||
"Open reports/reference-scan.md and decide which external patterns to borrow and which user references set the quality bar.",
|
||||
"Open reports/github-benchmark-scan.md and review the top three public benchmark repositories plus their borrow and avoid notes.",
|
||||
"Open reports/reference-scan.md and decide which benchmark patterns to borrow and which user references set the quality bar.",
|
||||
"Open reports/review-viewer.html to explain the package to a first-time reviewer.",
|
||||
"Use reports/iteration-directions.md to choose only one high-value next move before adding more files.",
|
||||
],
|
||||
"borrow_prompt": benchmark_scan.get("borrow_prompt"),
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
@@ -416,6 +445,20 @@ def command_reference_scan(args: argparse.Namespace) -> int:
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_github_benchmark_scan(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir, "--query", args.query, "--top-n", str(args.top_n)]
|
||||
if args.fixture_dir:
|
||||
cmd.extend(["--fixture-dir", args.fixture_dir])
|
||||
if args.output_md:
|
||||
cmd.extend(["--output-md", args.output_md])
|
||||
if args.output_json:
|
||||
cmd.extend(["--output-json", args.output_json])
|
||||
result = run_script("github_benchmark_scan.py", cmd)
|
||||
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_intent_dialogue(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir]
|
||||
@@ -620,6 +663,9 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
init_cmd.add_argument("--external-reference", action="append", default=[])
|
||||
init_cmd.add_argument("--user-reference", action="append", default=[])
|
||||
init_cmd.add_argument("--local-constraint", action="append", default=[])
|
||||
init_cmd.add_argument("--github-query")
|
||||
init_cmd.add_argument("--github-top-n", type=int, default=3)
|
||||
init_cmd.add_argument("--github-fixture-dir")
|
||||
init_cmd.set_defaults(func=command_init)
|
||||
|
||||
quickstart_cmd = subparsers.add_parser(
|
||||
@@ -637,6 +683,9 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
quickstart_cmd.add_argument("--external-reference", action="append", default=[])
|
||||
quickstart_cmd.add_argument("--user-reference", action="append", default=[])
|
||||
quickstart_cmd.add_argument("--local-constraint", action="append", default=[])
|
||||
quickstart_cmd.add_argument("--github-query")
|
||||
quickstart_cmd.add_argument("--github-top-n", type=int, default=3)
|
||||
quickstart_cmd.add_argument("--github-fixture-dir")
|
||||
quickstart_cmd.set_defaults(func=command_quickstart)
|
||||
|
||||
validate_cmd = subparsers.add_parser("validate", help="Run validate, lint, governance, and resource checks.")
|
||||
@@ -714,6 +763,18 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
reference_scan_cmd.add_argument("--output-json")
|
||||
reference_scan_cmd.set_defaults(func=command_reference_scan)
|
||||
|
||||
github_scan_cmd = subparsers.add_parser(
|
||||
"github-benchmark-scan",
|
||||
help="Search top public GitHub repositories and extract borrow or avoid patterns for a skill.",
|
||||
)
|
||||
github_scan_cmd.add_argument("skill_dir", nargs="?", default=".")
|
||||
github_scan_cmd.add_argument("--query", required=True)
|
||||
github_scan_cmd.add_argument("--top-n", type=int, default=3)
|
||||
github_scan_cmd.add_argument("--fixture-dir")
|
||||
github_scan_cmd.add_argument("--output-md")
|
||||
github_scan_cmd.add_argument("--output-json")
|
||||
github_scan_cmd.set_defaults(func=command_github_benchmark_scan)
|
||||
|
||||
intent_dialogue_cmd = subparsers.add_parser(
|
||||
"intent-dialogue",
|
||||
help="Render a front-loaded intent dialogue guide for a skill package.",
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"search_items": [
|
||||
{
|
||||
"full_name": "anthropics/skills",
|
||||
"html_url": "https://github.com/anthropics/skills",
|
||||
"description": "A collection of composable skills for agent workflows, evaluation, and reusable operating patterns.",
|
||||
"stargazers_count": 12800,
|
||||
"topics": ["skills", "agents", "workflow", "evaluation"],
|
||||
"language": "Markdown",
|
||||
"fork": false,
|
||||
"archived": false
|
||||
},
|
||||
{
|
||||
"full_name": "openai/skills",
|
||||
"html_url": "https://github.com/openai/skills",
|
||||
"description": "Reference skills and patterns for creating portable, reusable agent capabilities.",
|
||||
"stargazers_count": 9400,
|
||||
"topics": ["skills", "portable", "agent", "toolchain"],
|
||||
"language": "Markdown",
|
||||
"fork": false,
|
||||
"archived": false
|
||||
},
|
||||
{
|
||||
"full_name": "microsoft/autogen",
|
||||
"html_url": "https://github.com/microsoft/autogen",
|
||||
"description": "Agent orchestration framework with workflow patterns, evaluation, tooling, and portability guidance.",
|
||||
"stargazers_count": 35500,
|
||||
"topics": ["agents", "automation", "toolchain", "workflow"],
|
||||
"language": "Python",
|
||||
"fork": false,
|
||||
"archived": false
|
||||
}
|
||||
],
|
||||
"readmes": {
|
||||
"anthropics/skills": "# Skills\n\nThis repository documents workflow patterns, evaluation loops, benchmark practices, and review steps for reusable agent skills. It includes examples, references, and quality checks for operating a repeatable process.\n",
|
||||
"openai/skills": "# Skills\n\nThis project emphasizes portable metadata, compatibility adapters, references, templates, and clear execution entrypoints. It keeps core semantics stable while allowing environment-specific packaging.\n",
|
||||
"microsoft/autogen": "# AutoGen\n\nThis framework provides toolchain commands, automation flows, orchestration guidance, and validation practices. It highlights scripts, command entrypoints, and reusable execution structures for agent workflows.\n"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = ROOT / "scripts" / "github_benchmark_scan.py"
|
||||
FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tmp_root = ROOT / "tests" / "tmp_github_benchmark_scan"
|
||||
if tmp_root.exists():
|
||||
subprocess.run(["rm", "-rf", str(tmp_root)], check=True)
|
||||
skill_dir = tmp_root / "benchmark-scan-demo"
|
||||
(skill_dir / "reports").mkdir(parents=True, exist_ok=True)
|
||||
(skill_dir / "SKILL.md").write_text(
|
||||
"---\nname: benchmark-scan-demo\ndescription: Turn rough release notes into a reusable governed skill.\n---\n\n# Benchmark Scan Demo\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(skill_dir),
|
||||
"--query",
|
||||
"release workflow evaluation portability",
|
||||
"--fixture-dir",
|
||||
str(FIXTURE_DIR),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
assert payload["ok"], payload
|
||||
assert payload["source"] == "fixture", payload
|
||||
assert len(payload["repositories"]) == 3, payload
|
||||
assert len(payload["external_references"]) == 3, payload
|
||||
assert payload["borrow_prompt"], payload
|
||||
|
||||
markdown = Path(payload["artifacts"]["markdown"]).read_text(encoding="utf-8")
|
||||
assert "## Top 3 Benchmark Repositories" in markdown, markdown[:400]
|
||||
assert "anthropics/skills" in markdown, markdown[:800]
|
||||
assert "Cross-Repo Borrow Recommendations" in markdown, markdown[:1400]
|
||||
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -7,6 +7,7 @@ from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
CLI = ROOT / "scripts" / "yao.py"
|
||||
BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
|
||||
|
||||
|
||||
def run(*args: str, input_text: str | None = None) -> dict:
|
||||
@@ -47,12 +48,16 @@ def main() -> None:
|
||||
"quickstart",
|
||||
"--output-dir",
|
||||
str(tmp_root),
|
||||
"--github-fixture-dir",
|
||||
str(BENCHMARK_FIXTURE_DIR),
|
||||
input_text="quickstart-skill\nTurn rough notes into a reusable package.\nA reusable markdown workflow.\nproduction\nproduction\nA top-tier internal workflow product\nhigh-star GitHub repo, official docs\nprivacy and naming\n",
|
||||
)
|
||||
assert quickstart_result["ok"], quickstart_result
|
||||
quickstart_root = Path(quickstart_result["payload"]["root"])
|
||||
assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root
|
||||
assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root
|
||||
assert quickstart_result["payload"]["archetype"] == "production", quickstart_result
|
||||
assert len(quickstart_result["payload"]["references"]["benchmark_repositories"]) == 3, quickstart_result
|
||||
assert quickstart_result["payload"]["references"]["user_references"] == ["A top-tier internal workflow product"], quickstart_result
|
||||
|
||||
validate_result = run("validate", str(created))
|
||||
@@ -81,6 +86,17 @@ def main() -> None:
|
||||
assert reference_scan_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-scan.md"), reference_scan_result
|
||||
assert len(reference_scan_result["payload"]["summary"]["user_references"]) == 1, reference_scan_result
|
||||
|
||||
github_benchmark_result = run(
|
||||
"github-benchmark-scan",
|
||||
str(created),
|
||||
"--query",
|
||||
"workflow evaluation portability",
|
||||
"--fixture-dir",
|
||||
str(BENCHMARK_FIXTURE_DIR),
|
||||
)
|
||||
assert github_benchmark_result["ok"], github_benchmark_result
|
||||
assert len(github_benchmark_result["payload"]["repositories"]) == 3, github_benchmark_result
|
||||
|
||||
intent_result = run("intent-dialogue", str(created))
|
||||
assert intent_result["ok"], intent_result
|
||||
assert intent_result["payload"]["artifacts"]["markdown"].endswith("reports/intent-dialogue.md"), intent_result
|
||||
|
||||
Reference in New Issue
Block a user