frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add examples evals and stronger packaging checks

Browse Source
This commit is contained in:
yaojingang
2026-03-31 20:48:39 +08:00
parent fed2fe1354
commit b6cc9ca8b9
26 changed files with 451 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cross_packager.py
+79 -1
View File
@@ -84,6 +84,37 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
}
PLATFORM_CONTRACTS = {
"openai": {
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
"required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
"field_mapping": {
"display_name": "interface.display_name",
"short_description": "interface.short_description",
"default_prompt": "interface.default_prompt",
},
},
"claude": {
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
"required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
"field_mapping": {
"display_name": "adapter.display_name",
"short_description": "adapter.short_description",
"default_prompt": "adapter.default_prompt",
},
},
"generic": {
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
"required_files": ["targets/generic/adapter.json"],
"field_mapping": {
"display_name": "adapter.display_name",
"short_description": "adapter.short_description",
"default_prompt": "adapter.default_prompt",
},
},
}
def write_yaml_like(path: Path, payload: dict) -> None:
interface = payload.get("interface", {})
lines = ["interface:"]
@@ -121,6 +152,7 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
else:
payload["install_hint"] = f"Use {skill_dir.name} as an Agent Skills compatible package."
path = target_dir / "adapter.json"
payload["contract"] = PLATFORM_CONTRACTS.get(platform, PLATFORM_CONTRACTS["generic"])
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
return path
@@ -143,11 +175,43 @@ def copy_manifest(skill_dir: Path, out_dir: Path) -> Path:
return manifest_path
def load_expectations(path: Path | None) -> dict:
if path is None:
return {}
return json.loads(path.read_text(encoding="utf-8"))
def validate_exports(out_dir: Path, expectations: dict) -> dict:
failures = []
required_targets = expectations.get("required_targets", [])
required_fields = expectations.get("required_fields", [])
required_by_target = {
"openai": expectations.get("openai_required_files", []),
"claude": expectations.get("claude_required_files", []),
"generic": expectations.get("generic_required_files", []),
}
for target in required_targets:
adapter_path = out_dir / "targets" / target / "adapter.json"
if not adapter_path.exists():
failures.append(f"missing adapter for target: {target}")
continue
payload = json.loads(adapter_path.read_text(encoding="utf-8"))
for field in required_fields:
if field not in payload:
failures.append(f"missing field '{field}' in {adapter_path.relative_to(out_dir)}")
for rel in required_by_target.get(target, []):
if not (out_dir / rel).exists():
failures.append(f"missing file: {rel}")
return {"ok": not failures, "failures": failures}
def main() -> None:
parser = argparse.ArgumentParser(description="Generate lightweight cross-platform packaging artifacts.")
parser.add_argument("skill_dir", help="Path to the skill directory")
parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic")
parser.add_argument("--output-dir", default="dist", help="Output directory")
parser.add_argument("--expectations", help="JSON file describing packaging expectations")
parser.add_argument("--zip", action="store_true", help="Create a zip package")
args = parser.parse_args()
@@ -164,7 +228,21 @@ def main() -> None:
if args.zip:
generated.append(str(make_zip(skill_dir, out_dir)))
print(json.dumps({"output_dir": str(out_dir), "generated": generated}, ensure_ascii=False, indent=2))
expectations = load_expectations(Path(args.expectations).resolve()) if args.expectations else {}
validation = validate_exports(out_dir, expectations) if expectations else None
report = {
"output_dir": str(out_dir),
"generated": generated,
"contracts": PLATFORM_CONTRACTS,
"validation": validation,
"failure_handling": {
"missing_required_file": "exit with code 2 when expectations are provided and validation fails",
"missing_required_field": "exit with code 2 when expectations are provided and validation fails",
},
}
print(json.dumps(report, ensure_ascii=False, indent=2))
if validation and not validation["ok"]:
raise SystemExit(2)
if __name__ == "__main__":
scripts/trigger_eval.py
+73 -12
View File
@@ -37,43 +37,96 @@ def score_prompt(description_words: set[str], prompt: str) -> float:
return len(overlap) / len(prompt_words)
def classify_bucket(bucket: str) -> bool:
return bucket == "should_trigger"
def evaluate(description: str, cases: dict, threshold: float) -> dict:
desc_words = words(description)
results = {"should_trigger": [], "should_not_trigger": []}
results = {"should_trigger": [], "should_not_trigger": [], "near_neighbor": []}
fp = 0
fn = 0
bucket_stats = {}
misfires = []
for bucket in ("should_trigger", "should_not_trigger"):
for bucket in ("should_trigger", "should_not_trigger", "near_neighbor"):
expected = classify_bucket(bucket)
total = 0
passed_count = 0
for prompt in cases.get(bucket, []):
score = score_prompt(desc_words, prompt)
predicted = score >= threshold
expected = bucket == "should_trigger"
passed = predicted == expected
total += 1
if passed:
passed_count += 1
if not passed and expected:
fn += 1
if not passed and not expected:
fp += 1
results[bucket].append(
{
"prompt": prompt,
"score": round(score, 3),
"predicted_trigger": predicted,
"passed": passed,
}
)
record = {
"prompt": prompt,
"score": round(score, 3),
"predicted_trigger": predicted,
"expected_trigger": expected,
"passed": passed,
}
if 0.75 * threshold <= score <= 1.25 * threshold:
record["boundary_case"] = True
results[bucket].append(record)
if not passed:
misfires.append(
{
"bucket": bucket,
"prompt": prompt,
"score": round(score, 3),
"reason": "false_negative" if expected else "false_positive",
}
)
bucket_stats[bucket] = {
"total": total,
"passed": passed_count,
"pass_rate": round(passed_count / total, 3) if total else None,
}
tp = sum(1 for item in results["should_trigger"] if item["predicted_trigger"])
precision = tp / (tp + fp) if (tp + fp) else None
recall = tp / (tp + fn) if (tp + fn) else None
return {
"threshold": threshold,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.",
"false_positives": fp,
"false_negatives": fn,
"precision": round(precision, 3) if precision is not None else None,
"recall": round(recall, 3) if recall is not None else None,
"bucket_stats": bucket_stats,
"misfires": misfires,
"results": results,
}
def compare_reports(baseline: dict, improved: dict) -> dict:
return {
"baseline_false_positives": baseline["false_positives"],
"baseline_false_negatives": baseline["false_negatives"],
"improved_false_positives": improved["false_positives"],
"improved_false_negatives": improved["false_negatives"],
"false_positive_delta": improved["false_positives"] - baseline["false_positives"],
"false_negative_delta": improved["false_negatives"] - baseline["false_negatives"],
"baseline_precision": baseline["precision"],
"improved_precision": improved["precision"],
"baseline_recall": baseline["recall"],
"improved_recall": improved["recall"],
}
def main() -> None:
parser = argparse.ArgumentParser(description="Heuristic trigger quality evaluator.")
parser.add_argument("--description", help="Description string to evaluate")
parser.add_argument("--description-file", help="Read description text from file")
parser.add_argument("--baseline-description", help="Baseline description string to compare against")
parser.add_argument("--baseline-description-file", help="Read baseline description from file")
parser.add_argument("--cases", required=True, help="JSON file with should_trigger and should_not_trigger arrays")
parser.add_argument("--threshold", type=float, default=0.18, help="Token overlap threshold")
args = parser.parse_args()
@@ -84,7 +137,15 @@ def main() -> None:
if not description:
raise SystemExit("Provide --description or --description-file")
report = evaluate(description, load_cases(Path(args.cases)), args.threshold)
cases = load_cases(Path(args.cases))
report = evaluate(description, cases, args.threshold)
baseline = args.baseline_description
if args.baseline_description_file:
baseline = extract_description(Path(args.baseline_description_file).read_text(encoding="utf-8"))
if baseline:
report["comparison"] = compare_reports(evaluate(baseline, cases, args.threshold), report)
print(json.dumps(report, ensure_ascii=False, indent=2))
if report["false_positives"] > 2:
raise SystemExit(2)