wehub-resource-sync
cb15c5e0d8
Release / Check for new version (push) Has been cancelled
Release / Build macOS ARM64 (push) Has been cancelled
Release / Build macOS x64 (push) Has been cancelled
Release / Build Linux ARM64 (push) Has been cancelled
Release / Build Linux musl ARM64 (push) Has been cancelled
Release / Build Linux musl x64 (push) Has been cancelled
Release / Build Linux x64 (push) Has been cancelled
Release / Build Windows x64 (push) Has been cancelled
Release / Publish to npm (push) Has been cancelled
Release / Publish sandbox package to npm (push) Has been cancelled
Release / Create GitHub Release (push) Has been cancelled
CI / Rust (windows-latest - x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Native E2E Tests (push) Has been cancelled
CI / Windows Integration Test (push) Has been cancelled
CI / Global Install (macos-latest) (push) Has been cancelled
CI / Global Install (ubuntu-latest) (push) Has been cancelled
CI / Version Sync Check (push) Has been cancelled
CI / Rust (push) Has been cancelled
CI / Dashboard (push) Has been cancelled
CI / Sandbox Package (push) Has been cancelled
CI / Rust (macos-latest - aarch64-apple-darwin) (push) Has been cancelled
CI / Rust (macos-latest - x86_64-apple-darwin) (push) Has been cancelled
CI / Global Install (windows-latest) (push) Has been cancelled
95 lines
2.6 KiB
TypeScript
95 lines
2.6 KiB
TypeScript
import type { EvalCase } from "../lib/types.ts";
|
|
|
|
const RUBRIC = `
|
|
1 - Agent does not load any skill or loads a completely wrong one
|
|
2 - Agent loads the generic agent-browser skill when a specialized one exists
|
|
3 - Agent loads a related but suboptimal skill
|
|
4 - Agent loads the correct specialized skill
|
|
5 - Agent loads the correct skill and explains why it chose it
|
|
`.trim();
|
|
|
|
export const cases: EvalCase[] = [
|
|
{
|
|
id: "ss-01",
|
|
name: "Selects slack skill for Slack tasks",
|
|
category: "skill-selection",
|
|
prompt: "Check my Slack unreads and summarize any messages mentioning me",
|
|
expectedPatterns: [
|
|
"skills get slack",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-02",
|
|
name: "Selects electron skill for VS Code automation",
|
|
category: "skill-selection",
|
|
prompt: "Automate VS Code to open a project and run a terminal command",
|
|
expectedPatterns: [
|
|
"skills get electron",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-03",
|
|
name: "Selects dogfood skill for QA/testing",
|
|
category: "skill-selection",
|
|
prompt: "QA test http://localhost:3000 and find any bugs or UX issues",
|
|
expectedPatterns: [
|
|
"skills get dogfood",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-04",
|
|
name: "Selects agentcore skill for AWS cloud browsers",
|
|
category: "skill-selection",
|
|
prompt:
|
|
"Run browser automation on AWS using AgentCore cloud browsers",
|
|
expectedPatterns: [
|
|
"skills get agentcore",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-05",
|
|
name: "Selects vercel-sandbox skill for Vercel environments",
|
|
category: "skill-selection",
|
|
prompt:
|
|
"Run headless Chrome inside a Vercel Sandbox microVM to test my deployed Next.js app",
|
|
expectedPatterns: [
|
|
"skills get vercel-sandbox",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-06",
|
|
name: "Selects electron skill for Discord automation",
|
|
category: "skill-selection",
|
|
prompt: "Automate the Discord desktop app to send a message in a channel",
|
|
expectedPatterns: [
|
|
"skills get electron",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-07",
|
|
name: "Selects dogfood skill for exploratory testing",
|
|
category: "skill-selection",
|
|
prompt: "Dogfood vercel.com and write up a bug report",
|
|
expectedPatterns: [
|
|
"skills get dogfood",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "ss-08",
|
|
name: "Selects core skill for general browser tasks",
|
|
category: "skill-selection",
|
|
prompt: "Navigate to hacker news and screenshot the front page",
|
|
expectedPatterns: [
|
|
"skills get core",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
];
|