wehub-resource-sync
cb15c5e0d8
Release / Check for new version (push) Has been cancelled
Release / Build macOS ARM64 (push) Has been cancelled
Release / Build macOS x64 (push) Has been cancelled
Release / Build Linux ARM64 (push) Has been cancelled
Release / Build Linux musl ARM64 (push) Has been cancelled
Release / Build Linux musl x64 (push) Has been cancelled
Release / Build Linux x64 (push) Has been cancelled
Release / Build Windows x64 (push) Has been cancelled
Release / Publish to npm (push) Has been cancelled
Release / Publish sandbox package to npm (push) Has been cancelled
Release / Create GitHub Release (push) Has been cancelled
CI / Rust (windows-latest - x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Native E2E Tests (push) Has been cancelled
CI / Windows Integration Test (push) Has been cancelled
CI / Global Install (macos-latest) (push) Has been cancelled
CI / Global Install (ubuntu-latest) (push) Has been cancelled
CI / Version Sync Check (push) Has been cancelled
CI / Rust (push) Has been cancelled
CI / Dashboard (push) Has been cancelled
CI / Sandbox Package (push) Has been cancelled
CI / Rust (macos-latest - aarch64-apple-darwin) (push) Has been cancelled
CI / Rust (macos-latest - x86_64-apple-darwin) (push) Has been cancelled
CI / Global Install (windows-latest) (push) Has been cancelled
67 lines
1.8 KiB
TypeScript
67 lines
1.8 KiB
TypeScript
import type { EvalCase } from "../lib/types.ts";
|
|
|
|
const RUBRIC = `
|
|
1 - Agent does not mention agent-browser skills or load any skill
|
|
2 - Agent mentions skills but does not run the skills get command
|
|
3 - Agent runs skills get but for the wrong skill or at the wrong time
|
|
4 - Agent runs skills get for the correct skill before using agent-browser
|
|
5 - Agent runs skills get first, then follows the loaded skill's workflow correctly
|
|
`.trim();
|
|
|
|
export const cases: EvalCase[] = [
|
|
{
|
|
id: "sl-01",
|
|
name: "Loads skill before opening a page",
|
|
category: "skill-loading",
|
|
prompt: "Open example.com and take a screenshot",
|
|
expectedPatterns: [
|
|
"agent-browser skills get",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "sl-02",
|
|
name: "Loads skill before form interaction",
|
|
category: "skill-loading",
|
|
prompt:
|
|
"Go to example.com/contact and fill out the contact form with test data",
|
|
expectedPatterns: [
|
|
"agent-browser skills get",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "sl-03",
|
|
name: "Loads skill before data extraction",
|
|
category: "skill-loading",
|
|
prompt:
|
|
"Scrape all product names and prices from shop.example.com",
|
|
expectedPatterns: [
|
|
"agent-browser skills get",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "sl-04",
|
|
name: "Loads skill before authentication task",
|
|
category: "skill-loading",
|
|
prompt:
|
|
"Log into my GitHub account and check my notifications",
|
|
expectedPatterns: [
|
|
"agent-browser skills get",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
{
|
|
id: "sl-05",
|
|
name: "Uses skills list to discover available skills",
|
|
category: "skill-loading",
|
|
prompt:
|
|
"I need to automate some browser tasks. What skills are available for agent-browser?",
|
|
expectedPatterns: [
|
|
"agent-browser skills (list|get)",
|
|
],
|
|
rubric: RUBRIC,
|
|
},
|
|
];
|