chore: import upstream snapshot with attribution

2026-07-13 12:30:05 +08:00
commit 51d876a392
274 changed files with 121542 additions and 0 deletions
@@ -0,0 +1,277 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import fs from 'node:fs';
+import path from 'node:path';
+import {pathToFileURL} from 'node:url';
+import {parseArgs} from 'node:util';
+
+import {GoogleGenAI, mcpToTool} from '@google/genai';
+import {Client} from '@modelcontextprotocol/sdk/client/index.js';
+import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
+
+import {TestServer} from '../build/tests/server.js';
+
+const ROOT_DIR = path.resolve(import.meta.dirname, '..');
+const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
+const SKILL_PATH = path.join(ROOT_DIR, 'skills', 'chrome-devtools', 'SKILL.md');
+
+import type {CapturedFunctionCall, TestScenario} from './eval_result.ts';
+import {Result} from './eval_result.ts';
+export type {CapturedFunctionCall, TestScenario};
+export {Result};
+
+async function loadScenario(scenarioPath: string): Promise<TestScenario> {
+  const module = await import(pathToFileURL(scenarioPath).href);
+  if (!module.scenario) {
+    throw new Error(
+      `Scenario file ${scenarioPath} does not export a 'scenario' object.`,
+    );
+  }
+  return module.scenario;
+}
+
+async function runSingleScenario(
+  scenarioPath: string,
+  apiKey: string,
+  server: TestServer,
+  modelId: string,
+  debug: boolean,
+  includeSkill: boolean,
+  extraServerArgs: string[] = [],
+): Promise<void> {
+  const debugLog = (...args: unknown[]) => {
+    if (debug) {
+      console.log(...args);
+    }
+  };
+  const absolutePath = path.resolve(scenarioPath);
+  debugLog(
+    `\n### Running Scenario: ${path.relative(ROOT_DIR, absolutePath)} ###`,
+  );
+
+  let client: Client | undefined;
+  let transport: StdioClientTransport | undefined;
+
+  try {
+    const loadedScenario = await loadScenario(absolutePath);
+    const scenario = {...loadedScenario};
+
+    // Prepend skill content if requested
+    if (includeSkill) {
+      if (!fs.existsSync(SKILL_PATH)) {
+        throw new Error(
+          `Skill file not found at ${SKILL_PATH}. Please ensure the skill file exists.`,
+        );
+      }
+      const skillContent = fs.readFileSync(SKILL_PATH, 'utf-8');
+      scenario.prompt = `${skillContent}\n\n---\n\n${scenario.prompt}`;
+    }
+
+    // Append random queryid to avoid caching issues and test distinct runs
+    const randomId = Math.floor(Math.random() * 1000000);
+    scenario.prompt = `${scenario.prompt}\nqueryid=${randomId}`;
+
+    if (scenario.htmlRoute) {
+      server.addHtmlRoute(
+        scenario.htmlRoute.path,
+        scenario.htmlRoute.htmlContent,
+      );
+      scenario.prompt = scenario.prompt.replace(
+        '<TEST_URL>',
+        server.getRoute(scenario.htmlRoute.path),
+      );
+    }
+
+    // Path to the compiled MCP server
+    const serverPath = path.join(
+      ROOT_DIR,
+      'build/src/bin/chrome-devtools-mcp.js',
+    );
+    if (!fs.existsSync(serverPath)) {
+      throw new Error(
+        `MCP server not found at ${serverPath}. Please run 'npm run build' first.`,
+      );
+    }
+
+    // Environment variables
+    const env: Record<string, string> = {};
+    Object.entries(process.env).forEach(([key, value]) => {
+      if (value !== undefined) {
+        env[key] = value;
+      }
+    });
+    env['CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS'] = 'true';
+
+    const args = [serverPath];
+    if (!debug) {
+      args.push('--headless');
+    }
+    if (scenario.serverArgs) {
+      args.push(...scenario.serverArgs);
+    }
+    if (extraServerArgs.length > 0) {
+      args.push(...extraServerArgs);
+    }
+
+    transport = new StdioClientTransport({
+      command: 'node',
+      args,
+      env,
+      stderr: debug ? 'inherit' : 'ignore',
+    });
+
+    client = new Client(
+      {name: 'gemini-eval-client', version: '1.0.0'},
+      {capabilities: {}},
+    );
+
+    await client.connect(transport);
+
+    const allCalls: CapturedFunctionCall[] = [];
+    const originalCallTool = client.callTool.bind(client);
+    client.callTool = async (request, schema) => {
+      // NOTE: request.name is the original name as the MCP client sees it.
+      // mcpToTool handles the conversion from Gemini sanitized name to original name.
+      debugLog(
+        `Executing tool: ${request.name} with args: ${JSON.stringify(request.arguments)}`,
+      );
+      allCalls.push({
+        name: request.name,
+        args: (request.arguments as Record<string, unknown>) || {},
+      });
+      const response = await originalCallTool(request, schema);
+      debugLog(`Tool response: ${JSON.stringify(response)}`);
+      return response;
+    };
+
+    const ai = new GoogleGenAI({apiKey});
+
+    debugLog(`\n--- Prompt ---\n${scenario.prompt}`);
+
+    const result = await ai.models.generateContent({
+      model: modelId,
+      contents: scenario.prompt,
+      config: {
+        tools: [mcpToTool(client)],
+        automaticFunctionCalling: {
+          maximumRemoteCalls: scenario.maxTurns,
+        },
+      },
+    });
+
+    debugLog(`\n--- Response ---\n${result.text}`);
+
+    debugLog('\nVerifying expectations...');
+    scenario.expectations(new Result(allCalls, args));
+  } finally {
+    try {
+      await client?.close();
+    } catch (e) {
+      console.error('Error closing client:', e);
+    }
+  }
+}
+
+async function main() {
+  const apiKey = process.env.GEMINI_API_KEY;
+  if (!apiKey) {
+    throw new Error('GEMINI_API_KEY environment variable is required.');
+  }
+
+  const {values, positionals} = parseArgs({
+    options: {
+      model: {
+        type: 'string',
+        default: 'gemini-3-flash-preview',
+      },
+      debug: {
+        type: 'boolean',
+        default: false,
+      },
+      repeat: {
+        type: 'boolean',
+        default: false,
+      },
+      'include-skill': {
+        type: 'boolean',
+        default: false,
+      },
+      'server-args': {
+        type: 'string',
+      },
+    },
+    allowPositionals: true,
+  });
+
+  const modelId = values.model;
+  const debug = values.debug;
+  const repeat = values.repeat;
+  const includeSkill = values['include-skill'];
+  const extraServerArgs = values['server-args']
+    ? values['server-args'].split(/\s+/)
+    : [];
+
+  const scenarioFiles =
+    positionals.length > 0
+      ? positionals.map(p => path.resolve(p))
+      : fs
+          .readdirSync(SCENARIOS_DIR)
+          .filter(file => file.endsWith('.ts') || file.endsWith('.js'))
+          .map(file => path.join(SCENARIOS_DIR, file));
+
+  const server = new TestServer(TestServer.randomPort());
+  await server.start();
+
+  let successCount = 0;
+  let failureCount = 0;
+
+  try {
+    for (const scenarioPath of scenarioFiles) {
+      for (let i = 1; i <= (repeat ? 3 : 1); i++) {
+        try {
+          if (debug) {
+            console.log(
+              `Running scenario: ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i}/3)`,
+            );
+          }
+          await runSingleScenario(
+            scenarioPath,
+            apiKey,
+            server,
+            modelId,
+            debug,
+            includeSkill,
+            extraServerArgs,
+          );
+          console.log(`✔ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`);
+          successCount++;
+        } catch (e) {
+          console.error(
+            `✖ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`,
+          );
+          console.error(e);
+          failureCount++;
+        } finally {
+          server.restore();
+        }
+      }
+    }
+  } finally {
+    await server.stop();
+  }
+
+  console.log(`\nSummary: ${successCount} passed, ${failureCount} failed`);
+
+  if (failureCount > 0) {
+    process.exit(1);
+  }
+}
+
+main().catch(error => {
+  console.error('Fatal error:', error);
+  process.exit(1);
+});