chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const ROOT_DIR = process.cwd();
|
||||
const TARGET_DIR = path.join(ROOT_DIR, 'build/src/third_party');
|
||||
const SOURCE_DIR = path.join(ROOT_DIR, 'src/third_party');
|
||||
|
||||
function main() {
|
||||
const lighthouseNotices = fs.readFileSync(
|
||||
path.join(SOURCE_DIR, 'LIGHTHOUSE_MCP_BUNDLE_THIRD_PARTY_NOTICES'),
|
||||
'utf8',
|
||||
);
|
||||
const bundledNotices = fs.readFileSync(
|
||||
path.join(TARGET_DIR, 'THIRD_PARTY_NOTICES'),
|
||||
'utf8',
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(TARGET_DIR, 'THIRD_PARTY_NOTICES'),
|
||||
bundledNotices +
|
||||
'\n\n-------------------- DEPENDENCY DIVIDER --------------------\n\n' +
|
||||
lighthouseNotices,
|
||||
);
|
||||
console.log('Done.');
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {readFileSync} from 'node:fs';
|
||||
import {parseArgs} from 'node:util';
|
||||
|
||||
import {GoogleGenAI} from '@google/genai';
|
||||
|
||||
const ai = new GoogleGenAI({apiKey: process.env.GEMINI_API_KEY});
|
||||
|
||||
const {values, positionals} = parseArgs({
|
||||
options: {
|
||||
model: {
|
||||
type: 'string',
|
||||
default: 'gemini-2.5-flash',
|
||||
},
|
||||
file: {
|
||||
type: 'string',
|
||||
short: 'f',
|
||||
},
|
||||
},
|
||||
allowPositionals: true,
|
||||
});
|
||||
|
||||
let contents = positionals[0];
|
||||
|
||||
if (values.file) {
|
||||
contents = readFileSync(values.file, 'utf8');
|
||||
}
|
||||
|
||||
if (!contents) {
|
||||
console.error('Usage: npm run count-tokens -- [-f <file>] [<text>]');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const response = await ai.models.countTokens({
|
||||
model: values.model,
|
||||
contents,
|
||||
});
|
||||
console.log(`Input: ${values.file || positionals[0]}`);
|
||||
console.log(`Tokens: ${response.totalTokens}`);
|
||||
@@ -0,0 +1,100 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
const currentYear = new Date().getFullYear();
|
||||
const licenseHeader = `
|
||||
/**
|
||||
* @license
|
||||
* Copyright ${currentYear} Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
`;
|
||||
|
||||
export default {
|
||||
name: 'check-license',
|
||||
meta: {
|
||||
type: 'layout',
|
||||
docs: {
|
||||
description: 'Validate existence of license header',
|
||||
},
|
||||
fixable: 'code',
|
||||
schema: [],
|
||||
messages: {
|
||||
licenseRule: 'Add license header.',
|
||||
emptyLine: 'Add empty line after license header.',
|
||||
},
|
||||
},
|
||||
defaultOptions: [],
|
||||
create(context) {
|
||||
const sourceCode = context.getSourceCode();
|
||||
const comments = sourceCode.getAllComments();
|
||||
let insertAfter = [0, 0];
|
||||
let header = null;
|
||||
// Check only the first 2 comments
|
||||
for (let index = 0; index < 2; index++) {
|
||||
const comment = comments[index];
|
||||
if (!comment) {
|
||||
break;
|
||||
}
|
||||
// Shebang comments should be at the top
|
||||
if (
|
||||
comment.type === 'Shebang' ||
|
||||
(comment.type === 'Line' && comment.value.startsWith('#!'))
|
||||
) {
|
||||
insertAfter = comment.range;
|
||||
continue;
|
||||
}
|
||||
if (comment.type === 'Block') {
|
||||
header = comment;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
Program(node) {
|
||||
if (context.getFilename().endsWith('.json')) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
header &&
|
||||
(header.value.includes('@license') ||
|
||||
header.value.includes('License') ||
|
||||
header.value.includes('Copyright'))
|
||||
) {
|
||||
const nextToken = sourceCode.getTokenAfter(header, {
|
||||
includeComments: true,
|
||||
});
|
||||
if (
|
||||
nextToken &&
|
||||
nextToken.loc.start.line === header.loc.end.line + 1
|
||||
) {
|
||||
context.report({
|
||||
node: node,
|
||||
loc: header.loc,
|
||||
messageId: 'emptyLine',
|
||||
fix(fixer) {
|
||||
return fixer.insertTextAfter(header, '\n');
|
||||
},
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Add header license
|
||||
if (!header || !header.value.includes('@license')) {
|
||||
context.report({
|
||||
node: node,
|
||||
messageId: 'licenseRule',
|
||||
fix(fixer) {
|
||||
return fixer.insertTextAfterRange(insertAfter, licenseHeader);
|
||||
},
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
export default {
|
||||
name: 'enforce-zod-schema',
|
||||
meta: {
|
||||
type: 'problem',
|
||||
docs: {
|
||||
description:
|
||||
'Disallow .nullable() and .object() in tool schemas. Use optional strings to represent complex objects.',
|
||||
},
|
||||
schema: [],
|
||||
messages: {
|
||||
noNullable:
|
||||
'Do not use .nullable() in tool schemas. Use .optional() instead.',
|
||||
noObject:
|
||||
'Do not use .object() in tool schemas. Represent complex objects as a short formatted string.',
|
||||
},
|
||||
},
|
||||
defaultOptions: [],
|
||||
create(context) {
|
||||
return {
|
||||
CallExpression(node) {
|
||||
if (
|
||||
node.callee.type !== 'MemberExpression' ||
|
||||
node.callee.property.type !== 'Identifier'
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
const methodName = node.callee.property.name;
|
||||
|
||||
// We don't validate that .nullable() is called on a ZodObject
|
||||
// specifically - this intentionally catches all .nullable() calls
|
||||
// in tool schema files.
|
||||
if (methodName === 'nullable') {
|
||||
context.report({
|
||||
node: node.callee.property,
|
||||
messageId: 'noNullable',
|
||||
});
|
||||
}
|
||||
|
||||
if (methodName === 'object') {
|
||||
// Only flag zod.object() calls, not arbitrary .object() calls.
|
||||
const obj = node.callee.object;
|
||||
if (
|
||||
obj.type === 'Identifier' &&
|
||||
(obj.name === 'zod' || obj.name === 'z')
|
||||
) {
|
||||
context.report({
|
||||
node: node.callee.property,
|
||||
messageId: 'noObject',
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import checkLicenseRule from './check-license-rule.js';
|
||||
import enforceZodSchemaRule from './enforce-zod-schema-rule.js';
|
||||
import noDirectThirdPartyImportsRule from './no-direct-third-party-imports-rule.js';
|
||||
|
||||
export default {
|
||||
rules: {
|
||||
'check-license': checkLicenseRule,
|
||||
'no-direct-third-party-imports': noDirectThirdPartyImportsRule,
|
||||
'enforce-zod-schema': enforceZodSchemaRule,
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* ESLint rule that prevents value (non-type) imports of third-party packages
|
||||
* that should go through the `src/third_party/index.ts` barrel file.
|
||||
*
|
||||
* Type-only imports are allowed because they are erased at compile time and
|
||||
* do not affect the bundle.
|
||||
*
|
||||
* This catches a class of bugs where a direct import works in development
|
||||
* (because devDependencies are installed) but fails once the package is
|
||||
* bundled and published via `npm pack`.
|
||||
*
|
||||
* The list of bundled packages is derived dynamically by scanning
|
||||
* `src/third_party/*.ts` for import/export statements at ESLint load time.
|
||||
*
|
||||
* See https://github.com/ChromeDevTools/chrome-devtools-mcp/issues/1123
|
||||
*/
|
||||
|
||||
import {readdirSync, readFileSync} from 'node:fs';
|
||||
import {join} from 'node:path';
|
||||
|
||||
const THIRD_PARTY_DIR = join(
|
||||
import.meta.dirname,
|
||||
'..',
|
||||
'..',
|
||||
'src',
|
||||
'third_party',
|
||||
);
|
||||
|
||||
/**
|
||||
* Parse all .ts files in src/third_party/ and extract the bare package names
|
||||
* from import/export statements. Relative imports and node_modules paths
|
||||
* (used for chrome-devtools-frontend) are skipped.
|
||||
*/
|
||||
function discoverBundledPackages() {
|
||||
const packages = new Set();
|
||||
// Match `from 'pkg'` (may appear on a different line than `import`)
|
||||
// and side-effect imports like `import 'pkg'`.
|
||||
const fromRe = /from\s+['"]([^'"]+)['"]/g;
|
||||
const sideEffectRe = /^import\s+['"]([^'"]+)['"]/gm;
|
||||
|
||||
let files;
|
||||
try {
|
||||
files = readdirSync(THIRD_PARTY_DIR).filter(f => f.endsWith('.ts'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const content = readFileSync(join(THIRD_PARTY_DIR, file), 'utf8');
|
||||
for (const re of [fromRe, sideEffectRe]) {
|
||||
re.lastIndex = 0;
|
||||
let match;
|
||||
while ((match = re.exec(content)) !== null) {
|
||||
const source = match[1];
|
||||
// Skip relative imports and node_modules paths.
|
||||
if (source.startsWith('.') || source.startsWith('/')) {
|
||||
continue;
|
||||
}
|
||||
// Extract the bare package name (handle scoped packages like @foo/bar).
|
||||
const parts = source.split('/');
|
||||
const pkg = source.startsWith('@')
|
||||
? parts.slice(0, 2).join('/')
|
||||
: parts[0];
|
||||
packages.add(pkg);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...packages];
|
||||
}
|
||||
|
||||
const THIRD_PARTY_PACKAGES = discoverBundledPackages();
|
||||
|
||||
/** Matches any import source that starts with one of the restricted packages. */
|
||||
function isRestrictedSource(source) {
|
||||
return THIRD_PARTY_PACKAGES.some(
|
||||
pkg => source === pkg || source.startsWith(pkg + '/'),
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns true when the file is inside src/third_party/. */
|
||||
function isThirdPartyBarrel(filename) {
|
||||
const normalized = filename.replace(/\\/g, '/');
|
||||
return normalized.includes('/src/third_party/');
|
||||
}
|
||||
|
||||
export default {
|
||||
name: 'no-direct-third-party-imports',
|
||||
meta: {
|
||||
type: 'problem',
|
||||
docs: {
|
||||
description:
|
||||
'Disallow value imports of bundled third-party packages outside of src/third_party/',
|
||||
},
|
||||
schema: [],
|
||||
messages: {
|
||||
noDirectImport:
|
||||
'Do not import "{{source}}" directly. Use the re-export from "src/third_party/index.js" instead so the import survives bundling. (Type-only imports are fine.)',
|
||||
},
|
||||
},
|
||||
defaultOptions: [],
|
||||
create(context) {
|
||||
const filename = context.filename;
|
||||
if (isThirdPartyBarrel(filename)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {
|
||||
ImportDeclaration(node) {
|
||||
// `import type { Foo } from '...'` is always safe.
|
||||
if (node.importKind === 'type') {
|
||||
return;
|
||||
}
|
||||
|
||||
const source = node.source.value;
|
||||
if (!isRestrictedSource(source)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// If every specifier is `type`, the import is still safe.
|
||||
// e.g. `import { type Foo, type Bar } from '...'`
|
||||
const hasValueSpecifier = node.specifiers.some(
|
||||
s => s.type !== 'ImportSpecifier' || s.importKind !== 'type',
|
||||
);
|
||||
|
||||
if (!hasValueSpecifier) {
|
||||
return;
|
||||
}
|
||||
|
||||
context.report({
|
||||
node,
|
||||
messageId: 'noDirectImport',
|
||||
data: {source},
|
||||
});
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import {pathToFileURL} from 'node:url';
|
||||
import {parseArgs} from 'node:util';
|
||||
|
||||
import {GoogleGenAI, mcpToTool} from '@google/genai';
|
||||
import {Client} from '@modelcontextprotocol/sdk/client/index.js';
|
||||
import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
|
||||
|
||||
import {TestServer} from '../build/tests/server.js';
|
||||
|
||||
const ROOT_DIR = path.resolve(import.meta.dirname, '..');
|
||||
const SCENARIOS_DIR = path.join(import.meta.dirname, 'eval_scenarios');
|
||||
const SKILL_PATH = path.join(ROOT_DIR, 'skills', 'chrome-devtools', 'SKILL.md');
|
||||
|
||||
import type {CapturedFunctionCall, TestScenario} from './eval_result.ts';
|
||||
import {Result} from './eval_result.ts';
|
||||
export type {CapturedFunctionCall, TestScenario};
|
||||
export {Result};
|
||||
|
||||
async function loadScenario(scenarioPath: string): Promise<TestScenario> {
|
||||
const module = await import(pathToFileURL(scenarioPath).href);
|
||||
if (!module.scenario) {
|
||||
throw new Error(
|
||||
`Scenario file ${scenarioPath} does not export a 'scenario' object.`,
|
||||
);
|
||||
}
|
||||
return module.scenario;
|
||||
}
|
||||
|
||||
async function runSingleScenario(
|
||||
scenarioPath: string,
|
||||
apiKey: string,
|
||||
server: TestServer,
|
||||
modelId: string,
|
||||
debug: boolean,
|
||||
includeSkill: boolean,
|
||||
extraServerArgs: string[] = [],
|
||||
): Promise<void> {
|
||||
const debugLog = (...args: unknown[]) => {
|
||||
if (debug) {
|
||||
console.log(...args);
|
||||
}
|
||||
};
|
||||
const absolutePath = path.resolve(scenarioPath);
|
||||
debugLog(
|
||||
`\n### Running Scenario: ${path.relative(ROOT_DIR, absolutePath)} ###`,
|
||||
);
|
||||
|
||||
let client: Client | undefined;
|
||||
let transport: StdioClientTransport | undefined;
|
||||
|
||||
try {
|
||||
const loadedScenario = await loadScenario(absolutePath);
|
||||
const scenario = {...loadedScenario};
|
||||
|
||||
// Prepend skill content if requested
|
||||
if (includeSkill) {
|
||||
if (!fs.existsSync(SKILL_PATH)) {
|
||||
throw new Error(
|
||||
`Skill file not found at ${SKILL_PATH}. Please ensure the skill file exists.`,
|
||||
);
|
||||
}
|
||||
const skillContent = fs.readFileSync(SKILL_PATH, 'utf-8');
|
||||
scenario.prompt = `${skillContent}\n\n---\n\n${scenario.prompt}`;
|
||||
}
|
||||
|
||||
// Append random queryid to avoid caching issues and test distinct runs
|
||||
const randomId = Math.floor(Math.random() * 1000000);
|
||||
scenario.prompt = `${scenario.prompt}\nqueryid=${randomId}`;
|
||||
|
||||
if (scenario.htmlRoute) {
|
||||
server.addHtmlRoute(
|
||||
scenario.htmlRoute.path,
|
||||
scenario.htmlRoute.htmlContent,
|
||||
);
|
||||
scenario.prompt = scenario.prompt.replace(
|
||||
'<TEST_URL>',
|
||||
server.getRoute(scenario.htmlRoute.path),
|
||||
);
|
||||
}
|
||||
|
||||
// Path to the compiled MCP server
|
||||
const serverPath = path.join(
|
||||
ROOT_DIR,
|
||||
'build/src/bin/chrome-devtools-mcp.js',
|
||||
);
|
||||
if (!fs.existsSync(serverPath)) {
|
||||
throw new Error(
|
||||
`MCP server not found at ${serverPath}. Please run 'npm run build' first.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Environment variables
|
||||
const env: Record<string, string> = {};
|
||||
Object.entries(process.env).forEach(([key, value]) => {
|
||||
if (value !== undefined) {
|
||||
env[key] = value;
|
||||
}
|
||||
});
|
||||
env['CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS'] = 'true';
|
||||
|
||||
const args = [serverPath];
|
||||
if (!debug) {
|
||||
args.push('--headless');
|
||||
}
|
||||
if (scenario.serverArgs) {
|
||||
args.push(...scenario.serverArgs);
|
||||
}
|
||||
if (extraServerArgs.length > 0) {
|
||||
args.push(...extraServerArgs);
|
||||
}
|
||||
|
||||
transport = new StdioClientTransport({
|
||||
command: 'node',
|
||||
args,
|
||||
env,
|
||||
stderr: debug ? 'inherit' : 'ignore',
|
||||
});
|
||||
|
||||
client = new Client(
|
||||
{name: 'gemini-eval-client', version: '1.0.0'},
|
||||
{capabilities: {}},
|
||||
);
|
||||
|
||||
await client.connect(transport);
|
||||
|
||||
const allCalls: CapturedFunctionCall[] = [];
|
||||
const originalCallTool = client.callTool.bind(client);
|
||||
client.callTool = async (request, schema) => {
|
||||
// NOTE: request.name is the original name as the MCP client sees it.
|
||||
// mcpToTool handles the conversion from Gemini sanitized name to original name.
|
||||
debugLog(
|
||||
`Executing tool: ${request.name} with args: ${JSON.stringify(request.arguments)}`,
|
||||
);
|
||||
allCalls.push({
|
||||
name: request.name,
|
||||
args: (request.arguments as Record<string, unknown>) || {},
|
||||
});
|
||||
const response = await originalCallTool(request, schema);
|
||||
debugLog(`Tool response: ${JSON.stringify(response)}`);
|
||||
return response;
|
||||
};
|
||||
|
||||
const ai = new GoogleGenAI({apiKey});
|
||||
|
||||
debugLog(`\n--- Prompt ---\n${scenario.prompt}`);
|
||||
|
||||
const result = await ai.models.generateContent({
|
||||
model: modelId,
|
||||
contents: scenario.prompt,
|
||||
config: {
|
||||
tools: [mcpToTool(client)],
|
||||
automaticFunctionCalling: {
|
||||
maximumRemoteCalls: scenario.maxTurns,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
debugLog(`\n--- Response ---\n${result.text}`);
|
||||
|
||||
debugLog('\nVerifying expectations...');
|
||||
scenario.expectations(new Result(allCalls, args));
|
||||
} finally {
|
||||
try {
|
||||
await client?.close();
|
||||
} catch (e) {
|
||||
console.error('Error closing client:', e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const apiKey = process.env.GEMINI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error('GEMINI_API_KEY environment variable is required.');
|
||||
}
|
||||
|
||||
const {values, positionals} = parseArgs({
|
||||
options: {
|
||||
model: {
|
||||
type: 'string',
|
||||
default: 'gemini-3-flash-preview',
|
||||
},
|
||||
debug: {
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
repeat: {
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
'include-skill': {
|
||||
type: 'boolean',
|
||||
default: false,
|
||||
},
|
||||
'server-args': {
|
||||
type: 'string',
|
||||
},
|
||||
},
|
||||
allowPositionals: true,
|
||||
});
|
||||
|
||||
const modelId = values.model;
|
||||
const debug = values.debug;
|
||||
const repeat = values.repeat;
|
||||
const includeSkill = values['include-skill'];
|
||||
const extraServerArgs = values['server-args']
|
||||
? values['server-args'].split(/\s+/)
|
||||
: [];
|
||||
|
||||
const scenarioFiles =
|
||||
positionals.length > 0
|
||||
? positionals.map(p => path.resolve(p))
|
||||
: fs
|
||||
.readdirSync(SCENARIOS_DIR)
|
||||
.filter(file => file.endsWith('.ts') || file.endsWith('.js'))
|
||||
.map(file => path.join(SCENARIOS_DIR, file));
|
||||
|
||||
const server = new TestServer(TestServer.randomPort());
|
||||
await server.start();
|
||||
|
||||
let successCount = 0;
|
||||
let failureCount = 0;
|
||||
|
||||
try {
|
||||
for (const scenarioPath of scenarioFiles) {
|
||||
for (let i = 1; i <= (repeat ? 3 : 1); i++) {
|
||||
try {
|
||||
if (debug) {
|
||||
console.log(
|
||||
`Running scenario: ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i}/3)`,
|
||||
);
|
||||
}
|
||||
await runSingleScenario(
|
||||
scenarioPath,
|
||||
apiKey,
|
||||
server,
|
||||
modelId,
|
||||
debug,
|
||||
includeSkill,
|
||||
extraServerArgs,
|
||||
);
|
||||
console.log(`✔ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`);
|
||||
successCount++;
|
||||
} catch (e) {
|
||||
console.error(
|
||||
`✖ ${path.relative(ROOT_DIR, scenarioPath)} (Run ${i})`,
|
||||
);
|
||||
console.error(e);
|
||||
failureCount++;
|
||||
} finally {
|
||||
server.restore();
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
|
||||
console.log(`\nSummary: ${successCount} passed, ${failureCount} failed`);
|
||||
|
||||
if (failureCount > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
export interface CapturedFunctionCall {
|
||||
name: string;
|
||||
args: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export class Result {
|
||||
private nextCallIndex = 0;
|
||||
public readonly calls: CapturedFunctionCall[];
|
||||
public readonly serverArgs: string[];
|
||||
|
||||
constructor(calls: CapturedFunctionCall[], serverArgs: string[]) {
|
||||
this.calls = calls;
|
||||
this.serverArgs = serverArgs;
|
||||
}
|
||||
|
||||
get hasPageIdRouting(): boolean {
|
||||
return this.serverArgs.includes('--experimental-page-id-routing');
|
||||
}
|
||||
|
||||
get remainingCalls(): CapturedFunctionCall[] {
|
||||
return this.calls.slice(this.nextCallIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumes initial page navigation/setup boilerplate.
|
||||
* - Ignores/skips leading list_pages calls.
|
||||
* - Asserts that new_page or navigate_page was called.
|
||||
* - Determines the expected pageId.
|
||||
* - Returns the active pageId.
|
||||
*/
|
||||
consumePageNavigation(): number | undefined {
|
||||
if (this.calls[this.nextCallIndex]?.name === 'list_pages') {
|
||||
this.nextCallIndex++;
|
||||
}
|
||||
|
||||
const navCall = this.calls[this.nextCallIndex];
|
||||
assert.ok(
|
||||
navCall &&
|
||||
(navCall.name === 'new_page' || navCall.name === 'navigate_page'),
|
||||
`Expected navigation call (new_page or navigate_page), but got: ${navCall?.name || 'none'}`,
|
||||
);
|
||||
this.nextCallIndex++;
|
||||
|
||||
const isNewPage = navCall.name === 'new_page';
|
||||
let pageId: number | undefined;
|
||||
if (this.hasPageIdRouting) {
|
||||
pageId = isNewPage ? 2 : 1;
|
||||
}
|
||||
|
||||
return pageId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Asserts that the next call in sequence has the correct name and matches expected arguments.
|
||||
* Increments the internal call index.
|
||||
*/
|
||||
assertNextCall(
|
||||
name: string,
|
||||
expectedArgs?: Record<string, unknown>,
|
||||
): CapturedFunctionCall {
|
||||
const call = this.calls[this.nextCallIndex];
|
||||
assert.ok(
|
||||
call,
|
||||
`Expected call at index ${this.nextCallIndex} (name: '${name}') to exist`,
|
||||
);
|
||||
assert.strictEqual(
|
||||
call.name,
|
||||
name,
|
||||
`Expected call at index ${this.nextCallIndex} to be '${name}', but got '${call.name}'`,
|
||||
);
|
||||
|
||||
if (expectedArgs) {
|
||||
for (const entry of Object.entries(expectedArgs)) {
|
||||
const key = entry[0];
|
||||
const value = entry[1];
|
||||
assert.deepStrictEqual(
|
||||
call.args[key],
|
||||
value,
|
||||
`Expected argument '${key}' on call '${name}' to be ${JSON.stringify(value)}, got ${JSON.stringify(call.args[key])}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
this.nextCallIndex++;
|
||||
return call;
|
||||
}
|
||||
}
|
||||
|
||||
export interface TestScenario {
|
||||
prompt: string;
|
||||
maxTurns: number;
|
||||
expectations: (result: Result) => void;
|
||||
htmlRoute?: {
|
||||
path: string;
|
||||
htmlContent: string;
|
||||
};
|
||||
/** Extra CLI flags passed to the MCP server (e.g. '--experimental-page-id-routing'). */
|
||||
serverArgs?: string[];
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Navigate to <TEST_URL> and check the console messages.',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/console_test.html',
|
||||
htmlContent: `
|
||||
<script>
|
||||
console.log('Test log message');
|
||||
console.error('Test error message');
|
||||
</script>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.strictEqual(result.remainingCalls.length, 1);
|
||||
result.assertNextCall(
|
||||
'list_console_messages',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Emulate offline network conditions.',
|
||||
maxTurns: 2,
|
||||
expectations: result => {
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall('list_pages');
|
||||
result.assertNextCall('emulate', {
|
||||
networkConditions: 'Offline',
|
||||
pageId: result.hasPageIdRouting ? 1 : undefined,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Emulate current page with iPhone 14 user agent',
|
||||
maxTurns: 2,
|
||||
expectations: result => {
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
if (
|
||||
result.hasPageIdRouting ||
|
||||
result.remainingCalls[0]?.name === 'list_pages'
|
||||
) {
|
||||
result.assertNextCall('list_pages');
|
||||
}
|
||||
result.assertNextCall('emulate', {
|
||||
userAgent:
|
||||
'Mozilla/5.0 (iPhone; CPU iPhone OS 16_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1',
|
||||
pageId: result.hasPageIdRouting ? 1 : undefined,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Emulate current page with iPhone 14 viewport',
|
||||
maxTurns: 2,
|
||||
expectations: result => {
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
if (
|
||||
result.hasPageIdRouting ||
|
||||
result.remainingCalls[0]?.name === 'list_pages'
|
||||
) {
|
||||
result.assertNextCall('list_pages');
|
||||
}
|
||||
result.assertNextCall('emulate', {
|
||||
viewport: '390x844x3,mobile,touch',
|
||||
pageId: result.hasPageIdRouting ? 1 : undefined,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Go to <TEST_URL>, fill the form with size = 2 CPUs and components = [docker, nginx].',
|
||||
maxTurns: 5, // allow for at least one extra turn to verify there are no extra clicks after fill_form
|
||||
htmlRoute: {
|
||||
path: '/input_test.html',
|
||||
htmlContent: `
|
||||
<form action="/post" method="POST">
|
||||
<div>
|
||||
<label for="size">CPU/Memory size:</label>
|
||||
<select id="size" name="size" required>
|
||||
<option value="small">1 vCPU, 2GB RAM</option>
|
||||
<option value="medium">2 vCPU, 4GB RAM</option>
|
||||
<option value="large">4 vCPU, 8GB RAM</option>
|
||||
</select>
|
||||
</div>
|
||||
<br>
|
||||
<div>
|
||||
<p>Pre-installed components:</p>
|
||||
<input type="checkbox" id="docker" name="components" value="docker">
|
||||
<label for="docker">Docker</label><br>
|
||||
<input type="checkbox" id="nodejs" name="components" value="nodejs">
|
||||
<label for="nodejs">Node.js</label><br>
|
||||
<input type="checkbox" id="python" name="components" value="python">
|
||||
<label for="python">Python</label><br>
|
||||
<input type="checkbox" id="nginx" name="components" value="nginx">
|
||||
<label for="nginx">Nginx</label>
|
||||
</div>
|
||||
<button type="submit">Spawn Server</button>
|
||||
</form>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 2, 'Not enough calls made');
|
||||
result.assertNextCall(
|
||||
'take_snapshot',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
const fillFormCall = result.assertNextCall(
|
||||
'fill_form',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
|
||||
const elements = fillFormCall.args.elements;
|
||||
assert.ok(Array.isArray(elements), 'elements should be an array');
|
||||
assert.strictEqual(
|
||||
elements.length,
|
||||
3,
|
||||
'fill_form should be used with all form elements at once',
|
||||
);
|
||||
|
||||
const typedElements = elements.map(e => {
|
||||
assert.ok(e && typeof e === 'object' && 'uid' in e && 'value' in e);
|
||||
return {
|
||||
uid: String(e.uid),
|
||||
value: String(e.value),
|
||||
};
|
||||
});
|
||||
|
||||
const uids = new Set(typedElements.map(e => e.uid));
|
||||
assert.strictEqual(
|
||||
uids.size,
|
||||
3,
|
||||
'fill_form should target three distinct elements',
|
||||
);
|
||||
|
||||
const values = typedElements.map(e => e.value).sort();
|
||||
assert.deepStrictEqual(
|
||||
values,
|
||||
['2 vCPU, 4GB RAM', 'true', 'true'],
|
||||
'fill_form should be used with correct values',
|
||||
);
|
||||
|
||||
const submitUid = '1_15';
|
||||
|
||||
const extraFormInteractions = result.remainingCalls.filter(
|
||||
c => ['fill', 'click'].includes(c.name) && c.args.uid !== submitUid,
|
||||
);
|
||||
assert.deepEqual(
|
||||
extraFormInteractions.length,
|
||||
0,
|
||||
'No extra clicks and fills after fill_form',
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Eval scenario: user asks to fix issues with their webpage (no URL given).
|
||||
* When no URL is provided, the model should pick the current frontend and run
|
||||
* and inspect it. Verifies the MCP server is invoked and the model opens the
|
||||
* frontend and inspects it (snapshot, console, or network).
|
||||
*
|
||||
* Note: Tools like performance_start_trace, take_snapshot, list_console_messages,
|
||||
* and list_network_requests do not require a URL in the prompt—they operate on
|
||||
* the currently selected page. Only navigate_page/new_page need a URL to open
|
||||
* a page; the eval runner injects the test URL when htmlRoute is set.
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
const INSPECTION_TOOLS = [
|
||||
'take_snapshot',
|
||||
'list_console_messages',
|
||||
'list_network_requests',
|
||||
];
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Can you fix issues with my webpage at <TEST_URL>?',
|
||||
maxTurns: 4,
|
||||
htmlRoute: {
|
||||
path: '/fix_issues_test.html',
|
||||
htmlContent: `
|
||||
<h1>Test Page</h1>
|
||||
<p>Some content</p>
|
||||
<script>
|
||||
console.error('Intentional error for testing');
|
||||
</script>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
const inspectionCalls = result.remainingCalls.filter(c =>
|
||||
INSPECTION_TOOLS.includes(c.name),
|
||||
);
|
||||
assert.ok(
|
||||
inspectionCalls.length >= 1,
|
||||
`Expected at least one inspection tool (${INSPECTION_TOOLS.join(', ')}) after navigation, got: ${result.remainingCalls.map(c => c.name).join(', ')}`,
|
||||
);
|
||||
if (result.hasPageIdRouting) {
|
||||
for (const inspectionCall of inspectionCalls) {
|
||||
assert.strictEqual(
|
||||
inspectionCall.args.pageId,
|
||||
pageId,
|
||||
`Inspection call ${inspectionCall.name} should target pageId: ${pageId}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*
|
||||
* Eval scenario using "website"/"webpage" wording to verify the model invokes
|
||||
* the right tools when users ask to open a site and read its content.
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Open the website at <TEST_URL> and tell me what content is on the page.',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/frontend_snapshot.html',
|
||||
htmlContent: '<h1>Frontend Test</h1><p>This is a test webpage.</p>',
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall(
|
||||
'take_snapshot',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Go to <TEST_URL>, fill the input with "hello world" and click the button five times in parallel without using a script.',
|
||||
maxTurns: 10,
|
||||
htmlRoute: {
|
||||
path: '/input_test.html',
|
||||
htmlContent: `
|
||||
<input type="text" id="test-input" />
|
||||
<button id="test-button">Submit</button>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 7);
|
||||
result.assertNextCall(
|
||||
'take_snapshot',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
result.assertNextCall(
|
||||
'fill',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
for (let i = 2; i < 7; i++) {
|
||||
result.assertNextCall('click', {
|
||||
includeSnapshot: undefined,
|
||||
pageId: result.hasPageIdRouting ? pageId : undefined,
|
||||
});
|
||||
}
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,38 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Go to <TEST_URL>, fill the input with "hello world" and click the button.',
|
||||
maxTurns: 5,
|
||||
htmlRoute: {
|
||||
path: '/input_test.html',
|
||||
htmlContent: `
|
||||
<input type="text" id="test-input" />
|
||||
<button id="test-button">Submit</button>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 3);
|
||||
result.assertNextCall(
|
||||
'take_snapshot',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
result.assertNextCall(
|
||||
'fill',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
result.assertNextCall(
|
||||
'click',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Create a new page <TEST_URL> in an isolated context called contextB. Take a screenshot there.',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/isolated_context.html',
|
||||
htmlContent: '<h1>Isolated Context</h1>',
|
||||
},
|
||||
expectations: result => {
|
||||
result.assertNextCall('new_page', {isolatedContext: 'contextB'});
|
||||
result.assertNextCall(
|
||||
'take_screenshot',
|
||||
result.hasPageIdRouting ? {pageId: 2} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Check a11y issues on the page at <TEST_URL>',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/lighthouse_test.html',
|
||||
htmlContent: `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Lighthouse Audit Test</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Lighthouse Audit Test</h1>
|
||||
<p>This is a valid test page for running audits.</p>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall(
|
||||
'lighthouse_audit',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,37 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Check for best practices on the page at <TEST_URL>',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/lighthouse_test.html',
|
||||
htmlContent: `
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Lighthouse Audit Test</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Lighthouse Audit Test</h1>
|
||||
<p>This is a valid test page for running audits.</p>
|
||||
</body>
|
||||
</html>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall(
|
||||
'lighthouse_audit',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Navigate in current page to https://developers.chrome.com and tell me if it worked.',
|
||||
maxTurns: 2,
|
||||
expectations: result => {
|
||||
if (result.hasPageIdRouting) {
|
||||
result.assertNextCall('list_pages');
|
||||
}
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall('navigate_page', {
|
||||
url: 'https://developers.chrome.com',
|
||||
pageId: result.hasPageIdRouting ? 1 : undefined,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Navigate to <TEST_URL> and list all network requests.',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/network_test.html',
|
||||
htmlContent: `
|
||||
<h1>Network Test</h1>
|
||||
<script>
|
||||
fetch('/network_test.html'); // Self fetch to ensure at least one request
|
||||
</script>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall(
|
||||
'list_network_requests',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,66 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
serverArgs: ['--experimental-page-id-routing'],
|
||||
prompt: `Open two pages in the same isolated context "session":
|
||||
- Page 1 at data:text/html,<textarea id="ta"></textarea>
|
||||
- Page 2 at data:text/html,<h1>Other</h1>
|
||||
|
||||
Now use the press_key tool to type "a" on Page 1 without selecting it first. You must use press_key, not fill or type_text. If you encounter any errors, recover from them.`,
|
||||
maxTurns: 10,
|
||||
expectations: result => {
|
||||
// Should open 2 pages in the same context.
|
||||
const newPages = result.calls.filter(c => c.name === 'new_page');
|
||||
assert.strictEqual(newPages.length, 2, 'Should open 2 pages');
|
||||
assert.strictEqual(newPages[0].args.isolatedContext, 'session');
|
||||
assert.strictEqual(newPages[1].args.isolatedContext, 'session');
|
||||
|
||||
// Should attempt press_key at least once.
|
||||
const pressKeys = result.calls.filter(c => c.name === 'press_key');
|
||||
assert.ok(pressKeys.length >= 1, 'Should attempt press_key at least once');
|
||||
for (const pk of pressKeys) {
|
||||
assert.strictEqual(
|
||||
pk.args.pageId,
|
||||
2,
|
||||
'press_key should target Page 1 (pageId: 2)',
|
||||
);
|
||||
}
|
||||
|
||||
const selectPages = result.calls.filter(c => c.name === 'select_page');
|
||||
|
||||
if (selectPages.length > 0) {
|
||||
const firstPressKeyIndex = result.calls.indexOf(pressKeys[0]);
|
||||
const firstSelectPageIndex = result.calls.indexOf(selectPages[0]);
|
||||
|
||||
if (firstPressKeyIndex < firstSelectPageIndex) {
|
||||
// Error path: press_key was attempted first and failed.
|
||||
// Verify recovery: must have a second press_key after select_page.
|
||||
assert.ok(
|
||||
pressKeys.length >= 2,
|
||||
'Should retry press_key after error recovery',
|
||||
);
|
||||
const lastPressKeyIndex = result.calls.lastIndexOf(pressKeys.at(-1)!);
|
||||
assert.ok(
|
||||
firstSelectPageIndex < lastPressKeyIndex,
|
||||
'select_page should precede the successful press_key',
|
||||
);
|
||||
} else {
|
||||
// Proactive path: model selected page first.
|
||||
assert.ok(
|
||||
firstSelectPageIndex < firstPressKeyIndex,
|
||||
'select_page should precede press_key',
|
||||
);
|
||||
}
|
||||
}
|
||||
// If no select_page was called, the model found another recovery path.
|
||||
// This is acceptable as long as press_key was attempted.
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
serverArgs: ['--experimental-page-id-routing'],
|
||||
prompt: `Open two new pages in isolated contexts:
|
||||
- Page A (isolatedContext "contextA") at data:text/html,<button>Click A</button>
|
||||
- Page B (isolatedContext "contextB") at data:text/html,<button>Click B</button>
|
||||
Then take a snapshot of Page A, take a snapshot of Page B, and then click the button on Page A.`,
|
||||
maxTurns: 12,
|
||||
expectations: result => {
|
||||
// Should have 2 new_page calls with isolatedContext.
|
||||
const newPages = result.calls.filter(c => c.name === 'new_page');
|
||||
assert.strictEqual(newPages.length, 2, 'Should open 2 pages');
|
||||
for (const np of newPages) {
|
||||
assert.strictEqual(
|
||||
typeof np.args.isolatedContext,
|
||||
'string',
|
||||
'new_page should use isolatedContext',
|
||||
);
|
||||
}
|
||||
|
||||
// Should have at least 2 take_snapshot calls (one per page).
|
||||
// The model may use pageId directly or select_page before each snapshot.
|
||||
const snapshots = result.calls.filter(c => c.name === 'take_snapshot');
|
||||
assert.ok(snapshots.length >= 2, 'Should take at least 2 snapshots');
|
||||
const snapshotPageIds = snapshots.map(s => s.args.pageId);
|
||||
assert.ok(
|
||||
snapshotPageIds.includes(2),
|
||||
'Should snapshot Page A (pageId: 2)',
|
||||
);
|
||||
assert.ok(
|
||||
snapshotPageIds.includes(3),
|
||||
'Should snapshot Page B (pageId: 3)',
|
||||
);
|
||||
|
||||
// Should have a click call (resolving uid from Page A's snapshot
|
||||
// even though Page B was snapshotted after).
|
||||
const clicks = result.calls.filter(c => c.name === 'click');
|
||||
assert.ok(clicks.length >= 1, 'Should click the button on Page A');
|
||||
assert.strictEqual(
|
||||
clicks[0].args.pageId,
|
||||
2,
|
||||
'Click should target Page A (pageId: 2)',
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Check the performance of https://developers.chrome.com',
|
||||
maxTurns: 3,
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.ok(result.remainingCalls.length >= 1);
|
||||
result.assertNextCall(
|
||||
'performance_start_trace',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,27 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt:
|
||||
'Open new page <TEST_URL> and then open new page https://developers.chrome.com. Select the <TEST_URL> page.',
|
||||
maxTurns: 3,
|
||||
htmlRoute: {
|
||||
path: '/test.html',
|
||||
htmlContent: `
|
||||
<h1>test</h1>
|
||||
`,
|
||||
},
|
||||
expectations: result => {
|
||||
result.consumePageNavigation();
|
||||
assert.strictEqual(result.remainingCalls.length, 2);
|
||||
result.assertNextCall('new_page');
|
||||
result.assertNextCall('select_page', {pageId: 2, bringToFront: undefined});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
|
||||
import type {TestScenario} from '../eval_gemini.ts';
|
||||
|
||||
export const scenario: TestScenario = {
|
||||
prompt: 'Read the content of <TEST_URL>',
|
||||
maxTurns: 4,
|
||||
htmlRoute: {
|
||||
path: '/test.html',
|
||||
htmlContent: '<h1>Hello World</h1><p>This is a test.</p>',
|
||||
},
|
||||
expectations: result => {
|
||||
const pageId = result.consumePageNavigation();
|
||||
assert.strictEqual(result.remainingCalls.length, 1);
|
||||
result.assertNextCall(
|
||||
'take_snapshot',
|
||||
result.hasPageIdRouting ? {pageId} : undefined,
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import {Client} from '@modelcontextprotocol/sdk/client/index.js';
|
||||
import {StdioClientTransport} from '@modelcontextprotocol/sdk/client/stdio.js';
|
||||
|
||||
import {parseArguments} from '../build/src/bin/chrome-devtools-mcp-cli-options.js';
|
||||
import {buildFlag} from '../build/src/index.js';
|
||||
import {
|
||||
labels,
|
||||
ToolCategory,
|
||||
OFF_BY_DEFAULT_CATEGORIES,
|
||||
} from '../build/src/tools/categories.js';
|
||||
import {createTools} from '../build/src/tools/tools.js';
|
||||
|
||||
const OUTPUT_PATH = path.join(
|
||||
import.meta.dirname,
|
||||
'../src/bin/chrome-devtools-cli-options.ts',
|
||||
);
|
||||
|
||||
async function fetchTools() {
|
||||
console.log('Connecting to chrome-devtools-mcp to fetch tools...');
|
||||
// Use the local build of the server
|
||||
const serverPath = path.join(
|
||||
import.meta.dirname,
|
||||
'../build/src/bin/chrome-devtools-mcp.js',
|
||||
);
|
||||
|
||||
const transport = new StdioClientTransport({
|
||||
command: 'node',
|
||||
args: [serverPath, '--viaCli'],
|
||||
env: {...process.env, CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS: 'true'},
|
||||
});
|
||||
|
||||
const client = new Client(
|
||||
{
|
||||
name: 'chrome-devtools-cli-generator',
|
||||
version: '0.1.0',
|
||||
},
|
||||
{
|
||||
capabilities: {},
|
||||
},
|
||||
);
|
||||
|
||||
await client.connect(transport);
|
||||
try {
|
||||
const toolsResponse = await client.listTools();
|
||||
if (!toolsResponse.tools?.length) {
|
||||
throw new Error(`No tools were fetched`);
|
||||
}
|
||||
const tools = toolsResponse.tools || [];
|
||||
console.log(`Fetched ${tools.length} tools`);
|
||||
return tools;
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
}
|
||||
|
||||
interface CliOption {
|
||||
name: string;
|
||||
type: string;
|
||||
description: string;
|
||||
required: boolean;
|
||||
default?: unknown;
|
||||
enum?: unknown[];
|
||||
}
|
||||
|
||||
interface JsonSchema {
|
||||
type?: string | string[];
|
||||
description?: string;
|
||||
properties?: Record<string, JsonSchema>;
|
||||
required?: string[];
|
||||
default?: unknown;
|
||||
enum?: unknown[];
|
||||
}
|
||||
|
||||
function schemaToCLIOptions(schema: JsonSchema): CliOption[] {
|
||||
if (!schema || !schema.properties) {
|
||||
return [];
|
||||
}
|
||||
const required = schema.required || [];
|
||||
const properties = schema.properties;
|
||||
return Object.entries(properties).map(([name, prop]) => {
|
||||
const isRequired = required.includes(name);
|
||||
const description = prop.description || '';
|
||||
if (typeof prop.type !== 'string') {
|
||||
throw new Error(
|
||||
`Property ${name} has a complex type not supported by CLI.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
name,
|
||||
type: prop.type,
|
||||
description,
|
||||
required: isRequired,
|
||||
default: prop.default,
|
||||
enum: prop.enum,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function generateCli() {
|
||||
const tools = await fetchTools();
|
||||
|
||||
const staticTools = createTools(parseArguments());
|
||||
const toolNameToCategoryEnum = new Map<string, string>();
|
||||
const toolNameToConditions = new Map<string, string[]>();
|
||||
|
||||
for (const tool of staticTools) {
|
||||
toolNameToCategoryEnum.set(tool.name, tool.annotations.category);
|
||||
toolNameToConditions.set(tool.name, tool.annotations.conditions || []);
|
||||
}
|
||||
|
||||
// Sort tools by name
|
||||
const sortedTools = tools
|
||||
.sort((a, b) => a.name.localeCompare(b.name))
|
||||
.filter(tool => {
|
||||
// Skipping fill_form because it is not relevant in shell scripts
|
||||
// and CLI does not handle array/JSON args well.
|
||||
if (tool.name === 'fill_form') {
|
||||
return false;
|
||||
}
|
||||
// Skipping wait_for because CLI does not handle array/JSON args well
|
||||
// and shell scripts have many mechanisms for waiting.
|
||||
if (tool.name === 'wait_for') {
|
||||
return false;
|
||||
}
|
||||
// Skipping get_tab_id as it is for internal integrations
|
||||
if (tool.name === 'get_tab_id') {
|
||||
return false;
|
||||
}
|
||||
// Skipping in_page tools as they are not launched yet
|
||||
if (toolNameToCategoryEnum.get(tool.name) === ToolCategory.IN_PAGE) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
const commands: Record<
|
||||
string,
|
||||
{description: string; category: string; args: Record<string, CliOption>}
|
||||
> = {};
|
||||
|
||||
for (const tool of sortedTools) {
|
||||
const options = schemaToCLIOptions(tool.inputSchema);
|
||||
const args: Record<string, CliOption> = {};
|
||||
for (const opt of options) {
|
||||
args[opt.name] = opt;
|
||||
}
|
||||
|
||||
const categoryEnum = toolNameToCategoryEnum.get(tool.name);
|
||||
if (!categoryEnum) {
|
||||
throw new Error(`Tool ${tool.name} has no category.`);
|
||||
}
|
||||
const category = labels[categoryEnum as unknown as keyof typeof labels];
|
||||
if (!tool.description) {
|
||||
throw new Error(`Tool ${tool.name} is missing description`);
|
||||
}
|
||||
|
||||
let description = tool.description;
|
||||
const requiredFlags: string[] = [];
|
||||
|
||||
const isOffByDefault = OFF_BY_DEFAULT_CATEGORIES.includes(categoryEnum);
|
||||
if (isOffByDefault) {
|
||||
const categoryFlag = buildFlag(categoryEnum);
|
||||
requiredFlags.push(`--${categoryFlag}=true`);
|
||||
}
|
||||
|
||||
const conditions = toolNameToConditions.get(tool.name) || [];
|
||||
for (const condition of conditions) {
|
||||
requiredFlags.push(`--${condition}=true`);
|
||||
}
|
||||
|
||||
if (requiredFlags.length > 0) {
|
||||
description += ` (requires flag: ${requiredFlags.join(', ')})`;
|
||||
}
|
||||
|
||||
commands[tool.name] = {
|
||||
description,
|
||||
category,
|
||||
args,
|
||||
};
|
||||
}
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(`/**
|
||||
* @license
|
||||
* Copyright ${new Date().getFullYear()} Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
// NOTE: do not edit manually. Auto-generated by 'npm run cli:generate'.
|
||||
|
||||
export interface ArgDef {
|
||||
name: string;
|
||||
type: string;
|
||||
description: string;
|
||||
required: boolean;
|
||||
default?: string | number | boolean;
|
||||
enum?: ReadonlyArray<string | number>;
|
||||
}
|
||||
export type Commands = Record<
|
||||
string,
|
||||
{
|
||||
description: string;
|
||||
category: string;
|
||||
args: Record<string, ArgDef>
|
||||
}
|
||||
>;
|
||||
export const commands: Commands = ${JSON.stringify(commands, null, 2)} as const;
|
||||
`);
|
||||
|
||||
fs.mkdirSync(path.dirname(OUTPUT_PATH), {recursive: true});
|
||||
fs.writeFileSync(OUTPUT_PATH, lines.join(''));
|
||||
console.log(`Generated CLI at ${OUTPUT_PATH}`);
|
||||
}
|
||||
|
||||
generateCli().catch(err => {
|
||||
console.error('Error during generation:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,558 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import fs from 'node:fs';
|
||||
|
||||
import type {Tool} from '@modelcontextprotocol/sdk/types.js';
|
||||
|
||||
import {cliOptions} from '../build/src/bin/chrome-devtools-mcp-cli-options.js';
|
||||
import type {ParsedArguments} from '../build/src/bin/chrome-devtools-mcp-cli-options.js';
|
||||
import {buildFlag} from '../build/src/index.js';
|
||||
import {
|
||||
ToolCategory,
|
||||
OFF_BY_DEFAULT_CATEGORIES,
|
||||
labels,
|
||||
} from '../build/src/tools/categories.js';
|
||||
import {createTools} from '../build/src/tools/tools.js';
|
||||
|
||||
const OUTPUT_PATH = './docs/tool-reference.md';
|
||||
const SLIM_OUTPUT_PATH = './docs/slim-tool-reference.md';
|
||||
const README_PATH = './README.md';
|
||||
|
||||
// Extend the MCP Tool type to include our annotations
|
||||
interface ToolWithAnnotations extends Tool {
|
||||
annotations?: {
|
||||
title?: string;
|
||||
category?: typeof ToolCategory;
|
||||
conditions?: string[];
|
||||
};
|
||||
}
|
||||
|
||||
interface ZodCheck {
|
||||
kind: string;
|
||||
}
|
||||
|
||||
interface ZodDef {
|
||||
typeName: string;
|
||||
checks?: ZodCheck[];
|
||||
values?: string[];
|
||||
type?: ZodSchema;
|
||||
innerType?: ZodSchema;
|
||||
schema?: ZodSchema;
|
||||
defaultValue?: () => unknown;
|
||||
}
|
||||
|
||||
interface ZodSchema {
|
||||
_def: ZodDef;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
interface TypeInfo {
|
||||
type: string;
|
||||
enum?: string[];
|
||||
items?: TypeInfo;
|
||||
description?: string;
|
||||
default?: unknown;
|
||||
}
|
||||
|
||||
function escapeHtmlTags(text: string): string {
|
||||
return text
|
||||
.replace(/&(?![a-zA-Z]+;)/g, '&')
|
||||
.replace(/<([a-zA-Z][^>]*)>/g, '<$1>');
|
||||
}
|
||||
|
||||
function addCrossLinks(text: string, tools: ToolWithAnnotations[]): string {
|
||||
let result = text;
|
||||
|
||||
// Create a set of all tool names for efficient lookup
|
||||
const toolNames = new Set(tools.map(tool => tool.name));
|
||||
|
||||
// Sort tool names by length (descending) to match longer names first
|
||||
const sortedToolNames = Array.from(toolNames).sort(
|
||||
(a, b) => b.length - a.length,
|
||||
);
|
||||
|
||||
for (const toolName of sortedToolNames) {
|
||||
// Create regex to match tool name (case insensitive, word boundaries)
|
||||
const regex = new RegExp(`\\b${toolName}\\b`, 'gi');
|
||||
|
||||
result = result.replace(regex, match => {
|
||||
// Only create link if the match isn't already inside a link
|
||||
if (result.indexOf(`[${match}]`) !== -1) {
|
||||
return match; // Already linked
|
||||
}
|
||||
const anchorLink = toolName.toLowerCase();
|
||||
return `[\`${match}\`](#${anchorLink})`;
|
||||
});
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function sortTools(a: ToolWithAnnotations, b: ToolWithAnnotations): number {
|
||||
const aHasConditions = Boolean(a.annotations?.conditions?.length > 0);
|
||||
const bHasConditions = Boolean(b.annotations?.conditions?.length > 0);
|
||||
|
||||
if (aHasConditions && !bHasConditions) {
|
||||
return 1;
|
||||
}
|
||||
if (!aHasConditions && bHasConditions) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
return a.name.localeCompare(b.name);
|
||||
}
|
||||
|
||||
function generateToolsTOC(
|
||||
categories: Record<string, ToolWithAnnotations[]>,
|
||||
sortedCategories: string[],
|
||||
): string {
|
||||
let toc = '';
|
||||
|
||||
for (const category of sortedCategories) {
|
||||
const categoryTools = categories[category];
|
||||
const categoryName = labels[category];
|
||||
toc += `- **${categoryName}** (${categoryTools.length} tools)\n`;
|
||||
|
||||
// Sort tools within category for TOC
|
||||
categoryTools.sort(sortTools);
|
||||
for (const tool of categoryTools) {
|
||||
const anchorLink = tool.name.toLowerCase();
|
||||
toc += ` - [\`${tool.name}\`](docs/tool-reference.md#${anchorLink})\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return toc;
|
||||
}
|
||||
|
||||
function updateReadmeWithToolsTOC(toolsTOC: string): void {
|
||||
const readmeContent = fs.readFileSync(README_PATH, 'utf8');
|
||||
|
||||
const beginMarker = '<!-- BEGIN AUTO GENERATED TOOLS -->';
|
||||
const endMarker = '<!-- END AUTO GENERATED TOOLS -->';
|
||||
|
||||
const beginIndex = readmeContent.indexOf(beginMarker);
|
||||
const endIndex = readmeContent.indexOf(endMarker);
|
||||
|
||||
if (beginIndex === -1 || endIndex === -1) {
|
||||
console.warn('Could not find auto-generated tools markers in README.md');
|
||||
return;
|
||||
}
|
||||
|
||||
const before = readmeContent.substring(0, beginIndex + beginMarker.length);
|
||||
const after = readmeContent.substring(endIndex);
|
||||
|
||||
const updatedContent = before + '\n\n' + toolsTOC + '\n' + after;
|
||||
|
||||
fs.writeFileSync(README_PATH, updatedContent);
|
||||
console.log('Updated README.md with tools table of contents');
|
||||
}
|
||||
|
||||
function generateConfigOptionsMarkdown(): string {
|
||||
let markdown = '';
|
||||
|
||||
for (const [optionName, optionConfig] of Object.entries(cliOptions)) {
|
||||
// Skip hidden options
|
||||
if (optionConfig.hidden) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const aliasText = optionConfig.alias ? `, \`-${optionConfig.alias}\`` : '';
|
||||
const description = optionConfig.description || optionConfig.describe || '';
|
||||
|
||||
// Convert camelCase to dash-case
|
||||
const dashCaseName = optionName
|
||||
.replace(/([a-z])([A-Z])/g, '$1-$2')
|
||||
.toLowerCase();
|
||||
const nameDisplay =
|
||||
dashCaseName !== optionName
|
||||
? `\`--${optionName}\`/ \`--${dashCaseName}\``
|
||||
: `\`--${optionName}\``;
|
||||
|
||||
// Start with option name and description
|
||||
markdown += `- **${nameDisplay}${aliasText}**\n`;
|
||||
markdown += ` ${description}\n`;
|
||||
|
||||
// Add type information
|
||||
markdown += ` - **Type:** ${optionConfig.type}\n`;
|
||||
|
||||
// Add choices if available
|
||||
if (optionConfig.choices) {
|
||||
markdown += ` - **Choices:** ${optionConfig.choices.map(c => `\`${c}\``).join(', ')}\n`;
|
||||
}
|
||||
|
||||
// Add default if available
|
||||
markdown += ` - **Default:** \`${optionConfig.default ?? 'false'}\`\n`;
|
||||
|
||||
markdown += '\n';
|
||||
}
|
||||
|
||||
return markdown.trim();
|
||||
}
|
||||
|
||||
function updateReadmeWithOptionsMarkdown(optionsMarkdown: string): void {
|
||||
const readmeContent = fs.readFileSync(README_PATH, 'utf8');
|
||||
|
||||
const beginMarker = '<!-- BEGIN AUTO GENERATED OPTIONS -->';
|
||||
const endMarker = '<!-- END AUTO GENERATED OPTIONS -->';
|
||||
|
||||
const beginIndex = readmeContent.indexOf(beginMarker);
|
||||
const endIndex = readmeContent.indexOf(endMarker);
|
||||
|
||||
if (beginIndex === -1 || endIndex === -1) {
|
||||
console.warn('Could not find auto-generated options markers in README.md');
|
||||
return;
|
||||
}
|
||||
|
||||
const before = readmeContent.substring(0, beginIndex + beginMarker.length);
|
||||
const after = readmeContent.substring(endIndex);
|
||||
|
||||
const updatedContent = before + '\n\n' + optionsMarkdown + '\n\n' + after;
|
||||
|
||||
fs.writeFileSync(README_PATH, updatedContent);
|
||||
console.log('Updated README.md with options markdown');
|
||||
}
|
||||
|
||||
// Helper to convert Zod schema to JSON schema-like object for docs
|
||||
function getZodTypeInfo(schema: ZodSchema): TypeInfo {
|
||||
let description = schema.description;
|
||||
let def = schema._def;
|
||||
let defaultValue: unknown;
|
||||
|
||||
// Unwrap optional/default/effects
|
||||
while (
|
||||
def.typeName === 'ZodOptional' ||
|
||||
def.typeName === 'ZodDefault' ||
|
||||
def.typeName === 'ZodEffects'
|
||||
) {
|
||||
if (def.typeName === 'ZodDefault' && def.defaultValue) {
|
||||
defaultValue = def.defaultValue();
|
||||
}
|
||||
const next = def.innerType || def.schema;
|
||||
if (!next) {
|
||||
break;
|
||||
}
|
||||
schema = next;
|
||||
def = schema._def;
|
||||
if (!description && schema.description) {
|
||||
description = schema.description;
|
||||
}
|
||||
}
|
||||
|
||||
const result: TypeInfo = {type: 'unknown'};
|
||||
if (description) {
|
||||
result.description = description;
|
||||
}
|
||||
if (defaultValue !== undefined) {
|
||||
result.default = defaultValue;
|
||||
}
|
||||
|
||||
switch (def.typeName) {
|
||||
case 'ZodString':
|
||||
result.type = 'string';
|
||||
break;
|
||||
case 'ZodNumber':
|
||||
result.type = def.checks?.some((c: ZodCheck) => c.kind === 'int')
|
||||
? 'integer'
|
||||
: 'number';
|
||||
break;
|
||||
case 'ZodBoolean':
|
||||
result.type = 'boolean';
|
||||
break;
|
||||
case 'ZodEnum':
|
||||
result.type = 'string';
|
||||
result.enum = def.values;
|
||||
break;
|
||||
case 'ZodArray':
|
||||
result.type = 'array';
|
||||
if (def.type) {
|
||||
result.items = getZodTypeInfo(def.type);
|
||||
}
|
||||
break;
|
||||
default:
|
||||
result.type = 'unknown';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isRequired(schema: ZodSchema): boolean {
|
||||
let def = schema._def;
|
||||
while (def.typeName === 'ZodEffects') {
|
||||
if (!def.schema) {
|
||||
break;
|
||||
}
|
||||
schema = def.schema;
|
||||
def = schema._def;
|
||||
}
|
||||
return def.typeName !== 'ZodOptional' && def.typeName !== 'ZodDefault';
|
||||
}
|
||||
|
||||
async function generateReference(
|
||||
title: string,
|
||||
outputPath: string,
|
||||
toolsWithAnnotations: ToolWithAnnotations[],
|
||||
categories: Record<string, ToolWithAnnotations[]>,
|
||||
sortedCategories: string[],
|
||||
) {
|
||||
console.log(`Found ${toolsWithAnnotations.length} tools`);
|
||||
|
||||
// Generate markdown documentation
|
||||
let markdown = `<!-- AUTO GENERATED DO NOT EDIT - run 'npm run gen' to update-->
|
||||
|
||||
# ${title}
|
||||
|
||||
`;
|
||||
// Generate table of contents
|
||||
for (const category of sortedCategories) {
|
||||
const categoryTools = categories[category];
|
||||
const categoryName = labels[category];
|
||||
const anchorName = categoryName.toLowerCase().replace(/\s+/g, '-');
|
||||
markdown += `- **[${categoryName}](#${anchorName})** (${categoryTools.length} tools)\n`;
|
||||
|
||||
// Sort tools within category for TOC
|
||||
categoryTools.sort(sortTools);
|
||||
for (const tool of categoryTools) {
|
||||
// Generate proper markdown anchor link: backticks are removed, keep underscores, lowercase
|
||||
const anchorLink = tool.name.toLowerCase();
|
||||
markdown += ` - [\`${tool.name}\`](#${anchorLink})\n`;
|
||||
}
|
||||
}
|
||||
markdown += '\n';
|
||||
|
||||
for (const category of sortedCategories) {
|
||||
const categoryTools = categories[category];
|
||||
const categoryName = labels[category];
|
||||
|
||||
markdown += `## ${categoryName}\n\n`;
|
||||
|
||||
if (OFF_BY_DEFAULT_CATEGORIES.includes(category)) {
|
||||
const flagName = `--${buildFlag(category)}`;
|
||||
|
||||
markdown += `> NOTE: The ${categoryName} category is not active by default. Use the '${flagName}' flag.\n\n`;
|
||||
}
|
||||
|
||||
// Sort tools within category
|
||||
categoryTools.sort(sortTools);
|
||||
|
||||
for (const tool of categoryTools) {
|
||||
markdown += `### \`${tool.name}\`\n\n`;
|
||||
|
||||
if (tool.description) {
|
||||
// Escape HTML tags but preserve JS function syntax
|
||||
let escapedDescription = escapeHtmlTags(tool.description);
|
||||
|
||||
const requiredFlags: string[] = [];
|
||||
|
||||
const isOffByDefault = OFF_BY_DEFAULT_CATEGORIES.includes(category);
|
||||
if (isOffByDefault) {
|
||||
const categoryFlag = buildFlag(category);
|
||||
requiredFlags.push(`--${categoryFlag}=true`);
|
||||
}
|
||||
|
||||
const conditions = tool.annotations?.conditions || [];
|
||||
for (const condition of conditions) {
|
||||
requiredFlags.push(`--${condition}=true`);
|
||||
}
|
||||
|
||||
if (requiredFlags.length > 0) {
|
||||
escapedDescription += ` (requires flag: ${requiredFlags.join(', ')})`;
|
||||
}
|
||||
|
||||
// Add cross-links to mentioned tools
|
||||
escapedDescription = addCrossLinks(
|
||||
escapedDescription,
|
||||
toolsWithAnnotations,
|
||||
);
|
||||
markdown += `**Description:** ${escapedDescription}\n\n`;
|
||||
}
|
||||
|
||||
// Handle input schema
|
||||
if (
|
||||
tool.inputSchema &&
|
||||
tool.inputSchema.properties &&
|
||||
Object.keys(tool.inputSchema.properties).length > 0
|
||||
) {
|
||||
const properties = tool.inputSchema.properties;
|
||||
const required = tool.inputSchema.required || [];
|
||||
|
||||
markdown += '**Parameters:**\n\n';
|
||||
|
||||
const propertyNames = Object.keys(properties).sort((a, b) => {
|
||||
const aRequired = required.includes(a);
|
||||
const bRequired = required.includes(b);
|
||||
if (aRequired && !bRequired) {
|
||||
return -1;
|
||||
}
|
||||
if (!aRequired && bRequired) {
|
||||
return 1;
|
||||
}
|
||||
return a.localeCompare(b);
|
||||
});
|
||||
for (const propName of propertyNames) {
|
||||
const prop = properties[propName] as TypeInfo;
|
||||
const isRequired = required.includes(propName);
|
||||
const requiredText = isRequired ? ' **(required)**' : ' _(optional)_';
|
||||
|
||||
let typeInfo = prop.type || 'unknown';
|
||||
if (prop.enum) {
|
||||
typeInfo = `enum: ${prop.enum.map((v: string) => `"${v}"`).join(', ')}`;
|
||||
}
|
||||
|
||||
markdown += `- **${propName}** (${typeInfo})${requiredText}`;
|
||||
if (prop.description) {
|
||||
let escapedParamDesc = escapeHtmlTags(prop.description);
|
||||
|
||||
// Add cross-links to mentioned tools
|
||||
escapedParamDesc = addCrossLinks(
|
||||
escapedParamDesc,
|
||||
toolsWithAnnotations,
|
||||
);
|
||||
markdown += `: ${escapedParamDesc}`;
|
||||
}
|
||||
markdown += '\n';
|
||||
}
|
||||
markdown += '\n';
|
||||
} else {
|
||||
markdown += '**Parameters:** None\n\n';
|
||||
}
|
||||
|
||||
markdown += '---\n\n';
|
||||
}
|
||||
}
|
||||
|
||||
// Write the documentation to file
|
||||
fs.writeFileSync(outputPath, markdown.trim() + '\n');
|
||||
|
||||
console.log(
|
||||
`Generated documentation for ${toolsWithAnnotations.length} tools in ${outputPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
function getToolsAndCategories(tools: any) {
|
||||
// Convert ToolDefinitions to ToolWithAnnotations
|
||||
const toolsWithAnnotations: ToolWithAnnotations[] = tools
|
||||
.filter(tool => {
|
||||
// Skipping in_page tools as they are not launched yet
|
||||
if (tool.annotations.category === ToolCategory.IN_PAGE) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Skipping internal interop tools not meant for public documentation
|
||||
const skipTools = ['get_tab_id'];
|
||||
if (skipTools.includes(tool.name)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
})
|
||||
.map(tool => {
|
||||
const properties: Record<string, TypeInfo> = {};
|
||||
const required: string[] = [];
|
||||
|
||||
for (const [key, schema] of Object.entries(
|
||||
tool.schema as unknown as Record<string, ZodSchema>,
|
||||
)) {
|
||||
const info = getZodTypeInfo(schema);
|
||||
properties[key] = info;
|
||||
if (isRequired(schema)) {
|
||||
required.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties,
|
||||
required,
|
||||
},
|
||||
annotations: tool.annotations,
|
||||
};
|
||||
});
|
||||
// Group tools by category (based on annotations)
|
||||
const categories: Record<string, ToolWithAnnotations[]> = {};
|
||||
toolsWithAnnotations.forEach((tool: ToolWithAnnotations) => {
|
||||
const category = tool.annotations?.category || 'Uncategorized';
|
||||
if (!categories[category]) {
|
||||
categories[category] = [];
|
||||
}
|
||||
categories[category].push(tool);
|
||||
});
|
||||
|
||||
// Sort categories using the enum order
|
||||
const categoryOrder = Object.values(ToolCategory);
|
||||
const sortedCategories = Object.keys(categories).sort((a, b) => {
|
||||
const aOff = OFF_BY_DEFAULT_CATEGORIES.includes(a as ToolCategory);
|
||||
const bOff = OFF_BY_DEFAULT_CATEGORIES.includes(b as ToolCategory);
|
||||
|
||||
if (aOff !== bOff) {
|
||||
return aOff ? 1 : -1;
|
||||
}
|
||||
|
||||
const aIndex = categoryOrder.indexOf(a as ToolCategory);
|
||||
const bIndex = categoryOrder.indexOf(b as ToolCategory);
|
||||
|
||||
// Put known categories first, unknown categories last
|
||||
if (aIndex === -1 && bIndex === -1) {
|
||||
return a.localeCompare(b);
|
||||
}
|
||||
if (aIndex === -1) {
|
||||
return 1;
|
||||
}
|
||||
if (bIndex === -1) {
|
||||
return -1;
|
||||
}
|
||||
return aIndex - bIndex;
|
||||
});
|
||||
return {toolsWithAnnotations, categories, sortedCategories};
|
||||
}
|
||||
|
||||
async function generateToolDocumentation(): Promise<void> {
|
||||
try {
|
||||
console.log('Generating tool documentation from definitions...');
|
||||
|
||||
{
|
||||
const {toolsWithAnnotations, categories, sortedCategories} =
|
||||
getToolsAndCategories(createTools({slim: false} as ParsedArguments));
|
||||
await generateReference(
|
||||
'Chrome DevTools MCP Tool Reference',
|
||||
OUTPUT_PATH,
|
||||
toolsWithAnnotations,
|
||||
categories,
|
||||
sortedCategories,
|
||||
);
|
||||
|
||||
// Generate tools TOC and update README
|
||||
const toolsTOC = generateToolsTOC(categories, sortedCategories);
|
||||
updateReadmeWithToolsTOC(toolsTOC);
|
||||
}
|
||||
|
||||
{
|
||||
const {toolsWithAnnotations, categories, sortedCategories} =
|
||||
getToolsAndCategories(createTools({slim: true} as ParsedArguments));
|
||||
await generateReference(
|
||||
'Chrome DevTools MCP Slim Tool Reference',
|
||||
SLIM_OUTPUT_PATH,
|
||||
toolsWithAnnotations,
|
||||
categories,
|
||||
sortedCategories,
|
||||
);
|
||||
}
|
||||
|
||||
// Generate and update configuration options
|
||||
const optionsMarkdown = generateConfigOptionsMarkdown();
|
||||
updateReadmeWithOptionsMarkdown(optionsMarkdown);
|
||||
process.exit(0);
|
||||
} catch (error) {
|
||||
console.error('Error generating documentation:', error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Run the documentation generator
|
||||
generateToolDocumentation().catch(console.error);
|
||||
@@ -0,0 +1,151 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const BUILD_DIR = path.join(process.cwd(), 'build');
|
||||
|
||||
/**
|
||||
* Writes content to a file.
|
||||
* @param filePath The path to the file.
|
||||
* @param content The content to write.
|
||||
*/
|
||||
function writeFile(filePath: string, content: string): void {
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
}
|
||||
|
||||
function main(): void {
|
||||
const devtoolsThirdPartyPath =
|
||||
'node_modules/chrome-devtools-frontend/front_end/third_party';
|
||||
const devtoolsFrontEndCorePath =
|
||||
'node_modules/chrome-devtools-frontend/front_end/core';
|
||||
|
||||
// Create i18n mock
|
||||
const i18nDir = path.join(BUILD_DIR, devtoolsFrontEndCorePath, 'i18n');
|
||||
fs.mkdirSync(i18nDir, {recursive: true});
|
||||
const localesFile = path.join(i18nDir, 'locales.js');
|
||||
const localesContent = `
|
||||
export const LOCALES = [
|
||||
'en-US',
|
||||
];
|
||||
|
||||
export const BUNDLED_LOCALES = [
|
||||
'en-US',
|
||||
];
|
||||
|
||||
export const DEFAULT_LOCALE = 'en-US';
|
||||
|
||||
export const REMOTE_FETCH_PATTERN = '@HOST@/remote/serve_file/@VERSION@/core/i18n/locales/@LOCALE@.json';
|
||||
|
||||
export const LOCAL_FETCH_PATTERN = './locales/@LOCALE@.json';`;
|
||||
writeFile(localesFile, localesContent);
|
||||
|
||||
// Create codemirror.next mock.
|
||||
const codeMirrorDir = path.join(
|
||||
BUILD_DIR,
|
||||
devtoolsThirdPartyPath,
|
||||
'codemirror.next',
|
||||
);
|
||||
fs.mkdirSync(codeMirrorDir, {recursive: true});
|
||||
const codeMirrorFile = path.join(codeMirrorDir, 'codemirror.next.js');
|
||||
const codeMirrorContent = `
|
||||
export default {};
|
||||
export const cssStreamParser = () => Promise.resolve({ startState: () => ({}) });
|
||||
export class StringStream { constructor() {} }
|
||||
export const css = { cssLanguage: { parser: { parse: () => ({ topNode: { getChild: () => null } }) } } };
|
||||
`;
|
||||
writeFile(codeMirrorFile, codeMirrorContent);
|
||||
|
||||
// Create root mock
|
||||
const rootDir = path.join(BUILD_DIR, devtoolsFrontEndCorePath, 'root');
|
||||
fs.mkdirSync(rootDir, {recursive: true});
|
||||
const runtimeFile = path.join(rootDir, 'Runtime.js');
|
||||
const runtimeContent = `
|
||||
export function getChromeVersion() { return ''; };
|
||||
export function getRemoteBase() { return null; };
|
||||
export const hostConfig = {};
|
||||
export const GdpProfilesEnterprisePolicyValue = {
|
||||
ENABLED: 0,
|
||||
ENABLED_WITHOUT_BADGES: 1,
|
||||
DISABLED: 2,
|
||||
};
|
||||
export const Runtime = {
|
||||
isDescriptorEnabled: () => true,
|
||||
queryParam: () => null,
|
||||
}
|
||||
export const experiments = {
|
||||
isEnabled: () => false,
|
||||
}
|
||||
export const ExperimentName = {
|
||||
ALL: '*',
|
||||
CAPTURE_NODE_CREATION_STACKS: 'capture-node-creation-stacks',
|
||||
LIVE_HEAP_PROFILE: 'live-heap-profile',
|
||||
PROTOCOL_MONITOR: 'protocol-monitor',
|
||||
SAMPLING_HEAP_PROFILER_TIMELINE: 'sampling-heap-profiler-timeline',
|
||||
SHOW_OPTION_TO_EXPOSE_INTERNALS_IN_HEAP_SNAPSHOT: 'show-option-to-expose-internals-in-heap-snapshot',
|
||||
TIMELINE_INVALIDATION_TRACKING: 'timeline-invalidation-tracking',
|
||||
TIMELINE_SHOW_ALL_EVENTS: 'timeline-show-all-events',
|
||||
TIMELINE_V8_RUNTIME_CALL_STATS: 'timeline-v8-runtime-call-stats',
|
||||
APCA: 'apca',
|
||||
FONT_EDITOR: 'font-editor',
|
||||
FULL_ACCESSIBILITY_TREE: 'full-accessibility-tree',
|
||||
CONTRAST_ISSUES: 'contrast-issues',
|
||||
EXPERIMENTAL_COOKIE_FEATURES: 'experimental-cookie-features',
|
||||
INSTRUMENTATION_BREAKPOINTS: 'instrumentation-breakpoints',
|
||||
AUTHORED_DEPLOYED_GROUPING: 'authored-deployed-grouping',
|
||||
JUST_MY_CODE: 'just-my-code',
|
||||
USE_SOURCE_MAP_SCOPES: 'use-source-map-scopes',
|
||||
TIMELINE_SHOW_POST_MESSAGE_EVENTS: 'timeline-show-postmessage-events',
|
||||
TIMELINE_DEBUG_MODE: 'timeline-debug-mode',
|
||||
}
|
||||
`;
|
||||
writeFile(runtimeFile, runtimeContent);
|
||||
|
||||
// Copy missing CodeMirror .mjs files that tsc ignores due to .d.mts renames
|
||||
const codemirrorDir = path.join(
|
||||
BUILD_DIR,
|
||||
devtoolsThirdPartyPath,
|
||||
'codemirror',
|
||||
);
|
||||
const codemirrorSrcDir = path.join(
|
||||
process.cwd(),
|
||||
'node_modules',
|
||||
'chrome-devtools-frontend',
|
||||
'front_end',
|
||||
'third_party',
|
||||
'codemirror',
|
||||
);
|
||||
const filesToCopy = [
|
||||
'package/addon/runmode/runmode-standalone.mjs',
|
||||
'package/mode/css/css.mjs',
|
||||
'package/mode/javascript/javascript.mjs',
|
||||
'package/mode/xml/xml.mjs',
|
||||
];
|
||||
for (const file of filesToCopy) {
|
||||
const src = path.join(codemirrorSrcDir, file);
|
||||
const dest = path.join(codemirrorDir, file);
|
||||
fs.mkdirSync(path.dirname(dest), {recursive: true});
|
||||
fs.copyFileSync(src, dest);
|
||||
}
|
||||
|
||||
copyDevToolsDescriptionFiles();
|
||||
}
|
||||
|
||||
function copyDevToolsDescriptionFiles() {
|
||||
const devtoolsIssuesDescriptionPath =
|
||||
'node_modules/chrome-devtools-frontend/front_end/models/issues_manager/descriptions';
|
||||
const sourceDir = path.join(process.cwd(), devtoolsIssuesDescriptionPath);
|
||||
const destDir = path.join(
|
||||
BUILD_DIR,
|
||||
'src',
|
||||
'third_party',
|
||||
'issue-descriptions',
|
||||
);
|
||||
fs.cpSync(sourceDir, destDir, {recursive: true});
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {readFileSync, writeFileSync} from 'node:fs';
|
||||
import {rm} from 'node:fs/promises';
|
||||
import {resolve} from 'node:path';
|
||||
|
||||
const projectRoot = process.cwd();
|
||||
|
||||
const filesToRemove = [
|
||||
'node_modules/chrome-devtools-frontend/package.json',
|
||||
'node_modules/chrome-devtools-frontend/front_end/models/trace/lantern/testing',
|
||||
'node_modules/chrome-devtools-frontend/front_end/third_party/intl-messageformat/package/package.json',
|
||||
];
|
||||
|
||||
/**
|
||||
* Removes the conflicting global HTMLElementEventMap declaration from
|
||||
* @paulirish/trace_engine/models/trace/ModelImpl.d.ts to avoid TS2717 error
|
||||
* when both chrome-devtools-frontend and @paulirish/trace_engine declare
|
||||
* the same property.
|
||||
*/
|
||||
function removeConflictingGlobalDeclaration(): void {
|
||||
const filePath = resolve(
|
||||
projectRoot,
|
||||
'node_modules/@paulirish/trace_engine/models/trace/ModelImpl.d.ts',
|
||||
);
|
||||
console.log(
|
||||
'Removing conflicting global declaration from @paulirish/trace_engine...',
|
||||
);
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
// Remove the declare global block using regex
|
||||
// Matches: declare global { ... interface HTMLElementEventMap { ... } ... }
|
||||
const newContent = content.replace(
|
||||
/declare global\s*\{\s*interface HTMLElementEventMap\s*\{[^}]*\[ModelUpdateEvent\.eventName\]:\s*ModelUpdateEvent;\s*\}\s*\}/s,
|
||||
'',
|
||||
);
|
||||
writeFileSync(filePath, newContent, 'utf-8');
|
||||
console.log('Successfully removed conflicting global declaration.');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Running prepare script to clean up chrome-devtools-frontend...');
|
||||
for (const file of filesToRemove) {
|
||||
const fullPath = resolve(projectRoot, file);
|
||||
console.log(`Removing: ${file}`);
|
||||
try {
|
||||
await rm(fullPath, {recursive: true, force: true});
|
||||
} catch (error) {
|
||||
console.error(`Failed to remove ${file}:`, error);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
console.log('Clean up of chrome-devtools-frontend complete.');
|
||||
|
||||
removeConflictingGlobalDeclaration();
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,116 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
// Note: can be converted to ts file once node 20 support is dropped.
|
||||
// Node 20 does not support --experimental-strip-types flag.
|
||||
|
||||
import {spawn, execSync} from 'node:child_process';
|
||||
import {readFile} from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import process from 'node:process';
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const userArgs = args.filter(arg => !arg.startsWith('-'));
|
||||
const flags = args.filter(arg => arg.startsWith('-'));
|
||||
|
||||
const files = [];
|
||||
|
||||
let shouldRetry = false;
|
||||
const retryIndex = flags.indexOf('--retry');
|
||||
if (retryIndex !== -1) {
|
||||
shouldRetry = true;
|
||||
flags.splice(retryIndex, 1);
|
||||
}
|
||||
|
||||
if (userArgs.length > 0) {
|
||||
for (const arg of userArgs) {
|
||||
// Map .ts files to build/ .js files
|
||||
let testPath = arg;
|
||||
if (testPath.endsWith('.ts')) {
|
||||
testPath = testPath.replace(/\.ts$/, '.js');
|
||||
if (!testPath.startsWith('build/')) {
|
||||
testPath = path.join('build', testPath);
|
||||
}
|
||||
}
|
||||
files.push(testPath);
|
||||
}
|
||||
} else {
|
||||
if (flags.includes('--test-only')) {
|
||||
const {glob} = await import('node:fs/promises');
|
||||
for await (const tsFile of glob('tests/**/*.test.ts')) {
|
||||
const content = await readFile(tsFile, 'utf8');
|
||||
if (content.includes('.only(')) {
|
||||
files.push(path.join('build', tsFile.replace(/\.ts$/, '.js')));
|
||||
}
|
||||
}
|
||||
if (files.length === 0) {
|
||||
console.warn('no files contain .only');
|
||||
process.exit(0);
|
||||
}
|
||||
} else if (files.length === 0) {
|
||||
files.push('build/tests/**/*.test.js');
|
||||
}
|
||||
}
|
||||
|
||||
const nodeArgs = [
|
||||
'--import',
|
||||
'./build/tests/setup.js',
|
||||
'--no-warnings=ExperimentalWarning',
|
||||
'--test-reporter',
|
||||
(process.env['NODE_TEST_REPORTER'] ?? process.env['CI']) ? 'spec' : 'dot',
|
||||
'--test-force-exit',
|
||||
'--test',
|
||||
'--test-timeout=120000',
|
||||
...flags,
|
||||
...files,
|
||||
];
|
||||
|
||||
function _installChrome(version) {
|
||||
try {
|
||||
return execSync(
|
||||
`npx puppeteer browsers install chrome@${version} --format "{{path}}"`,
|
||||
)
|
||||
.toString()
|
||||
.trim();
|
||||
} catch (e) {
|
||||
console.error(`Failed to install Chrome ${version}:`, e);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests(attempt) {
|
||||
if (attempt > 1) {
|
||||
console.log(`\nRun attempt ${attempt}...\n`);
|
||||
}
|
||||
return new Promise(resolve => {
|
||||
const child = spawn('node', nodeArgs, {
|
||||
stdio: 'inherit',
|
||||
env: {
|
||||
...process.env,
|
||||
CHROME_DEVTOOLS_MCP_NO_USAGE_STATISTICS: true,
|
||||
CHROME_DEVTOOLS_MCP_CRASH_ON_UNCAUGHT: true,
|
||||
CHROME_DEVTOOLS_MCP_NO_UPDATE_CHECKS: true,
|
||||
...(process.env['RUNNER_DEBUG'] === '1' ? {DEBUG: 'puppeteer:*'} : {}),
|
||||
},
|
||||
});
|
||||
|
||||
child.on('close', code => {
|
||||
resolve(code);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const maxAttempts = shouldRetry ? 3 : 1;
|
||||
let exitCode = 1;
|
||||
|
||||
for (let i = 1; i <= maxAttempts; i++) {
|
||||
exitCode = await runTests(i);
|
||||
if (exitCode === 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(exitCode ?? 1);
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"extends": "../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"target": "esnext",
|
||||
"module": "nodenext",
|
||||
"moduleResolution": "nodenext",
|
||||
"outDir": "./ignored",
|
||||
"rootDir": ".",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"noImplicitReturns": true,
|
||||
"noImplicitOverride": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"incremental": true,
|
||||
"allowJs": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"noEmit": true,
|
||||
"useUnknownInCatchVariables": false
|
||||
},
|
||||
"include": ["./**/*.ts", "./**/*.js", "./**/*.mjs"]
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {execSync} from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
const ROOT_DIR = process.cwd();
|
||||
const LIGHTHOUSE_DIR = path.resolve(ROOT_DIR, '../lighthouse');
|
||||
const DEST_DIR = path.join(ROOT_DIR, 'src/third_party');
|
||||
|
||||
function main() {
|
||||
if (!fs.existsSync(LIGHTHOUSE_DIR)) {
|
||||
console.error(`Lighthouse directory not found at ${LIGHTHOUSE_DIR}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('Running yarn in lighthouse directory...');
|
||||
execSync('yarn', {cwd: LIGHTHOUSE_DIR, stdio: 'inherit'});
|
||||
|
||||
console.log('Building lighthouse-devtools-mcp bundle...');
|
||||
execSync('yarn build-devtools-mcp', {cwd: LIGHTHOUSE_DIR, stdio: 'inherit'});
|
||||
|
||||
const bundlePath = path.join(
|
||||
LIGHTHOUSE_DIR,
|
||||
'dist',
|
||||
'lighthouse-devtools-mcp-bundle.js',
|
||||
);
|
||||
|
||||
console.log(`Copying bundle from ${bundlePath} to ${DEST_DIR}...`);
|
||||
fs.copyFileSync(
|
||||
bundlePath,
|
||||
path.join(DEST_DIR, 'lighthouse-devtools-mcp-bundle.js'),
|
||||
);
|
||||
|
||||
const noticesPath = path.join(
|
||||
LIGHTHOUSE_DIR,
|
||||
'dist',
|
||||
'LIGHTHOUSE_MCP_BUNDLE_THIRD_PARTY_NOTICES',
|
||||
);
|
||||
|
||||
console.log(`Copying notices from ${noticesPath} to ${DEST_DIR}...`);
|
||||
fs.copyFileSync(
|
||||
noticesPath,
|
||||
path.join(DEST_DIR, 'LIGHTHOUSE_MCP_BUNDLE_THIRD_PARTY_NOTICES'),
|
||||
);
|
||||
|
||||
console.log('Done.');
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
import {
|
||||
cliOptions,
|
||||
parseArguments,
|
||||
} from '../build/src/bin/chrome-devtools-mcp-cli-options.js';
|
||||
import {ErrorCode} from '../build/src/telemetry/errors.js';
|
||||
import {
|
||||
getPossibleFlagMetrics,
|
||||
type FlagMetric,
|
||||
} from '../build/src/telemetry/flagUtils.js';
|
||||
import {
|
||||
applyToExisting,
|
||||
applyToExistingMetrics,
|
||||
generateToolMetrics,
|
||||
type ToolMetric,
|
||||
} from '../build/src/telemetry/metricsRegistry.js';
|
||||
import {createTools} from '../build/src/tools/tools.js';
|
||||
|
||||
export function HaveUniqueNames(tools: Array<{name: string}>): boolean {
|
||||
const toolNames = tools.map(tool => tool.name);
|
||||
const toolNamesSet = new Set(toolNames);
|
||||
return toolNamesSet.size === toolNames.length;
|
||||
}
|
||||
|
||||
function writeToolCallMetricsConfig() {
|
||||
const outputPath = path.resolve('src/telemetry/tool_call_metrics.json');
|
||||
|
||||
const dir = path.dirname(outputPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
throw new Error(`Error: Directory ${dir} does not exist.`);
|
||||
}
|
||||
|
||||
// Avoid 'as ParsedArguments' by using parseArguments
|
||||
const fullTools = createTools(parseArguments('0.0.0', ['', '']));
|
||||
const slimTools = createTools(parseArguments('0.0.0', ['', '', '--slim']));
|
||||
|
||||
const allTools = [...fullTools, ...slimTools];
|
||||
|
||||
if (!HaveUniqueNames(allTools)) {
|
||||
throw new Error('Error: Duplicate tool names found.');
|
||||
}
|
||||
|
||||
let existingMetrics: ToolMetric[] = [];
|
||||
if (fs.existsSync(outputPath)) {
|
||||
try {
|
||||
existingMetrics = JSON.parse(
|
||||
fs.readFileSync(outputPath, 'utf8'),
|
||||
) as ToolMetric[];
|
||||
} catch {
|
||||
console.warn(
|
||||
`Warning: Failed to parse existing metrics from ${outputPath}. Starting fresh.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const newMetrics = generateToolMetrics(allTools);
|
||||
const mergedMetrics = applyToExistingMetrics(existingMetrics, newMetrics);
|
||||
|
||||
fs.writeFileSync(outputPath, JSON.stringify(mergedMetrics, null, 2) + '\n');
|
||||
|
||||
console.log(
|
||||
`Successfully wrote ${mergedMetrics.length} total tool metrics (including deprecated ones) to ${outputPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
function writeFlagUsageMetrics() {
|
||||
const outputPath = path.resolve('src/telemetry/flag_usage_metrics.json');
|
||||
|
||||
const dir = path.dirname(outputPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
throw new Error(`Error: Directory ${dir} does not exist.`);
|
||||
}
|
||||
|
||||
let existingMetrics: FlagMetric[] = [];
|
||||
if (fs.existsSync(outputPath)) {
|
||||
try {
|
||||
existingMetrics = JSON.parse(
|
||||
fs.readFileSync(outputPath, 'utf8'),
|
||||
) as FlagMetric[];
|
||||
} catch {
|
||||
console.warn(
|
||||
`Warning: Failed to parse existing metrics from ${outputPath}. Starting fresh.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const newMetrics = getPossibleFlagMetrics(cliOptions);
|
||||
const mergedMetrics = applyToExisting<FlagMetric>(
|
||||
existingMetrics,
|
||||
newMetrics,
|
||||
);
|
||||
|
||||
fs.writeFileSync(outputPath, JSON.stringify(mergedMetrics, null, 2) + '\n');
|
||||
|
||||
console.log(
|
||||
`Successfully wrote ${mergedMetrics.length} flag usage metrics to ${outputPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
function validateErrorCodes(): void {
|
||||
// The compiled JavaScript object has both forward and backward mappings of the enum,
|
||||
// for example: { '0': 'ERROR_CODE_UNSPECIFIED', 'ERROR_CODE_UNSPECIFIED': 0 }.
|
||||
// This filters out the numeric keys.
|
||||
const stringKeysOnly = Object.entries(ErrorCode).filter(([key]) =>
|
||||
isNaN(Number(key)),
|
||||
);
|
||||
|
||||
let expectedIndex = 0;
|
||||
for (const [key, index] of stringKeysOnly) {
|
||||
if (index !== expectedIndex) {
|
||||
throw new Error(
|
||||
`Error: ErrorCode enums must be sequentially numbered from 0.`,
|
||||
);
|
||||
}
|
||||
if (/_\d/.test(key)) {
|
||||
throw new Error(
|
||||
`Error: ErrorCode enum ${key} is invalid. No numbers should be preceded with an underscore.`,
|
||||
);
|
||||
}
|
||||
expectedIndex++;
|
||||
}
|
||||
|
||||
console.log(`Successfully validated ${expectedIndex} ErrorCode enums.`);
|
||||
}
|
||||
|
||||
function main() {
|
||||
validateErrorCodes();
|
||||
writeToolCallMetricsConfig();
|
||||
writeFlagUsageMetrics();
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,41 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {execSync} from 'node:child_process';
|
||||
|
||||
// Checks that the select build files are present using `npm publish --dry-run`.
|
||||
function verifyPackageContents() {
|
||||
try {
|
||||
const output = execSync('npm publish --dry-run --json --silent', {
|
||||
encoding: 'utf8',
|
||||
});
|
||||
// skip non-JSON output from prepare.
|
||||
const data = JSON.parse(output.substring(output.indexOf('{')));
|
||||
const files = data['chrome-devtools-mcp'].files.map(f => f.path);
|
||||
// Check some important files.
|
||||
const requiredPaths = [
|
||||
'build/src/index.js',
|
||||
'build/src/third_party/index.js',
|
||||
];
|
||||
for (const requiredPath of requiredPaths) {
|
||||
const hasBuildFolder = files.some(path => path.startsWith(requiredPath));
|
||||
if (!hasBuildFolder) {
|
||||
console.error(
|
||||
`Assertion Failed: "${requiredPath}" not found in tarball.`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`npm publish --dry-run contained ${JSON.stringify(requiredPaths)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
console.error('failed to parse npm publish output', err);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
verifyPackageContents();
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import {execSync} from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
const serverJsonFilePath = path.join(process.cwd(), 'server.json');
|
||||
const serverJson = JSON.parse(fs.readFileSync(serverJsonFilePath, 'utf-8'));
|
||||
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mcp-verify-'));
|
||||
|
||||
try {
|
||||
const osName = os.platform();
|
||||
const arch = os.arch();
|
||||
let platform = '';
|
||||
if (osName === 'darwin') {
|
||||
platform = 'darwin';
|
||||
} else if (osName === 'linux') {
|
||||
platform = 'linux';
|
||||
}
|
||||
// mcp-publisher does not support windows
|
||||
else {
|
||||
throw new Error(`Unsupported platform: ${osName}`);
|
||||
}
|
||||
|
||||
let archName = '';
|
||||
if (arch === 'x64') {
|
||||
archName = 'amd64';
|
||||
} else if (arch === 'arm64') {
|
||||
archName = 'arm64';
|
||||
} else {
|
||||
throw new Error(`Unsupported architecture: ${arch}`);
|
||||
}
|
||||
|
||||
const osArch = `${platform}_${archName}`;
|
||||
const binName = 'mcp-publisher';
|
||||
const downloadUrl = `https://github.com/modelcontextprotocol/registry/releases/latest/download/${binName}_${osArch}.tar.gz`;
|
||||
|
||||
console.log(`Downloading ${binName} from ${downloadUrl}`);
|
||||
const downloadCmd = `curl -L "${downloadUrl}" | tar xz -C "${tmpDir}" ${binName}`;
|
||||
execSync(downloadCmd, {stdio: 'inherit'});
|
||||
|
||||
const publisherPath = path.join(tmpDir, binName);
|
||||
fs.chmodSync(publisherPath, 0o755);
|
||||
console.log(`Downloaded to ${publisherPath}`);
|
||||
|
||||
// Create the new server.json in the temporary directory
|
||||
execSync(`${publisherPath} init`, {cwd: tmpDir, stdio: 'inherit'});
|
||||
|
||||
const newServerJsonPath = path.join(tmpDir, 'server.json');
|
||||
const newServerJson = JSON.parse(fs.readFileSync(newServerJsonPath, 'utf-8'));
|
||||
|
||||
const propertyToVerify = ['$schema'];
|
||||
const diffProps = [];
|
||||
|
||||
for (const prop of propertyToVerify) {
|
||||
if (serverJson[prop] !== newServerJson[prop]) {
|
||||
diffProps.push(prop);
|
||||
}
|
||||
}
|
||||
|
||||
if (diffProps.length) {
|
||||
throw new Error(
|
||||
`The following props in ${serverJsonFilePath} did not match the latest init value:\n${diffProps.map(
|
||||
prop =>
|
||||
`- "${prop}": expected "${newServerJson[prop]}", got "${serverJson[prop]}"`,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, {recursive: true, force: true});
|
||||
}
|
||||
Reference in New Issue
Block a user