chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 11:58:09 +08:00
commit c48e26cdd0
2909 changed files with 1591131 additions and 0 deletions
+68
View File
@@ -0,0 +1,68 @@
# Gemini CLI Test Utils (`@google/gemini-cli-test-utils`)
Shared test utilities used across the monorepo. This is a private package — not
published to npm.
## Key Modules
- `src/test-rig.ts`: The primary test rig for spinning up end-to-end CLI
sessions with mock responses.
- `src/file-system-test-helpers.ts`: Helpers for creating temporary file system
fixtures.
- `src/mock-utils.ts`: Common mock utilities.
- `src/test-mcp-server.ts`: Helper for building test MCP servers for tests.
- `src/test-mcp-server-template.mjs`: Generic template script for running
isolated MCP processes.
## Test MCP Servers
The `TestRig` provides a fully isolated, compliant way to test tool triggers and
workflows using local test MCP servers. This isolates your tests from live API
endpoints and rate-limiting.
### Usage
1. **Programmatic Builder:**
```typescript
import { TestMcpServerBuilder } from '@google/gemini-cli-test-utils';
const builder = new TestMcpServerBuilder('weather-server').addTool(
'get_weather',
'Get weather',
'It is rainy',
);
rig.addTestMcpServer('weather-server', builder.build());
```
2. **Predefined configurations via JSON:** Place a configuration file in
`packages/test-utils/assets/test-servers/google-workspace.json` and load it
by title:
```typescript
rig.addTestMcpServer('workspace-server', 'google-workspace');
```
**JSON Format Structure (`TestMcpConfig`):**
```json
{
"name": "string (Fallback server name)",
"tools": [
{
"name": "string (Tool execution name)",
"description": "string (Helpful summary for router)",
"inputSchema": {
"type": "object",
"properties": { ... }
},
"response": "string | object (The forced reply payload)"
}
]
}
```
## Usage
Import from `@google/gemini-cli-test-utils` in test files across the monorepo.
File diff suppressed because it is too large Load Diff
+7
View File
@@ -0,0 +1,7 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
export * from './src/file-system-test-helpers.js';
+25
View File
@@ -0,0 +1,25 @@
{
"name": "@google/gemini-cli-test-utils",
"version": "0.52.0-nightly.20260707.g27a3da3e8",
"private": true,
"main": "src/index.ts",
"license": "Apache-2.0",
"type": "module",
"scripts": {
"build": "node ../../scripts/build_package.js",
"typecheck": "tsc --noEmit"
},
"dependencies": {
"@google/gemini-cli-core": "file:../core",
"@lydell/node-pty": "1.1.0",
"asciichart": "1.5.25",
"strip-ansi": "7.1.2",
"vitest": "3.2.4"
},
"devDependencies": {
"typescript": "5.8.3"
},
"engines": {
"node": ">=20"
}
}
+35
View File
@@ -0,0 +1,35 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { join } from 'node:path';
/**
* Isolate the test environment by setting environment variables
* to point to a temporary run directory.
*
* @param runDir - The temporary directory for this test run.
*/
export function isolateTestEnv(runDir: string): void {
// Set the home directory to the test run directory to avoid conflicts
// with the user's local config.
process.env['HOME'] = runDir;
if (process.platform === 'win32') {
process.env['USERPROFILE'] = runDir;
}
// We also need to set the config dir explicitly, since the code might
// construct the path before the HOME env var is set.
process.env['GEMINI_CONFIG_DIR'] = join(runDir, '.gemini');
// Force file storage to avoid keychain prompts/hangs in CI, especially on macOS
process.env['GEMINI_FORCE_FILE_STORAGE'] = 'true';
// Mark as integration test
process.env['GEMINI_CLI_INTEGRATION_TEST'] = 'true';
// Isolate telemetry log
process.env['TELEMETRY_LOG_FILE'] = join(runDir, 'telemetry.log');
}
@@ -0,0 +1,117 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import * as fs from 'node:fs/promises';
import * as path from 'node:path';
import * as os from 'node:os';
/**
* Defines the structure of a virtual file system to be created for testing.
* Keys are file or directory names, and values can be:
* - A string: The content of a file.
* - A `FileSystemStructure` object: Represents a subdirectory with its own structure.
* - An array of strings or `FileSystemStructure` objects: Represents a directory
* where strings are empty files and objects are subdirectories.
*
* @example
* // Example 1: Simple files and directories
* const structure1 = {
* 'file1.txt': 'Hello, world!',
* 'empty-dir': [],
* 'src': {
* 'main.js': '// Main application file',
* 'utils.ts': '// Utility functions',
* },
* };
*
* @example
* // Example 2: Nested directories and empty files within an array
* const structure2 = {
* 'config.json': '{ "port": 3000 }',
* 'data': [
* 'users.csv',
* 'products.json',
* {
* 'logs': [
* 'error.log',
* 'access.log',
* ],
* },
* ],
* };
*/
export type FileSystemStructure = {
[name: string]:
| string
| FileSystemStructure
| Array<string | FileSystemStructure>;
};
/**
* Recursively creates files and directories based on the provided `FileSystemStructure`.
* @param dir The base directory where the structure will be created.
* @param structure The `FileSystemStructure` defining the files and directories.
*/
async function create(dir: string, structure: FileSystemStructure) {
for (const [name, content] of Object.entries(structure)) {
const newPath = path.join(dir, name);
if (typeof content === 'string') {
await fs.writeFile(newPath, content);
} else if (Array.isArray(content)) {
await fs.mkdir(newPath, { recursive: true });
for (const item of content) {
if (typeof item === 'string') {
await fs.writeFile(path.join(newPath, item), '');
} else {
await create(newPath, item);
}
}
} else if (typeof content === 'object' && content !== null) {
await fs.mkdir(newPath, { recursive: true });
await create(newPath, content);
}
}
}
/**
* Creates a temporary directory and populates it with a given file system structure.
* @param structure The `FileSystemStructure` to create within the temporary directory.
* @returns A promise that resolves to the absolute path of the created temporary directory.
*/
export async function createTmpDir(
structure: FileSystemStructure,
): Promise<string> {
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gemini-cli-test-'));
await create(tmpDir, structure);
return tmpDir;
}
/**
* Cleans up (deletes) a temporary directory and its contents.
* @param dir The absolute path to the temporary directory to clean up.
*/
export async function cleanupTmpDir(dir: string | undefined) {
if (!dir) {
return;
}
try {
const exists = await fs
.access(dir)
.then(() => true)
.catch(() => false);
if (exists) {
if (process.platform === 'win32') {
// Give Windows a moment to release file handles
await new Promise((resolve) => setTimeout(resolve, 100));
}
await fs.rm(dir, { recursive: true, force: true });
}
} catch {
// Ignore errors during cleanup (e.g., directory already deleted)
}
}
+152
View File
@@ -0,0 +1,152 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Represents a test agent used in evaluations and tests.
*/
export interface TestAgent {
/** The unique name of the agent. */
readonly name: string;
/** The full YAML/Markdown definition of the agent. */
readonly definition: string;
/** The standard path where this agent should be saved in a test project. */
readonly path: string;
/** A helper to spread this agent directly into a 'files' object for evalTest. */
readonly asFile: () => Record<string, string>;
}
/**
* Helper to create a TestAgent with consistent formatting and pathing.
*/
function createAgent(options: {
name: string;
description: string;
tools: string[];
body: string;
}): TestAgent {
const definition = `---
name: ${options.name}
description: ${options.description}
tools:
${options.tools.map((t) => ` - ${t}`).join('\n')}
---
${options.body}
`;
const path = `.gemini/agents/${options.name}.md`;
return {
name: options.name,
definition,
path,
asFile: () => ({ [path]: definition }),
};
}
/**
* A collection of predefined test agents for use in evaluations and tests.
*/
export const TEST_AGENTS = {
/**
* An agent with expertise in updating documentation.
*/
DOCS_AGENT: createAgent({
name: 'docs-agent',
description: 'An agent with expertise in updating documentation.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the docs agent. Update documentation clearly and accurately.',
}),
/**
* An agent with expertise in writing and updating tests.
*/
TESTING_AGENT: createAgent({
name: 'testing-agent',
description: 'An agent with expertise in writing and updating tests.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the test agent. Add or update tests.',
}),
/**
* An agent with expertise in database schemas, SQL, and creating database migrations.
*/
DATABASE_AGENT: createAgent({
name: 'database-agent',
description:
'An expert in database schemas, SQL, and creating database migrations.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the database agent. Create and update SQL migrations.',
}),
/**
* An agent with expertise in CSS, styling, and UI design.
*/
CSS_AGENT: createAgent({
name: 'css-agent',
description: 'An expert in CSS, styling, and UI design.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the CSS agent.',
}),
/**
* An agent with expertise in internationalization and translations.
*/
I18N_AGENT: createAgent({
name: 'i18n-agent',
description: 'An expert in internationalization and translations.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the i18n agent.',
}),
/**
* An agent with expertise in security audits and vulnerability patches.
*/
SECURITY_AGENT: createAgent({
name: 'security-agent',
description: 'An expert in security audits and vulnerability patches.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the security agent.',
}),
/**
* An agent with expertise in CI/CD, Docker, and deployment scripts.
*/
DEVOPS_AGENT: createAgent({
name: 'devops-agent',
description: 'An expert in CI/CD, Docker, and deployment scripts.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the devops agent.',
}),
/**
* An agent with expertise in tracking, analytics, and metrics.
*/
ANALYTICS_AGENT: createAgent({
name: 'analytics-agent',
description: 'An expert in tracking, analytics, and metrics.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the analytics agent.',
}),
/**
* An agent with expertise in web accessibility and ARIA roles.
*/
ACCESSIBILITY_AGENT: createAgent({
name: 'accessibility-agent',
description: 'An expert in web accessibility and ARIA roles.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the accessibility agent.',
}),
/**
* An agent with expertise in React Native and mobile app development.
*/
MOBILE_AGENT: createAgent({
name: 'mobile-agent',
description: 'An expert in React Native and mobile app development.',
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
body: 'You are the mobile agent.',
}),
} as const;
+15
View File
@@ -0,0 +1,15 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
export * from './file-system-test-helpers.js';
export * from './fixtures/agents.js';
export * from './memory-baselines.js';
export * from './memory-test-harness.js';
export * from './perf-test-harness.js';
export * from './mock-utils.js';
export * from './test-mcp-server.js';
export * from './test-rig.js';
export * from './env-setup.js';
@@ -0,0 +1,79 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
/**
* Baseline entry for a single memory test scenario.
*/
export interface MemoryBaseline {
heapUsedMB: number;
heapTotalMB: number;
rssMB: number;
externalMB: number;
timestamp: string;
}
/**
* Top-level structure of the baselines JSON file.
*/
export interface MemoryBaselineFile {
version: number;
updatedAt: string;
scenarios: Record<string, MemoryBaseline>;
}
/**
* Load baselines from a JSON file.
* Returns an empty baseline file if the file does not exist yet.
*/
export function loadBaselines(path: string): MemoryBaselineFile {
if (!existsSync(path)) {
return {
version: 1,
updatedAt: new Date().toISOString(),
scenarios: {},
};
}
const content = readFileSync(path, 'utf-8');
return JSON.parse(content) as MemoryBaselineFile;
}
/**
* Save baselines to a JSON file.
*/
export function saveBaselines(
path: string,
baselines: MemoryBaselineFile,
): void {
baselines.updatedAt = new Date().toISOString();
writeFileSync(path, JSON.stringify(baselines, null, 2) + '\n');
}
/**
* Update (or create) a single scenario baseline in the file.
*/
export function updateBaseline(
path: string,
scenarioName: string,
measured: {
heapUsedMB: number;
heapTotalMB: number;
rssMB: number;
externalMB: number;
},
): void {
const baselines = loadBaselines(path);
baselines.scenarios[scenarioName] = {
heapUsedMB: measured.heapUsedMB,
heapTotalMB: measured.heapTotalMB,
rssMB: measured.rssMB,
externalMB: measured.externalMB,
timestamp: new Date().toISOString(),
};
saveBaselines(path, baselines);
}
@@ -0,0 +1,403 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { loadBaselines, updateBaseline } from './memory-baselines.js';
import type { MemoryBaseline, MemoryBaselineFile } from './memory-baselines.js';
import type { TestRig } from './test-rig.js';
/** Configuration for asciichart plot function. */
interface PlotConfig {
height?: number;
format?: (x: number) => string;
}
/** Type for the asciichart plot function. */
type PlotFn = (series: number[], config?: PlotConfig) => string;
/**
* A single memory snapshot at a point in time.
*/
export interface MemorySnapshot {
timestamp: number;
label: string;
heapUsed: number;
heapTotal: number;
rss: number;
external: number;
}
/**
* Result from running a memory test scenario.
*/
export interface MemoryTestResult {
scenarioName: string;
snapshots: MemorySnapshot[];
peakHeapUsed: number;
peakRss: number;
peakExternal: number;
finalHeapUsed: number;
finalRss: number;
finalExternal: number;
baseline: MemoryBaseline | undefined;
withinTolerance: boolean;
deltaPercent: number;
}
/**
* Options for the MemoryTestHarness.
*/
export interface MemoryTestHarnessOptions {
/** Path to the baselines JSON file */
baselinesPath: string;
/** Default tolerance percentage (0-100). Default: 10 */
defaultTolerancePercent?: number;
/** Number of GC cycles to run before each snapshot. Default: 3 */
gcCycles?: number;
/** Delay in ms between GC cycles. Default: 100 */
gcDelayMs?: number;
/** Number of samples to take for median calculation. Default: 3 */
sampleCount?: number;
}
/**
* MemoryTestHarness provides infrastructure for running memory usage tests.
*
* It handles:
* - Extracting memory metrics from CLI process telemetry
* - Comparing against baselines with configurable tolerance
* - Generating ASCII chart reports of memory trends
*/
export class MemoryTestHarness {
private baselines: MemoryBaselineFile;
private readonly baselinesPath: string;
private readonly defaultTolerancePercent: number;
private allResults: MemoryTestResult[] = [];
constructor(options: MemoryTestHarnessOptions) {
this.baselinesPath = options.baselinesPath;
this.defaultTolerancePercent = options.defaultTolerancePercent ?? 10;
this.baselines = loadBaselines(this.baselinesPath);
}
/**
* Extract memory snapshot from TestRig telemetry.
*/
async takeSnapshot(
rig: TestRig,
label: string = 'snapshot',
strategy: 'peak' | 'last' = 'last',
): Promise<MemorySnapshot> {
const metrics = rig.readMemoryMetrics(strategy);
return {
timestamp: metrics.timestamp,
label,
heapUsed: metrics.heapUsed,
heapTotal: metrics.heapTotal,
rss: metrics.rss,
external: metrics.external,
};
}
/**
* Run a memory test scenario.
*
* @param rig - The TestRig instance running the CLI
* @param name - Scenario name (must match baseline key)
* @param fn - Async function that executes the scenario. Receives a
* `recordSnapshot` callback for recording intermediate snapshots.
* @param tolerancePercent - Override default tolerance for this scenario
*/
async runScenario(
rig: TestRig,
name: string,
fn: (
recordSnapshot: (label: string) => Promise<MemorySnapshot>,
) => Promise<void>,
tolerancePercent?: number,
): Promise<MemoryTestResult> {
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
const snapshots: MemorySnapshot[] = [];
// Record initial snapshot
const beforeSnap = await this.takeSnapshot(rig, 'before');
snapshots.push(beforeSnap);
// Record a callback for intermediate snapshots
const recordSnapshot = async (label: string): Promise<MemorySnapshot> => {
// Small delay to allow telemetry to flush if needed
await rig.waitForTelemetryReady();
const snap = await this.takeSnapshot(rig, label);
snapshots.push(snap);
return snap;
};
// Run the scenario
await fn(recordSnapshot);
// Final wait for telemetry to ensure everything is flushed
await rig.waitForTelemetryReady();
// After snapshot
const afterSnap = await this.takeSnapshot(rig, 'after');
snapshots.push(afterSnap);
// Calculate peak values from ALL snapshots seen during the scenario
const allSnapshots = rig.readAllMemorySnapshots();
const scenarioSnapshots = allSnapshots.filter(
(s) =>
s.timestamp >= beforeSnap.timestamp &&
s.timestamp <= afterSnap.timestamp,
);
const peakHeapUsed = Math.max(
...scenarioSnapshots.map((s) => s.heapUsed),
...snapshots.map((s) => s.heapUsed),
);
const peakRss = Math.max(
...scenarioSnapshots.map((s) => s.rss),
...snapshots.map((s) => s.rss),
);
const peakExternal = Math.max(
...scenarioSnapshots.map((s) => s.external),
...snapshots.map((s) => s.external),
);
// Get baseline
const baseline = this.baselines.scenarios[name];
// Determine if within tolerance
let deltaPercent = 0;
let withinTolerance = true;
if (baseline) {
const measuredMB = afterSnap.heapUsed / (1024 * 1024);
deltaPercent =
((measuredMB - baseline.heapUsedMB) / baseline.heapUsedMB) * 100;
withinTolerance = deltaPercent <= tolerance;
}
const result: MemoryTestResult = {
scenarioName: name,
snapshots,
peakHeapUsed,
peakRss,
peakExternal,
finalHeapUsed: afterSnap.heapUsed,
finalRss: afterSnap.rss,
finalExternal: afterSnap.external,
baseline,
withinTolerance,
deltaPercent,
};
this.allResults.push(result);
return result;
}
/**
* Assert that a scenario result is within the baseline tolerance.
* Throws an assertion error with details if it exceeds the threshold.
*/
assertWithinBaseline(
result: MemoryTestResult,
tolerancePercent?: number,
): void {
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
if (!result.baseline) {
console.warn(
`⚠ No baseline found for "${result.scenarioName}". ` +
`Run with UPDATE_MEMORY_BASELINES=true to create one. ` +
`Measured: ${formatMB(result.finalHeapUsed)} heap used.`,
);
return; // Don't fail if no baseline exists yet
}
const measuredMB = result.finalHeapUsed / (1024 * 1024);
const deltaPercent =
((measuredMB - result.baseline.heapUsedMB) / result.baseline.heapUsedMB) *
100;
if (deltaPercent > tolerance) {
throw new Error(
`Memory regression detected for "${result.scenarioName}"!\n` +
` Measured: ${formatMB(result.finalHeapUsed)} heap used\n` +
` Baseline: ${result.baseline.heapUsedMB.toFixed(1)} MB heap used\n` +
` Delta: ${deltaPercent.toFixed(1)}% (tolerance: ${tolerance}%)\n` +
` Peak heap: ${formatMB(result.peakHeapUsed)}\n` +
` Peak RSS: ${formatMB(result.peakRss)}\n` +
` Peak External: ${formatMB(result.peakExternal)}`,
);
}
}
/**
* Update the baseline for a scenario with the current measured values.
*/
updateScenarioBaseline(result: MemoryTestResult): void {
const lastSnapshot = result.snapshots[result.snapshots.length - 1];
updateBaseline(this.baselinesPath, result.scenarioName, {
heapUsedMB: Number((result.finalHeapUsed / (1024 * 1024)).toFixed(1)),
heapTotalMB: Number(
((lastSnapshot?.heapTotal ?? 0) / (1024 * 1024)).toFixed(1),
),
rssMB: Number((result.finalRss / (1024 * 1024)).toFixed(1)),
externalMB: Number((result.finalExternal / (1024 * 1024)).toFixed(1)),
});
// Reload baselines after update
this.baselines = loadBaselines(this.baselinesPath);
}
/**
* Analyze snapshots to detect sustained leaks.
* A leak is flagged if growth is observed in both phases.
*/
analyzeSnapshots(
snapshots: MemorySnapshot[],
thresholdBytes: number = 1024 * 1024, // 1 MB
): { leaked: boolean; message: string } {
if (snapshots.length < 3) {
return { leaked: false, message: 'Not enough snapshots to analyze' };
}
const snap1 = snapshots[snapshots.length - 3];
const snap2 = snapshots[snapshots.length - 2];
const snap3 = snapshots[snapshots.length - 1];
const growth1 = snap2.heapUsed - snap1.heapUsed;
const growth2 = snap3.heapUsed - snap2.heapUsed;
const leaked = growth1 > thresholdBytes && growth2 > thresholdBytes;
let message = leaked
? `Memory bloat detected: sustained growth (${formatMB(growth1)} -> ${formatMB(growth2)})`
: `No sustained growth detected above threshold.`;
return { leaked, message };
}
/**
* Assert that memory returns to a baseline level after a peak.
* Useful for verifying that large tool outputs or history are not retained.
*/
assertMemoryReturnsToBaseline(
snapshots: MemorySnapshot[],
tolerancePercent: number = 10,
): void {
if (snapshots.length < 3) {
throw new Error('Need at least 3 snapshots to check return to baseline');
}
// Find the first non-zero snapshot as baseline
const baseline = snapshots.find((s) => s.heapUsed > 0);
if (!baseline) {
return; // No memory reported yet
}
const final = snapshots[snapshots.length - 1]!;
const tolerance = baseline.heapUsed * (tolerancePercent / 100);
const delta = final.heapUsed - baseline.heapUsed;
if (delta > tolerance) {
throw new Error(
`Memory did not return to baseline!\n` +
` Baseline: ${formatMB(baseline.heapUsed)} (${baseline.label})\n` +
` Final: ${formatMB(final.heapUsed)} (${final.label})\n` +
` Delta: ${formatMB(delta)} (tolerance: ${formatMB(tolerance)})`,
);
}
}
/**
* Generate a report with ASCII charts and summary table.
* Uses the `asciichart` library for terminal visualization.
*/
async generateReport(results?: MemoryTestResult[]): Promise<string> {
const resultsToReport = results ?? this.allResults;
const lines: string[] = [];
lines.push('');
lines.push('═══════════════════════════════════════════════════');
lines.push(' MEMORY USAGE TEST REPORT');
lines.push('═══════════════════════════════════════════════════');
lines.push('');
for (const result of resultsToReport) {
const measured = formatMB(result.finalHeapUsed);
const baseline = result.baseline
? `${result.baseline.heapUsedMB.toFixed(1)} MB`
: 'N/A';
const delta = result.baseline
? `${result.deltaPercent >= 0 ? '+' : ''}${result.deltaPercent.toFixed(1)}%`
: 'N/A';
const status = !result.baseline
? 'NEW'
: result.withinTolerance
? '✅'
: '❌';
lines.push(
`${result.scenarioName}: ${measured} (Baseline: ${baseline}, Delta: ${delta}) ${status}`,
);
}
lines.push('');
// Generate ASCII chart for each scenario with multiple snapshots
try {
// @ts-expect-error - asciichart may not have types
const asciichart = (await import('asciichart')) as {
default?: { plot?: PlotFn };
plot?: PlotFn;
};
const plot: PlotFn | undefined =
asciichart.default?.plot ?? asciichart.plot;
for (const result of resultsToReport) {
if (result.snapshots.length > 2) {
lines.push(`📈 Memory trend: ${result.scenarioName}`);
lines.push('─'.repeat(60));
const heapDataMB = result.snapshots.map(
(s) => s.heapUsed / (1024 * 1024),
);
if (plot) {
const chart = plot(heapDataMB, {
height: 10,
format: (x: number) => `${x.toFixed(1)} MB`.padStart(10),
});
lines.push(chart);
}
// Label the x-axis with snapshot labels
const labels = result.snapshots.map((s) => s.label);
lines.push(' ' + labels.join(' → '));
lines.push('');
}
}
} catch {
lines.push(
'(asciichart not available — install with: npm install --save-dev asciichart)',
);
lines.push('');
}
lines.push('═══════════════════════════════════════════════════');
lines.push('');
const report = lines.join('\n');
console.log(report);
return report;
}
}
/**
* Format bytes as a human-readable MB string.
*/
function formatMB(bytes: number): string {
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
}
+18
View File
@@ -0,0 +1,18 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import type { SandboxConfig } from '@google/gemini-cli-core';
export function createMockSandboxConfig(
overrides?: Partial<SandboxConfig>,
): SandboxConfig {
return {
enabled: true,
allowedPaths: [],
networkAccess: false,
...overrides,
};
}
@@ -0,0 +1,551 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { performance } from 'node:perf_hooks';
import { setTimeout as sleep } from 'node:timers/promises';
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
/** Configuration for asciichart plot function. */
interface PlotConfig {
height?: number;
format?: (x: number) => string;
}
/** Type for the asciichart plot function. */
type PlotFn = (series: number[], config?: PlotConfig) => string;
/**
* Baseline entry for a single performance test scenario.
*/
export interface PerfBaseline {
wallClockMs: number;
cpuTotalUs: number;
timestamp: string;
}
/**
* Top-level structure of the perf baselines JSON file.
*/
export interface PerfBaselineFile {
version: number;
updatedAt: string;
scenarios: Record<string, PerfBaseline>;
}
/**
* A single performance snapshot at a point in time.
*/
export interface PerfSnapshot {
timestamp: number;
label: string;
wallClockMs: number;
cpuUserUs: number;
cpuSystemUs: number;
cpuTotalUs: number;
eventLoopDelayP50Ms: number;
eventLoopDelayP95Ms: number;
eventLoopDelayMaxMs: number;
childEventLoopDelayP50Ms?: number;
childEventLoopDelayP95Ms?: number;
childEventLoopDelayMaxMs?: number;
}
/**
* Result from running a performance test scenario.
*/
export interface PerfTestResult {
scenarioName: string;
samples: PerfSnapshot[];
filteredSamples: PerfSnapshot[];
median: PerfSnapshot;
baseline: PerfBaseline | undefined;
withinTolerance: boolean;
deltaPercent: number;
cpuDeltaPercent: number;
}
/**
* Options for the PerfTestHarness.
*/
export interface PerfTestHarnessOptions {
/** Path to the baselines JSON file */
baselinesPath: string;
/** Default tolerance percentage (0-100). Default: 15 */
defaultTolerancePercent?: number;
/** Default CPU tolerance percentage (0-100). Optional */
defaultCpuTolerancePercent?: number;
/** Number of samples per scenario. Default: 5 */
sampleCount?: number;
/** Number of warmup runs to discard. Default: 1 */
warmupCount?: number;
/** Pause in ms between samples. Default: 100 */
samplePauseMs?: number;
}
/**
* Active timer state tracked internally.
*/
interface ActiveTimer {
label: string;
startTime: number;
startCpuUsage: NodeJS.CpuUsage;
}
/**
* PerfTestHarness provides infrastructure for running CPU performance tests.
*
* It handles:
* - High-resolution wall-clock timing via performance.now()
* - CPU usage measurement via process.cpuUsage()
* - Event loop delay monitoring via perf_hooks.monitorEventLoopDelay()
* - IQR outlier filtering for noise reduction
* - Warmup runs to avoid JIT compilation noise
* - Comparing against baselines with configurable tolerance
* - Generating ASCII chart reports
*/
export class PerfTestHarness {
private baselines: PerfBaselineFile;
private readonly baselinesPath: string;
private readonly defaultTolerancePercent: number;
private readonly defaultCpuTolerancePercent?: number;
private readonly sampleCount: number;
private readonly warmupCount: number;
private readonly samplePauseMs: number;
private allResults: PerfTestResult[] = [];
private activeTimers: Map<string, ActiveTimer> = new Map();
constructor(options: PerfTestHarnessOptions) {
this.baselinesPath = options.baselinesPath;
this.defaultTolerancePercent = options.defaultTolerancePercent ?? 15;
this.defaultCpuTolerancePercent = options.defaultCpuTolerancePercent;
this.sampleCount = options.sampleCount ?? 5;
this.warmupCount = options.warmupCount ?? 1;
this.samplePauseMs = options.samplePauseMs ?? 100;
this.baselines = loadPerfBaselines(this.baselinesPath);
}
/**
* Start a high-resolution timer with CPU tracking.
*/
startTimer(label: string): void {
this.activeTimers.set(label, {
label,
startTime: performance.now(),
startCpuUsage: process.cpuUsage(),
});
}
/**
* Stop a timer and return the snapshot.
*/
stopTimer(label: string): PerfSnapshot {
const timer = this.activeTimers.get(label);
if (!timer) {
throw new Error(`No active timer found for label "${label}"`);
}
// Round wall-clock time to nearest 0.1 ms
const wallClockMs =
Math.round((performance.now() - timer.startTime) * 10) / 10;
const cpuDelta = process.cpuUsage(timer.startCpuUsage);
this.activeTimers.delete(label);
return {
timestamp: Date.now(),
label,
wallClockMs,
cpuUserUs: cpuDelta.user,
cpuSystemUs: cpuDelta.system,
cpuTotalUs: cpuDelta.user + cpuDelta.system,
eventLoopDelayP50Ms: 0,
eventLoopDelayP95Ms: 0,
eventLoopDelayMaxMs: 0,
};
}
/**
* Measure a function's wall-clock time and CPU usage.
* Returns the snapshot with timing data.
*/
async measure(label: string, fn: () => Promise<void>): Promise<PerfSnapshot> {
this.startTimer(label);
await fn();
return this.stopTimer(label);
}
/**
* Measure a function with event loop delay monitoring.
* Uses perf_hooks.monitorEventLoopDelay() for histogram data.
*/
async measureWithEventLoop(
label: string,
fn: () => Promise<void>,
): Promise<PerfSnapshot> {
// monitorEventLoopDelay is available in Node.js 12+
const { monitorEventLoopDelay } = await import('node:perf_hooks');
const histogram = monitorEventLoopDelay({ resolution: 10 });
histogram.enable();
this.startTimer(label);
await fn();
const snapshot = this.stopTimer(label);
histogram.disable();
// Convert from nanoseconds to milliseconds
snapshot.eventLoopDelayP50Ms = histogram.percentile(50) / 1e6;
snapshot.eventLoopDelayP95Ms = histogram.percentile(95) / 1e6;
snapshot.eventLoopDelayMaxMs = histogram.max / 1e6;
return snapshot;
}
/**
* Run a scenario multiple times with warmup, outlier filtering, and baseline comparison.
*
* @param name - Scenario name (must match baseline key)
* @param fn - Async function that executes one sample of the scenario.
* Must return a PerfSnapshot with measured values.
* @param tolerancePercent - Override default tolerance for this scenario
*/
async runScenario(
name: string,
fn: () => Promise<PerfSnapshot>,
tolerancePercent?: number,
): Promise<PerfTestResult> {
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
const totalRuns = this.warmupCount + this.sampleCount;
const allSnapshots: PerfSnapshot[] = [];
for (let i = 0; i < totalRuns; i++) {
const isWarmup = i < this.warmupCount;
const snapshot = await fn();
snapshot.label = isWarmup
? `warmup-${i}`
: `sample-${i - this.warmupCount}`;
if (!isWarmup) {
allSnapshots.push(snapshot);
}
// Brief pause between samples
await sleep(this.samplePauseMs);
}
// Apply IQR outlier filtering on wall-clock time
const filteredSnapshots = this.filterOutliers(allSnapshots, 'wallClockMs');
// Get median of filtered samples
const median = this.getMedianSnapshot(filteredSnapshots);
median.label = 'median';
// Get baseline
const baseline = this.baselines.scenarios[name];
// Determine if within tolerance
let deltaPercent = 0;
let cpuDeltaPercent = 0;
let withinTolerance = true;
if (baseline) {
deltaPercent =
((median.wallClockMs - baseline.wallClockMs) / baseline.wallClockMs) *
100;
cpuDeltaPercent =
((median.cpuTotalUs - baseline.cpuTotalUs) / baseline.cpuTotalUs) * 100;
withinTolerance = deltaPercent <= tolerance;
}
const result: PerfTestResult = {
scenarioName: name,
samples: allSnapshots,
filteredSamples: filteredSnapshots,
median,
baseline,
withinTolerance,
deltaPercent,
cpuDeltaPercent,
};
this.allResults.push(result);
return result;
}
/**
* Assert that a scenario result is within the baseline tolerance.
*/
assertWithinBaseline(
result: PerfTestResult,
tolerancePercent?: number,
cpuTolerancePercent?: number,
): void {
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
const cpuTolerance = cpuTolerancePercent ?? this.defaultCpuTolerancePercent;
if (!result.baseline) {
console.warn(
`⚠ No baseline found for "${result.scenarioName}". ` +
`Run with UPDATE_PERF_BASELINES=true to create one. ` +
`Measured: ${result.median.wallClockMs.toFixed(1)} ms wall-clock.`,
);
return;
}
const deltaPercent =
((result.median.wallClockMs - result.baseline.wallClockMs) /
result.baseline.wallClockMs) *
100;
if (deltaPercent > tolerance) {
throw new Error(
`Performance regression detected for "${result.scenarioName}"!\n` +
` Measured: ${result.median.wallClockMs.toFixed(1)} ms wall-clock\n` +
` Baseline: ${result.baseline.wallClockMs.toFixed(1)} ms wall-clock\n` +
` Delta: ${deltaPercent.toFixed(1)}% (tolerance: ${tolerance}%)\n` +
` CPU total: ${formatUs(result.median.cpuTotalUs)}\n` +
` Samples: ${result.samples.length} (${result.filteredSamples.length} after IQR filter)`,
);
}
if (cpuTolerance !== undefined && result.cpuDeltaPercent > cpuTolerance) {
throw new Error(
`CPU usage regression detected for "${result.scenarioName}"!\n` +
` Measured: ${formatUs(result.median.cpuTotalUs)}\n` +
` Baseline: ${formatUs(result.baseline.cpuTotalUs)}\n` +
` Delta: ${result.cpuDeltaPercent.toFixed(1)}% (tolerance: ${cpuTolerance}%)\n` +
` Wall-clock: ${result.median.wallClockMs.toFixed(1)} ms`,
);
}
}
/**
* Update the baseline for a scenario with the current measured values.
*/
updateScenarioBaseline(result: PerfTestResult): void {
updatePerfBaseline(this.baselinesPath, result.scenarioName, {
wallClockMs: result.median.wallClockMs,
cpuTotalUs: result.median.cpuTotalUs,
});
// Reload baselines after update
this.baselines = loadPerfBaselines(this.baselinesPath);
console.log(
`Updated baseline for ${result.scenarioName}: ${result.median.wallClockMs.toFixed(1)} ms`,
);
}
/**
* Generate an ASCII report with summary table and charts.
*/
async generateReport(results?: PerfTestResult[]): Promise<string> {
const resultsToReport = results ?? this.allResults;
const lines: string[] = [];
lines.push('');
lines.push('═══════════════════════════════════════════════════');
lines.push(' PERFORMANCE TEST REPORT');
lines.push('═══════════════════════════════════════════════════');
lines.push('');
for (const result of resultsToReport) {
const measured = `${result.median.wallClockMs.toFixed(1)} ms`;
const baseline = result.baseline
? `${result.baseline.wallClockMs.toFixed(1)} ms`
: 'N/A';
const delta = result.baseline
? `${result.deltaPercent >= 0 ? '+' : ''}${result.deltaPercent.toFixed(1)}%`
: 'N/A';
const status = !result.baseline
? 'NEW'
: result.withinTolerance
? '✅'
: '❌';
lines.push(
`${result.scenarioName}: ${measured} (Baseline: ${baseline}, Delta: ${delta}) ${status}`,
);
// Show CPU breakdown
const cpuMs = `${(result.median.cpuTotalUs / 1000).toFixed(1)} ms`;
lines.push(
` CPU: ${cpuMs} (user: ${formatUs(result.median.cpuUserUs)}, system: ${formatUs(result.median.cpuSystemUs)})`,
);
if (result.median.eventLoopDelayMaxMs > 0) {
lines.push(
` Event loop (runner): p50=${result.median.eventLoopDelayP50Ms.toFixed(1)}ms p95=${result.median.eventLoopDelayP95Ms.toFixed(1)}ms max=${result.median.eventLoopDelayMaxMs.toFixed(1)}ms`,
);
}
if (
result.median.childEventLoopDelayMaxMs !== undefined &&
result.median.childEventLoopDelayMaxMs > 0
) {
lines.push(
` Event loop (CLI): p50=${result.median.childEventLoopDelayP50Ms!.toFixed(1)}ms p95=${result.median.childEventLoopDelayP95Ms!.toFixed(1)}ms max=${result.median.childEventLoopDelayMaxMs!.toFixed(1)}ms`,
);
}
lines.push(
` Samples: ${result.samples.length}${result.filteredSamples.length} after IQR filter`,
);
}
lines.push('');
// Generate ASCII chart for wall-clock per scenario
try {
// @ts-expect-error - asciichart may not have types
const asciichart = (await import('asciichart')) as {
default?: { plot?: PlotFn };
plot?: PlotFn;
};
const plot: PlotFn | undefined =
asciichart.default?.plot ?? asciichart.plot;
for (const result of resultsToReport) {
if (result.filteredSamples.length > 2) {
lines.push(`📈 Wall-clock trend: ${result.scenarioName}`);
lines.push('─'.repeat(60));
const wallClockData = result.filteredSamples.map(
(s) => s.wallClockMs,
);
if (plot) {
const chart = plot(wallClockData, {
height: 8,
format: (x: number) => `${x.toFixed(0)} ms`.padStart(10),
});
lines.push(chart);
}
const labels = result.filteredSamples.map((s) => s.label);
lines.push(' ' + labels.join(' → '));
lines.push('');
}
}
} catch {
lines.push(
'(asciichart not available — install with: npm install --save-dev asciichart)',
);
lines.push('');
}
lines.push('═══════════════════════════════════════════════════');
lines.push('');
const report = lines.join('\n');
console.log(report);
return report;
}
/**
* Filter outliers using the Interquartile Range (IQR) method.
* Removes samples where the given metric falls outside Q1 - 1.5*IQR or Q3 + 1.5*IQR.
*/
private filterOutliers(
snapshots: PerfSnapshot[],
metric: keyof PerfSnapshot,
): PerfSnapshot[] {
if (snapshots.length < 4) {
// Not enough data for meaningful IQR filtering
return [...snapshots];
}
const sorted = [...snapshots].sort(
(a, b) => (a[metric] as number) - (b[metric] as number),
);
const q1Idx = Math.floor(sorted.length * 0.25);
const q3Idx = Math.floor(sorted.length * 0.75);
const q1 = sorted[q1Idx]![metric] as number;
const q3 = sorted[q3Idx]![metric] as number;
const iqr = q3 - q1;
const lowerBound = q1 - 1.5 * iqr;
const upperBound = q3 + 1.5 * iqr;
return snapshots.filter((s) => {
const val = s[metric] as number;
return val >= lowerBound && val <= upperBound;
});
}
/**
* Get the median snapshot by wall-clock time from a sorted list.
*/
private getMedianSnapshot(snapshots: PerfSnapshot[]): PerfSnapshot {
if (snapshots.length === 0) {
throw new Error('Cannot compute median of empty snapshot list');
}
const sorted = [...snapshots].sort((a, b) => a.wallClockMs - b.wallClockMs);
const medianIdx = Math.floor(sorted.length / 2);
return { ...sorted[medianIdx]! };
}
}
// ─── Baseline management ─────────────────────────────────────────────
/**
* Load perf baselines from a JSON file.
*/
export function loadPerfBaselines(path: string): PerfBaselineFile {
if (!existsSync(path)) {
return {
version: 1,
updatedAt: new Date().toISOString(),
scenarios: {},
};
}
const content = readFileSync(path, 'utf-8');
return JSON.parse(content) as PerfBaselineFile;
}
/**
* Save perf baselines to a JSON file.
*/
export function savePerfBaselines(
path: string,
baselines: PerfBaselineFile,
): void {
baselines.updatedAt = new Date().toISOString();
writeFileSync(path, JSON.stringify(baselines, null, 2) + '\n');
}
/**
* Update (or create) a single scenario baseline in the file.
*/
export function updatePerfBaseline(
path: string,
scenarioName: string,
measured: {
wallClockMs: number;
cpuTotalUs: number;
},
): void {
const baselines = loadPerfBaselines(path);
baselines.scenarios[scenarioName] = {
wallClockMs: measured.wallClockMs,
cpuTotalUs: measured.cpuTotalUs,
timestamp: new Date().toISOString(),
};
savePerfBaselines(path, baselines);
}
// ─── Helpers ─────────────────────────────────────────────────────────
/**
* Format microseconds as a human-readable string.
*/
function formatUs(us: number): string {
if (us > 1_000_000) {
return `${(us / 1_000_000).toFixed(2)} s`;
}
if (us > 1_000) {
return `${(us / 1_000).toFixed(1)} ms`;
}
return `${us} μs`;
}
@@ -0,0 +1,69 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
import {
ListToolsRequestSchema,
CallToolRequestSchema,
} from '@modelcontextprotocol/sdk/types.js';
import fs from 'fs';
const configPath = process.argv[2];
if (!configPath) {
console.error('Usage: node template.mjs <config-path>');
process.exit(1);
}
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
const server = new Server(
{
name: config.name,
version: config.version || '1.0.0',
},
{
capabilities: {
tools: {},
},
},
);
// Add tools handler
server.setRequestHandler(ListToolsRequestSchema, async () => {
return {
tools: (config.tools || []).map((tool) => ({
name: tool.name,
description: tool.description,
inputSchema: tool.inputSchema || { type: 'object', properties: {} },
})),
};
});
// Add call handler
server.setRequestHandler(CallToolRequestSchema, async (request) => {
const toolName = request.params.name;
const tool = (config.tools || []).find((t) => t.name === toolName);
if (!tool) {
return {
content: [
{
type: 'text',
text: `Error: Tool ${toolName} not found`,
},
],
isError: true,
};
}
return tool.response;
});
const transport = new StdioServerTransport();
await server.connect(transport);
// server.connect resolves when transport connects, but listening continues
console.error(`Test MCP Server '${config.name}' connected and listening.`);
@@ -0,0 +1,75 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
/**
* Response structure for a test tool call.
*/
export interface TestToolResponse {
content: { type: 'text'; text: string }[];
isError?: boolean;
}
/**
* Definition of a test tool.
*/
export interface TestTool {
name: string;
description: string;
/** JSON Schema for input arguments */
inputSchema?: Record<string, unknown>;
response: TestToolResponse;
}
/**
* Configuration structure for the generic test MCP server template.
*/
export interface TestMcpConfig {
name: string;
version?: string;
tools: TestTool[];
}
/**
* Builder to easily configure a Test MCP Server in tests.
*/
export class TestMcpServerBuilder {
private config: TestMcpConfig;
constructor(name: string) {
this.config = { name, tools: [] };
}
/**
* Adds a tool to the test server configuration.
* @param name Tool name
* @param description Tool description
* @param response The response to return. Can be a string for simple text responses.
* @param inputSchema Optional JSON Schema for validation/documentation
*/
addTool(
name: string,
description: string,
response: TestToolResponse | string,
inputSchema?: Record<string, unknown>,
): this {
const responseObj =
typeof response === 'string'
? { content: [{ type: 'text' as const, text: response }] }
: response;
this.config.tools.push({
name,
description,
inputSchema,
response: responseObj,
});
return this;
}
build(): TestMcpConfig {
return this.config;
}
}
File diff suppressed because it is too large Load Diff
+11
View File
@@ -0,0 +1,11 @@
{
"extends": "../../tsconfig.json",
"compilerOptions": {
"outDir": "dist",
"lib": ["DOM", "DOM.Iterable", "ES2021"],
"composite": true,
"types": ["node"]
},
"include": ["index.ts", "src/**/*.ts", "src/**/*.json"],
"exclude": ["node_modules", "dist"]
}
+23
View File
@@ -0,0 +1,23 @@
/**
* @license
* Copyright 2025 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
reporters: ['default', 'junit'],
silent: true,
outputFile: {
junit: 'junit.xml',
},
poolOptions: {
threads: {
minThreads: 8,
maxThreads: 16,
},
},
},
});