chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
# Gemini CLI Test Utils (`@google/gemini-cli-test-utils`)
|
||||
|
||||
Shared test utilities used across the monorepo. This is a private package — not
|
||||
published to npm.
|
||||
|
||||
## Key Modules
|
||||
|
||||
- `src/test-rig.ts`: The primary test rig for spinning up end-to-end CLI
|
||||
sessions with mock responses.
|
||||
- `src/file-system-test-helpers.ts`: Helpers for creating temporary file system
|
||||
fixtures.
|
||||
- `src/mock-utils.ts`: Common mock utilities.
|
||||
- `src/test-mcp-server.ts`: Helper for building test MCP servers for tests.
|
||||
- `src/test-mcp-server-template.mjs`: Generic template script for running
|
||||
isolated MCP processes.
|
||||
|
||||
## Test MCP Servers
|
||||
|
||||
The `TestRig` provides a fully isolated, compliant way to test tool triggers and
|
||||
workflows using local test MCP servers. This isolates your tests from live API
|
||||
endpoints and rate-limiting.
|
||||
|
||||
### Usage
|
||||
|
||||
1. **Programmatic Builder:**
|
||||
|
||||
```typescript
|
||||
import { TestMcpServerBuilder } from '@google/gemini-cli-test-utils';
|
||||
|
||||
const builder = new TestMcpServerBuilder('weather-server').addTool(
|
||||
'get_weather',
|
||||
'Get weather',
|
||||
'It is rainy',
|
||||
);
|
||||
|
||||
rig.addTestMcpServer('weather-server', builder.build());
|
||||
```
|
||||
|
||||
2. **Predefined configurations via JSON:** Place a configuration file in
|
||||
`packages/test-utils/assets/test-servers/google-workspace.json` and load it
|
||||
by title:
|
||||
|
||||
```typescript
|
||||
rig.addTestMcpServer('workspace-server', 'google-workspace');
|
||||
```
|
||||
|
||||
**JSON Format Structure (`TestMcpConfig`):**
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "string (Fallback server name)",
|
||||
"tools": [
|
||||
{
|
||||
"name": "string (Tool execution name)",
|
||||
"description": "string (Helpful summary for router)",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": { ... }
|
||||
},
|
||||
"response": "string | object (The forced reply payload)"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
Import from `@google/gemini-cli-test-utils` in test files across the monorepo.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
export * from './src/file-system-test-helpers.js';
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"name": "@google/gemini-cli-test-utils",
|
||||
"version": "0.52.0-nightly.20260707.g27a3da3e8",
|
||||
"private": true,
|
||||
"main": "src/index.ts",
|
||||
"license": "Apache-2.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "node ../../scripts/build_package.js",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google/gemini-cli-core": "file:../core",
|
||||
"@lydell/node-pty": "1.1.0",
|
||||
"asciichart": "1.5.25",
|
||||
"strip-ansi": "7.1.2",
|
||||
"vitest": "3.2.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "5.8.3"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { join } from 'node:path';
|
||||
|
||||
/**
|
||||
* Isolate the test environment by setting environment variables
|
||||
* to point to a temporary run directory.
|
||||
*
|
||||
* @param runDir - The temporary directory for this test run.
|
||||
*/
|
||||
export function isolateTestEnv(runDir: string): void {
|
||||
// Set the home directory to the test run directory to avoid conflicts
|
||||
// with the user's local config.
|
||||
process.env['HOME'] = runDir;
|
||||
if (process.platform === 'win32') {
|
||||
process.env['USERPROFILE'] = runDir;
|
||||
}
|
||||
|
||||
// We also need to set the config dir explicitly, since the code might
|
||||
// construct the path before the HOME env var is set.
|
||||
process.env['GEMINI_CONFIG_DIR'] = join(runDir, '.gemini');
|
||||
|
||||
// Force file storage to avoid keychain prompts/hangs in CI, especially on macOS
|
||||
process.env['GEMINI_FORCE_FILE_STORAGE'] = 'true';
|
||||
|
||||
// Mark as integration test
|
||||
process.env['GEMINI_CLI_INTEGRATION_TEST'] = 'true';
|
||||
|
||||
// Isolate telemetry log
|
||||
process.env['TELEMETRY_LOG_FILE'] = join(runDir, 'telemetry.log');
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
|
||||
/**
|
||||
* Defines the structure of a virtual file system to be created for testing.
|
||||
* Keys are file or directory names, and values can be:
|
||||
* - A string: The content of a file.
|
||||
* - A `FileSystemStructure` object: Represents a subdirectory with its own structure.
|
||||
* - An array of strings or `FileSystemStructure` objects: Represents a directory
|
||||
* where strings are empty files and objects are subdirectories.
|
||||
*
|
||||
* @example
|
||||
* // Example 1: Simple files and directories
|
||||
* const structure1 = {
|
||||
* 'file1.txt': 'Hello, world!',
|
||||
* 'empty-dir': [],
|
||||
* 'src': {
|
||||
* 'main.js': '// Main application file',
|
||||
* 'utils.ts': '// Utility functions',
|
||||
* },
|
||||
* };
|
||||
*
|
||||
* @example
|
||||
* // Example 2: Nested directories and empty files within an array
|
||||
* const structure2 = {
|
||||
* 'config.json': '{ "port": 3000 }',
|
||||
* 'data': [
|
||||
* 'users.csv',
|
||||
* 'products.json',
|
||||
* {
|
||||
* 'logs': [
|
||||
* 'error.log',
|
||||
* 'access.log',
|
||||
* ],
|
||||
* },
|
||||
* ],
|
||||
* };
|
||||
*/
|
||||
export type FileSystemStructure = {
|
||||
[name: string]:
|
||||
| string
|
||||
| FileSystemStructure
|
||||
| Array<string | FileSystemStructure>;
|
||||
};
|
||||
|
||||
/**
|
||||
* Recursively creates files and directories based on the provided `FileSystemStructure`.
|
||||
* @param dir The base directory where the structure will be created.
|
||||
* @param structure The `FileSystemStructure` defining the files and directories.
|
||||
*/
|
||||
async function create(dir: string, structure: FileSystemStructure) {
|
||||
for (const [name, content] of Object.entries(structure)) {
|
||||
const newPath = path.join(dir, name);
|
||||
if (typeof content === 'string') {
|
||||
await fs.writeFile(newPath, content);
|
||||
} else if (Array.isArray(content)) {
|
||||
await fs.mkdir(newPath, { recursive: true });
|
||||
for (const item of content) {
|
||||
if (typeof item === 'string') {
|
||||
await fs.writeFile(path.join(newPath, item), '');
|
||||
} else {
|
||||
await create(newPath, item);
|
||||
}
|
||||
}
|
||||
} else if (typeof content === 'object' && content !== null) {
|
||||
await fs.mkdir(newPath, { recursive: true });
|
||||
await create(newPath, content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a temporary directory and populates it with a given file system structure.
|
||||
* @param structure The `FileSystemStructure` to create within the temporary directory.
|
||||
* @returns A promise that resolves to the absolute path of the created temporary directory.
|
||||
*/
|
||||
export async function createTmpDir(
|
||||
structure: FileSystemStructure,
|
||||
): Promise<string> {
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gemini-cli-test-'));
|
||||
await create(tmpDir, structure);
|
||||
return tmpDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cleans up (deletes) a temporary directory and its contents.
|
||||
* @param dir The absolute path to the temporary directory to clean up.
|
||||
*/
|
||||
export async function cleanupTmpDir(dir: string | undefined) {
|
||||
if (!dir) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const exists = await fs
|
||||
.access(dir)
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
|
||||
if (exists) {
|
||||
if (process.platform === 'win32') {
|
||||
// Give Windows a moment to release file handles
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
} catch {
|
||||
// Ignore errors during cleanup (e.g., directory already deleted)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* Represents a test agent used in evaluations and tests.
|
||||
*/
|
||||
export interface TestAgent {
|
||||
/** The unique name of the agent. */
|
||||
readonly name: string;
|
||||
/** The full YAML/Markdown definition of the agent. */
|
||||
readonly definition: string;
|
||||
/** The standard path where this agent should be saved in a test project. */
|
||||
readonly path: string;
|
||||
/** A helper to spread this agent directly into a 'files' object for evalTest. */
|
||||
readonly asFile: () => Record<string, string>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper to create a TestAgent with consistent formatting and pathing.
|
||||
*/
|
||||
function createAgent(options: {
|
||||
name: string;
|
||||
description: string;
|
||||
tools: string[];
|
||||
body: string;
|
||||
}): TestAgent {
|
||||
const definition = `---
|
||||
name: ${options.name}
|
||||
description: ${options.description}
|
||||
tools:
|
||||
${options.tools.map((t) => ` - ${t}`).join('\n')}
|
||||
---
|
||||
${options.body}
|
||||
`;
|
||||
|
||||
const path = `.gemini/agents/${options.name}.md`;
|
||||
|
||||
return {
|
||||
name: options.name,
|
||||
definition,
|
||||
path,
|
||||
asFile: () => ({ [path]: definition }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A collection of predefined test agents for use in evaluations and tests.
|
||||
*/
|
||||
export const TEST_AGENTS = {
|
||||
/**
|
||||
* An agent with expertise in updating documentation.
|
||||
*/
|
||||
DOCS_AGENT: createAgent({
|
||||
name: 'docs-agent',
|
||||
description: 'An agent with expertise in updating documentation.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the docs agent. Update documentation clearly and accurately.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in writing and updating tests.
|
||||
*/
|
||||
TESTING_AGENT: createAgent({
|
||||
name: 'testing-agent',
|
||||
description: 'An agent with expertise in writing and updating tests.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the test agent. Add or update tests.',
|
||||
}),
|
||||
/**
|
||||
* An agent with expertise in database schemas, SQL, and creating database migrations.
|
||||
*/
|
||||
DATABASE_AGENT: createAgent({
|
||||
name: 'database-agent',
|
||||
description:
|
||||
'An expert in database schemas, SQL, and creating database migrations.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the database agent. Create and update SQL migrations.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in CSS, styling, and UI design.
|
||||
*/
|
||||
CSS_AGENT: createAgent({
|
||||
name: 'css-agent',
|
||||
description: 'An expert in CSS, styling, and UI design.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the CSS agent.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in internationalization and translations.
|
||||
*/
|
||||
I18N_AGENT: createAgent({
|
||||
name: 'i18n-agent',
|
||||
description: 'An expert in internationalization and translations.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the i18n agent.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in security audits and vulnerability patches.
|
||||
*/
|
||||
SECURITY_AGENT: createAgent({
|
||||
name: 'security-agent',
|
||||
description: 'An expert in security audits and vulnerability patches.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the security agent.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in CI/CD, Docker, and deployment scripts.
|
||||
*/
|
||||
DEVOPS_AGENT: createAgent({
|
||||
name: 'devops-agent',
|
||||
description: 'An expert in CI/CD, Docker, and deployment scripts.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the devops agent.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in tracking, analytics, and metrics.
|
||||
*/
|
||||
ANALYTICS_AGENT: createAgent({
|
||||
name: 'analytics-agent',
|
||||
description: 'An expert in tracking, analytics, and metrics.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the analytics agent.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in web accessibility and ARIA roles.
|
||||
*/
|
||||
ACCESSIBILITY_AGENT: createAgent({
|
||||
name: 'accessibility-agent',
|
||||
description: 'An expert in web accessibility and ARIA roles.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the accessibility agent.',
|
||||
}),
|
||||
|
||||
/**
|
||||
* An agent with expertise in React Native and mobile app development.
|
||||
*/
|
||||
MOBILE_AGENT: createAgent({
|
||||
name: 'mobile-agent',
|
||||
description: 'An expert in React Native and mobile app development.',
|
||||
tools: ['read_file', 'write_file', 'list_directory', 'grep_search', 'glob'],
|
||||
body: 'You are the mobile agent.',
|
||||
}),
|
||||
} as const;
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
export * from './file-system-test-helpers.js';
|
||||
export * from './fixtures/agents.js';
|
||||
export * from './memory-baselines.js';
|
||||
export * from './memory-test-harness.js';
|
||||
export * from './perf-test-harness.js';
|
||||
export * from './mock-utils.js';
|
||||
export * from './test-mcp-server.js';
|
||||
export * from './test-rig.js';
|
||||
export * from './env-setup.js';
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
||||
|
||||
/**
|
||||
* Baseline entry for a single memory test scenario.
|
||||
*/
|
||||
export interface MemoryBaseline {
|
||||
heapUsedMB: number;
|
||||
heapTotalMB: number;
|
||||
rssMB: number;
|
||||
externalMB: number;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Top-level structure of the baselines JSON file.
|
||||
*/
|
||||
export interface MemoryBaselineFile {
|
||||
version: number;
|
||||
updatedAt: string;
|
||||
scenarios: Record<string, MemoryBaseline>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load baselines from a JSON file.
|
||||
* Returns an empty baseline file if the file does not exist yet.
|
||||
*/
|
||||
export function loadBaselines(path: string): MemoryBaselineFile {
|
||||
if (!existsSync(path)) {
|
||||
return {
|
||||
version: 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
scenarios: {},
|
||||
};
|
||||
}
|
||||
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
return JSON.parse(content) as MemoryBaselineFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save baselines to a JSON file.
|
||||
*/
|
||||
export function saveBaselines(
|
||||
path: string,
|
||||
baselines: MemoryBaselineFile,
|
||||
): void {
|
||||
baselines.updatedAt = new Date().toISOString();
|
||||
writeFileSync(path, JSON.stringify(baselines, null, 2) + '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Update (or create) a single scenario baseline in the file.
|
||||
*/
|
||||
export function updateBaseline(
|
||||
path: string,
|
||||
scenarioName: string,
|
||||
measured: {
|
||||
heapUsedMB: number;
|
||||
heapTotalMB: number;
|
||||
rssMB: number;
|
||||
externalMB: number;
|
||||
},
|
||||
): void {
|
||||
const baselines = loadBaselines(path);
|
||||
baselines.scenarios[scenarioName] = {
|
||||
heapUsedMB: measured.heapUsedMB,
|
||||
heapTotalMB: measured.heapTotalMB,
|
||||
rssMB: measured.rssMB,
|
||||
externalMB: measured.externalMB,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
saveBaselines(path, baselines);
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { loadBaselines, updateBaseline } from './memory-baselines.js';
|
||||
import type { MemoryBaseline, MemoryBaselineFile } from './memory-baselines.js';
|
||||
import type { TestRig } from './test-rig.js';
|
||||
|
||||
/** Configuration for asciichart plot function. */
|
||||
interface PlotConfig {
|
||||
height?: number;
|
||||
format?: (x: number) => string;
|
||||
}
|
||||
|
||||
/** Type for the asciichart plot function. */
|
||||
type PlotFn = (series: number[], config?: PlotConfig) => string;
|
||||
|
||||
/**
|
||||
* A single memory snapshot at a point in time.
|
||||
*/
|
||||
export interface MemorySnapshot {
|
||||
timestamp: number;
|
||||
label: string;
|
||||
heapUsed: number;
|
||||
heapTotal: number;
|
||||
rss: number;
|
||||
external: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result from running a memory test scenario.
|
||||
*/
|
||||
export interface MemoryTestResult {
|
||||
scenarioName: string;
|
||||
snapshots: MemorySnapshot[];
|
||||
peakHeapUsed: number;
|
||||
peakRss: number;
|
||||
peakExternal: number;
|
||||
finalHeapUsed: number;
|
||||
finalRss: number;
|
||||
finalExternal: number;
|
||||
baseline: MemoryBaseline | undefined;
|
||||
withinTolerance: boolean;
|
||||
deltaPercent: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for the MemoryTestHarness.
|
||||
*/
|
||||
export interface MemoryTestHarnessOptions {
|
||||
/** Path to the baselines JSON file */
|
||||
baselinesPath: string;
|
||||
/** Default tolerance percentage (0-100). Default: 10 */
|
||||
defaultTolerancePercent?: number;
|
||||
/** Number of GC cycles to run before each snapshot. Default: 3 */
|
||||
gcCycles?: number;
|
||||
/** Delay in ms between GC cycles. Default: 100 */
|
||||
gcDelayMs?: number;
|
||||
/** Number of samples to take for median calculation. Default: 3 */
|
||||
sampleCount?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* MemoryTestHarness provides infrastructure for running memory usage tests.
|
||||
*
|
||||
* It handles:
|
||||
* - Extracting memory metrics from CLI process telemetry
|
||||
* - Comparing against baselines with configurable tolerance
|
||||
* - Generating ASCII chart reports of memory trends
|
||||
*/
|
||||
export class MemoryTestHarness {
|
||||
private baselines: MemoryBaselineFile;
|
||||
private readonly baselinesPath: string;
|
||||
private readonly defaultTolerancePercent: number;
|
||||
private allResults: MemoryTestResult[] = [];
|
||||
|
||||
constructor(options: MemoryTestHarnessOptions) {
|
||||
this.baselinesPath = options.baselinesPath;
|
||||
this.defaultTolerancePercent = options.defaultTolerancePercent ?? 10;
|
||||
this.baselines = loadBaselines(this.baselinesPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract memory snapshot from TestRig telemetry.
|
||||
*/
|
||||
async takeSnapshot(
|
||||
rig: TestRig,
|
||||
label: string = 'snapshot',
|
||||
strategy: 'peak' | 'last' = 'last',
|
||||
): Promise<MemorySnapshot> {
|
||||
const metrics = rig.readMemoryMetrics(strategy);
|
||||
|
||||
return {
|
||||
timestamp: metrics.timestamp,
|
||||
label,
|
||||
heapUsed: metrics.heapUsed,
|
||||
heapTotal: metrics.heapTotal,
|
||||
rss: metrics.rss,
|
||||
external: metrics.external,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a memory test scenario.
|
||||
*
|
||||
* @param rig - The TestRig instance running the CLI
|
||||
* @param name - Scenario name (must match baseline key)
|
||||
* @param fn - Async function that executes the scenario. Receives a
|
||||
* `recordSnapshot` callback for recording intermediate snapshots.
|
||||
* @param tolerancePercent - Override default tolerance for this scenario
|
||||
*/
|
||||
async runScenario(
|
||||
rig: TestRig,
|
||||
name: string,
|
||||
fn: (
|
||||
recordSnapshot: (label: string) => Promise<MemorySnapshot>,
|
||||
) => Promise<void>,
|
||||
tolerancePercent?: number,
|
||||
): Promise<MemoryTestResult> {
|
||||
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
|
||||
const snapshots: MemorySnapshot[] = [];
|
||||
|
||||
// Record initial snapshot
|
||||
const beforeSnap = await this.takeSnapshot(rig, 'before');
|
||||
snapshots.push(beforeSnap);
|
||||
|
||||
// Record a callback for intermediate snapshots
|
||||
const recordSnapshot = async (label: string): Promise<MemorySnapshot> => {
|
||||
// Small delay to allow telemetry to flush if needed
|
||||
await rig.waitForTelemetryReady();
|
||||
const snap = await this.takeSnapshot(rig, label);
|
||||
snapshots.push(snap);
|
||||
return snap;
|
||||
};
|
||||
|
||||
// Run the scenario
|
||||
await fn(recordSnapshot);
|
||||
|
||||
// Final wait for telemetry to ensure everything is flushed
|
||||
await rig.waitForTelemetryReady();
|
||||
|
||||
// After snapshot
|
||||
const afterSnap = await this.takeSnapshot(rig, 'after');
|
||||
snapshots.push(afterSnap);
|
||||
|
||||
// Calculate peak values from ALL snapshots seen during the scenario
|
||||
const allSnapshots = rig.readAllMemorySnapshots();
|
||||
const scenarioSnapshots = allSnapshots.filter(
|
||||
(s) =>
|
||||
s.timestamp >= beforeSnap.timestamp &&
|
||||
s.timestamp <= afterSnap.timestamp,
|
||||
);
|
||||
|
||||
const peakHeapUsed = Math.max(
|
||||
...scenarioSnapshots.map((s) => s.heapUsed),
|
||||
...snapshots.map((s) => s.heapUsed),
|
||||
);
|
||||
const peakRss = Math.max(
|
||||
...scenarioSnapshots.map((s) => s.rss),
|
||||
...snapshots.map((s) => s.rss),
|
||||
);
|
||||
const peakExternal = Math.max(
|
||||
...scenarioSnapshots.map((s) => s.external),
|
||||
...snapshots.map((s) => s.external),
|
||||
);
|
||||
|
||||
// Get baseline
|
||||
const baseline = this.baselines.scenarios[name];
|
||||
|
||||
// Determine if within tolerance
|
||||
let deltaPercent = 0;
|
||||
let withinTolerance = true;
|
||||
|
||||
if (baseline) {
|
||||
const measuredMB = afterSnap.heapUsed / (1024 * 1024);
|
||||
deltaPercent =
|
||||
((measuredMB - baseline.heapUsedMB) / baseline.heapUsedMB) * 100;
|
||||
withinTolerance = deltaPercent <= tolerance;
|
||||
}
|
||||
|
||||
const result: MemoryTestResult = {
|
||||
scenarioName: name,
|
||||
snapshots,
|
||||
peakHeapUsed,
|
||||
peakRss,
|
||||
peakExternal,
|
||||
finalHeapUsed: afterSnap.heapUsed,
|
||||
finalRss: afterSnap.rss,
|
||||
finalExternal: afterSnap.external,
|
||||
baseline,
|
||||
withinTolerance,
|
||||
deltaPercent,
|
||||
};
|
||||
|
||||
this.allResults.push(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that a scenario result is within the baseline tolerance.
|
||||
* Throws an assertion error with details if it exceeds the threshold.
|
||||
*/
|
||||
assertWithinBaseline(
|
||||
result: MemoryTestResult,
|
||||
tolerancePercent?: number,
|
||||
): void {
|
||||
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
|
||||
|
||||
if (!result.baseline) {
|
||||
console.warn(
|
||||
`⚠ No baseline found for "${result.scenarioName}". ` +
|
||||
`Run with UPDATE_MEMORY_BASELINES=true to create one. ` +
|
||||
`Measured: ${formatMB(result.finalHeapUsed)} heap used.`,
|
||||
);
|
||||
return; // Don't fail if no baseline exists yet
|
||||
}
|
||||
|
||||
const measuredMB = result.finalHeapUsed / (1024 * 1024);
|
||||
const deltaPercent =
|
||||
((measuredMB - result.baseline.heapUsedMB) / result.baseline.heapUsedMB) *
|
||||
100;
|
||||
|
||||
if (deltaPercent > tolerance) {
|
||||
throw new Error(
|
||||
`Memory regression detected for "${result.scenarioName}"!\n` +
|
||||
` Measured: ${formatMB(result.finalHeapUsed)} heap used\n` +
|
||||
` Baseline: ${result.baseline.heapUsedMB.toFixed(1)} MB heap used\n` +
|
||||
` Delta: ${deltaPercent.toFixed(1)}% (tolerance: ${tolerance}%)\n` +
|
||||
` Peak heap: ${formatMB(result.peakHeapUsed)}\n` +
|
||||
` Peak RSS: ${formatMB(result.peakRss)}\n` +
|
||||
` Peak External: ${formatMB(result.peakExternal)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the baseline for a scenario with the current measured values.
|
||||
*/
|
||||
updateScenarioBaseline(result: MemoryTestResult): void {
|
||||
const lastSnapshot = result.snapshots[result.snapshots.length - 1];
|
||||
updateBaseline(this.baselinesPath, result.scenarioName, {
|
||||
heapUsedMB: Number((result.finalHeapUsed / (1024 * 1024)).toFixed(1)),
|
||||
heapTotalMB: Number(
|
||||
((lastSnapshot?.heapTotal ?? 0) / (1024 * 1024)).toFixed(1),
|
||||
),
|
||||
rssMB: Number((result.finalRss / (1024 * 1024)).toFixed(1)),
|
||||
externalMB: Number((result.finalExternal / (1024 * 1024)).toFixed(1)),
|
||||
});
|
||||
// Reload baselines after update
|
||||
this.baselines = loadBaselines(this.baselinesPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze snapshots to detect sustained leaks.
|
||||
* A leak is flagged if growth is observed in both phases.
|
||||
*/
|
||||
analyzeSnapshots(
|
||||
snapshots: MemorySnapshot[],
|
||||
thresholdBytes: number = 1024 * 1024, // 1 MB
|
||||
): { leaked: boolean; message: string } {
|
||||
if (snapshots.length < 3) {
|
||||
return { leaked: false, message: 'Not enough snapshots to analyze' };
|
||||
}
|
||||
|
||||
const snap1 = snapshots[snapshots.length - 3];
|
||||
const snap2 = snapshots[snapshots.length - 2];
|
||||
const snap3 = snapshots[snapshots.length - 1];
|
||||
|
||||
const growth1 = snap2.heapUsed - snap1.heapUsed;
|
||||
const growth2 = snap3.heapUsed - snap2.heapUsed;
|
||||
|
||||
const leaked = growth1 > thresholdBytes && growth2 > thresholdBytes;
|
||||
let message = leaked
|
||||
? `Memory bloat detected: sustained growth (${formatMB(growth1)} -> ${formatMB(growth2)})`
|
||||
: `No sustained growth detected above threshold.`;
|
||||
|
||||
return { leaked, message };
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that memory returns to a baseline level after a peak.
|
||||
* Useful for verifying that large tool outputs or history are not retained.
|
||||
*/
|
||||
assertMemoryReturnsToBaseline(
|
||||
snapshots: MemorySnapshot[],
|
||||
tolerancePercent: number = 10,
|
||||
): void {
|
||||
if (snapshots.length < 3) {
|
||||
throw new Error('Need at least 3 snapshots to check return to baseline');
|
||||
}
|
||||
|
||||
// Find the first non-zero snapshot as baseline
|
||||
const baseline = snapshots.find((s) => s.heapUsed > 0);
|
||||
if (!baseline) {
|
||||
return; // No memory reported yet
|
||||
}
|
||||
|
||||
const final = snapshots[snapshots.length - 1]!;
|
||||
|
||||
const tolerance = baseline.heapUsed * (tolerancePercent / 100);
|
||||
const delta = final.heapUsed - baseline.heapUsed;
|
||||
|
||||
if (delta > tolerance) {
|
||||
throw new Error(
|
||||
`Memory did not return to baseline!\n` +
|
||||
` Baseline: ${formatMB(baseline.heapUsed)} (${baseline.label})\n` +
|
||||
` Final: ${formatMB(final.heapUsed)} (${final.label})\n` +
|
||||
` Delta: ${formatMB(delta)} (tolerance: ${formatMB(tolerance)})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a report with ASCII charts and summary table.
|
||||
* Uses the `asciichart` library for terminal visualization.
|
||||
*/
|
||||
async generateReport(results?: MemoryTestResult[]): Promise<string> {
|
||||
const resultsToReport = results ?? this.allResults;
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push('');
|
||||
lines.push('═══════════════════════════════════════════════════');
|
||||
lines.push(' MEMORY USAGE TEST REPORT');
|
||||
lines.push('═══════════════════════════════════════════════════');
|
||||
lines.push('');
|
||||
|
||||
for (const result of resultsToReport) {
|
||||
const measured = formatMB(result.finalHeapUsed);
|
||||
const baseline = result.baseline
|
||||
? `${result.baseline.heapUsedMB.toFixed(1)} MB`
|
||||
: 'N/A';
|
||||
const delta = result.baseline
|
||||
? `${result.deltaPercent >= 0 ? '+' : ''}${result.deltaPercent.toFixed(1)}%`
|
||||
: 'N/A';
|
||||
const status = !result.baseline
|
||||
? 'NEW'
|
||||
: result.withinTolerance
|
||||
? '✅'
|
||||
: '❌';
|
||||
|
||||
lines.push(
|
||||
`${result.scenarioName}: ${measured} (Baseline: ${baseline}, Delta: ${delta}) ${status}`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Generate ASCII chart for each scenario with multiple snapshots
|
||||
try {
|
||||
// @ts-expect-error - asciichart may not have types
|
||||
const asciichart = (await import('asciichart')) as {
|
||||
default?: { plot?: PlotFn };
|
||||
plot?: PlotFn;
|
||||
};
|
||||
const plot: PlotFn | undefined =
|
||||
asciichart.default?.plot ?? asciichart.plot;
|
||||
|
||||
for (const result of resultsToReport) {
|
||||
if (result.snapshots.length > 2) {
|
||||
lines.push(`📈 Memory trend: ${result.scenarioName}`);
|
||||
lines.push('─'.repeat(60));
|
||||
|
||||
const heapDataMB = result.snapshots.map(
|
||||
(s) => s.heapUsed / (1024 * 1024),
|
||||
);
|
||||
|
||||
if (plot) {
|
||||
const chart = plot(heapDataMB, {
|
||||
height: 10,
|
||||
format: (x: number) => `${x.toFixed(1)} MB`.padStart(10),
|
||||
});
|
||||
lines.push(chart);
|
||||
}
|
||||
|
||||
// Label the x-axis with snapshot labels
|
||||
const labels = result.snapshots.map((s) => s.label);
|
||||
lines.push(' ' + labels.join(' → '));
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
lines.push(
|
||||
'(asciichart not available — install with: npm install --save-dev asciichart)',
|
||||
);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push('═══════════════════════════════════════════════════');
|
||||
lines.push('');
|
||||
|
||||
const report = lines.join('\n');
|
||||
console.log(report);
|
||||
return report;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Format bytes as a human-readable MB string.
|
||||
*/
|
||||
function formatMB(bytes: number): string {
|
||||
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { SandboxConfig } from '@google/gemini-cli-core';
|
||||
|
||||
export function createMockSandboxConfig(
|
||||
overrides?: Partial<SandboxConfig>,
|
||||
): SandboxConfig {
|
||||
return {
|
||||
enabled: true,
|
||||
allowedPaths: [],
|
||||
networkAccess: false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,551 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
||||
|
||||
/** Configuration for asciichart plot function. */
|
||||
interface PlotConfig {
|
||||
height?: number;
|
||||
format?: (x: number) => string;
|
||||
}
|
||||
|
||||
/** Type for the asciichart plot function. */
|
||||
type PlotFn = (series: number[], config?: PlotConfig) => string;
|
||||
|
||||
/**
|
||||
* Baseline entry for a single performance test scenario.
|
||||
*/
|
||||
export interface PerfBaseline {
|
||||
wallClockMs: number;
|
||||
cpuTotalUs: number;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Top-level structure of the perf baselines JSON file.
|
||||
*/
|
||||
export interface PerfBaselineFile {
|
||||
version: number;
|
||||
updatedAt: string;
|
||||
scenarios: Record<string, PerfBaseline>;
|
||||
}
|
||||
|
||||
/**
|
||||
* A single performance snapshot at a point in time.
|
||||
*/
|
||||
export interface PerfSnapshot {
|
||||
timestamp: number;
|
||||
label: string;
|
||||
wallClockMs: number;
|
||||
cpuUserUs: number;
|
||||
cpuSystemUs: number;
|
||||
cpuTotalUs: number;
|
||||
eventLoopDelayP50Ms: number;
|
||||
eventLoopDelayP95Ms: number;
|
||||
eventLoopDelayMaxMs: number;
|
||||
childEventLoopDelayP50Ms?: number;
|
||||
childEventLoopDelayP95Ms?: number;
|
||||
childEventLoopDelayMaxMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Result from running a performance test scenario.
|
||||
*/
|
||||
export interface PerfTestResult {
|
||||
scenarioName: string;
|
||||
samples: PerfSnapshot[];
|
||||
filteredSamples: PerfSnapshot[];
|
||||
median: PerfSnapshot;
|
||||
baseline: PerfBaseline | undefined;
|
||||
withinTolerance: boolean;
|
||||
deltaPercent: number;
|
||||
cpuDeltaPercent: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for the PerfTestHarness.
|
||||
*/
|
||||
export interface PerfTestHarnessOptions {
|
||||
/** Path to the baselines JSON file */
|
||||
baselinesPath: string;
|
||||
/** Default tolerance percentage (0-100). Default: 15 */
|
||||
defaultTolerancePercent?: number;
|
||||
/** Default CPU tolerance percentage (0-100). Optional */
|
||||
defaultCpuTolerancePercent?: number;
|
||||
/** Number of samples per scenario. Default: 5 */
|
||||
sampleCount?: number;
|
||||
/** Number of warmup runs to discard. Default: 1 */
|
||||
warmupCount?: number;
|
||||
/** Pause in ms between samples. Default: 100 */
|
||||
samplePauseMs?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Active timer state tracked internally.
|
||||
*/
|
||||
interface ActiveTimer {
|
||||
label: string;
|
||||
startTime: number;
|
||||
startCpuUsage: NodeJS.CpuUsage;
|
||||
}
|
||||
|
||||
/**
|
||||
* PerfTestHarness provides infrastructure for running CPU performance tests.
|
||||
*
|
||||
* It handles:
|
||||
* - High-resolution wall-clock timing via performance.now()
|
||||
* - CPU usage measurement via process.cpuUsage()
|
||||
* - Event loop delay monitoring via perf_hooks.monitorEventLoopDelay()
|
||||
* - IQR outlier filtering for noise reduction
|
||||
* - Warmup runs to avoid JIT compilation noise
|
||||
* - Comparing against baselines with configurable tolerance
|
||||
* - Generating ASCII chart reports
|
||||
*/
|
||||
export class PerfTestHarness {
|
||||
private baselines: PerfBaselineFile;
|
||||
private readonly baselinesPath: string;
|
||||
private readonly defaultTolerancePercent: number;
|
||||
private readonly defaultCpuTolerancePercent?: number;
|
||||
private readonly sampleCount: number;
|
||||
private readonly warmupCount: number;
|
||||
private readonly samplePauseMs: number;
|
||||
private allResults: PerfTestResult[] = [];
|
||||
private activeTimers: Map<string, ActiveTimer> = new Map();
|
||||
|
||||
constructor(options: PerfTestHarnessOptions) {
|
||||
this.baselinesPath = options.baselinesPath;
|
||||
this.defaultTolerancePercent = options.defaultTolerancePercent ?? 15;
|
||||
this.defaultCpuTolerancePercent = options.defaultCpuTolerancePercent;
|
||||
this.sampleCount = options.sampleCount ?? 5;
|
||||
this.warmupCount = options.warmupCount ?? 1;
|
||||
this.samplePauseMs = options.samplePauseMs ?? 100;
|
||||
this.baselines = loadPerfBaselines(this.baselinesPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a high-resolution timer with CPU tracking.
|
||||
*/
|
||||
startTimer(label: string): void {
|
||||
this.activeTimers.set(label, {
|
||||
label,
|
||||
startTime: performance.now(),
|
||||
startCpuUsage: process.cpuUsage(),
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop a timer and return the snapshot.
|
||||
*/
|
||||
stopTimer(label: string): PerfSnapshot {
|
||||
const timer = this.activeTimers.get(label);
|
||||
if (!timer) {
|
||||
throw new Error(`No active timer found for label "${label}"`);
|
||||
}
|
||||
|
||||
// Round wall-clock time to nearest 0.1 ms
|
||||
const wallClockMs =
|
||||
Math.round((performance.now() - timer.startTime) * 10) / 10;
|
||||
const cpuDelta = process.cpuUsage(timer.startCpuUsage);
|
||||
this.activeTimers.delete(label);
|
||||
|
||||
return {
|
||||
timestamp: Date.now(),
|
||||
label,
|
||||
wallClockMs,
|
||||
cpuUserUs: cpuDelta.user,
|
||||
cpuSystemUs: cpuDelta.system,
|
||||
cpuTotalUs: cpuDelta.user + cpuDelta.system,
|
||||
eventLoopDelayP50Ms: 0,
|
||||
eventLoopDelayP95Ms: 0,
|
||||
eventLoopDelayMaxMs: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Measure a function's wall-clock time and CPU usage.
|
||||
* Returns the snapshot with timing data.
|
||||
*/
|
||||
async measure(label: string, fn: () => Promise<void>): Promise<PerfSnapshot> {
|
||||
this.startTimer(label);
|
||||
await fn();
|
||||
return this.stopTimer(label);
|
||||
}
|
||||
|
||||
/**
|
||||
* Measure a function with event loop delay monitoring.
|
||||
* Uses perf_hooks.monitorEventLoopDelay() for histogram data.
|
||||
*/
|
||||
async measureWithEventLoop(
|
||||
label: string,
|
||||
fn: () => Promise<void>,
|
||||
): Promise<PerfSnapshot> {
|
||||
// monitorEventLoopDelay is available in Node.js 12+
|
||||
const { monitorEventLoopDelay } = await import('node:perf_hooks');
|
||||
const histogram = monitorEventLoopDelay({ resolution: 10 });
|
||||
histogram.enable();
|
||||
|
||||
this.startTimer(label);
|
||||
await fn();
|
||||
const snapshot = this.stopTimer(label);
|
||||
|
||||
histogram.disable();
|
||||
|
||||
// Convert from nanoseconds to milliseconds
|
||||
snapshot.eventLoopDelayP50Ms = histogram.percentile(50) / 1e6;
|
||||
snapshot.eventLoopDelayP95Ms = histogram.percentile(95) / 1e6;
|
||||
snapshot.eventLoopDelayMaxMs = histogram.max / 1e6;
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a scenario multiple times with warmup, outlier filtering, and baseline comparison.
|
||||
*
|
||||
* @param name - Scenario name (must match baseline key)
|
||||
* @param fn - Async function that executes one sample of the scenario.
|
||||
* Must return a PerfSnapshot with measured values.
|
||||
* @param tolerancePercent - Override default tolerance for this scenario
|
||||
*/
|
||||
async runScenario(
|
||||
name: string,
|
||||
fn: () => Promise<PerfSnapshot>,
|
||||
tolerancePercent?: number,
|
||||
): Promise<PerfTestResult> {
|
||||
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
|
||||
const totalRuns = this.warmupCount + this.sampleCount;
|
||||
const allSnapshots: PerfSnapshot[] = [];
|
||||
|
||||
for (let i = 0; i < totalRuns; i++) {
|
||||
const isWarmup = i < this.warmupCount;
|
||||
const snapshot = await fn();
|
||||
snapshot.label = isWarmup
|
||||
? `warmup-${i}`
|
||||
: `sample-${i - this.warmupCount}`;
|
||||
|
||||
if (!isWarmup) {
|
||||
allSnapshots.push(snapshot);
|
||||
}
|
||||
|
||||
// Brief pause between samples
|
||||
await sleep(this.samplePauseMs);
|
||||
}
|
||||
|
||||
// Apply IQR outlier filtering on wall-clock time
|
||||
const filteredSnapshots = this.filterOutliers(allSnapshots, 'wallClockMs');
|
||||
|
||||
// Get median of filtered samples
|
||||
const median = this.getMedianSnapshot(filteredSnapshots);
|
||||
median.label = 'median';
|
||||
|
||||
// Get baseline
|
||||
const baseline = this.baselines.scenarios[name];
|
||||
|
||||
// Determine if within tolerance
|
||||
let deltaPercent = 0;
|
||||
let cpuDeltaPercent = 0;
|
||||
let withinTolerance = true;
|
||||
|
||||
if (baseline) {
|
||||
deltaPercent =
|
||||
((median.wallClockMs - baseline.wallClockMs) / baseline.wallClockMs) *
|
||||
100;
|
||||
cpuDeltaPercent =
|
||||
((median.cpuTotalUs - baseline.cpuTotalUs) / baseline.cpuTotalUs) * 100;
|
||||
withinTolerance = deltaPercent <= tolerance;
|
||||
}
|
||||
|
||||
const result: PerfTestResult = {
|
||||
scenarioName: name,
|
||||
samples: allSnapshots,
|
||||
filteredSamples: filteredSnapshots,
|
||||
median,
|
||||
baseline,
|
||||
withinTolerance,
|
||||
deltaPercent,
|
||||
cpuDeltaPercent,
|
||||
};
|
||||
|
||||
this.allResults.push(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert that a scenario result is within the baseline tolerance.
|
||||
*/
|
||||
assertWithinBaseline(
|
||||
result: PerfTestResult,
|
||||
tolerancePercent?: number,
|
||||
cpuTolerancePercent?: number,
|
||||
): void {
|
||||
const tolerance = tolerancePercent ?? this.defaultTolerancePercent;
|
||||
const cpuTolerance = cpuTolerancePercent ?? this.defaultCpuTolerancePercent;
|
||||
|
||||
if (!result.baseline) {
|
||||
console.warn(
|
||||
`⚠ No baseline found for "${result.scenarioName}". ` +
|
||||
`Run with UPDATE_PERF_BASELINES=true to create one. ` +
|
||||
`Measured: ${result.median.wallClockMs.toFixed(1)} ms wall-clock.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const deltaPercent =
|
||||
((result.median.wallClockMs - result.baseline.wallClockMs) /
|
||||
result.baseline.wallClockMs) *
|
||||
100;
|
||||
|
||||
if (deltaPercent > tolerance) {
|
||||
throw new Error(
|
||||
`Performance regression detected for "${result.scenarioName}"!\n` +
|
||||
` Measured: ${result.median.wallClockMs.toFixed(1)} ms wall-clock\n` +
|
||||
` Baseline: ${result.baseline.wallClockMs.toFixed(1)} ms wall-clock\n` +
|
||||
` Delta: ${deltaPercent.toFixed(1)}% (tolerance: ${tolerance}%)\n` +
|
||||
` CPU total: ${formatUs(result.median.cpuTotalUs)}\n` +
|
||||
` Samples: ${result.samples.length} (${result.filteredSamples.length} after IQR filter)`,
|
||||
);
|
||||
}
|
||||
|
||||
if (cpuTolerance !== undefined && result.cpuDeltaPercent > cpuTolerance) {
|
||||
throw new Error(
|
||||
`CPU usage regression detected for "${result.scenarioName}"!\n` +
|
||||
` Measured: ${formatUs(result.median.cpuTotalUs)}\n` +
|
||||
` Baseline: ${formatUs(result.baseline.cpuTotalUs)}\n` +
|
||||
` Delta: ${result.cpuDeltaPercent.toFixed(1)}% (tolerance: ${cpuTolerance}%)\n` +
|
||||
` Wall-clock: ${result.median.wallClockMs.toFixed(1)} ms`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the baseline for a scenario with the current measured values.
|
||||
*/
|
||||
updateScenarioBaseline(result: PerfTestResult): void {
|
||||
updatePerfBaseline(this.baselinesPath, result.scenarioName, {
|
||||
wallClockMs: result.median.wallClockMs,
|
||||
cpuTotalUs: result.median.cpuTotalUs,
|
||||
});
|
||||
// Reload baselines after update
|
||||
this.baselines = loadPerfBaselines(this.baselinesPath);
|
||||
console.log(
|
||||
`Updated baseline for ${result.scenarioName}: ${result.median.wallClockMs.toFixed(1)} ms`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an ASCII report with summary table and charts.
|
||||
*/
|
||||
async generateReport(results?: PerfTestResult[]): Promise<string> {
|
||||
const resultsToReport = results ?? this.allResults;
|
||||
const lines: string[] = [];
|
||||
|
||||
lines.push('');
|
||||
lines.push('═══════════════════════════════════════════════════');
|
||||
lines.push(' PERFORMANCE TEST REPORT');
|
||||
lines.push('═══════════════════════════════════════════════════');
|
||||
lines.push('');
|
||||
|
||||
for (const result of resultsToReport) {
|
||||
const measured = `${result.median.wallClockMs.toFixed(1)} ms`;
|
||||
const baseline = result.baseline
|
||||
? `${result.baseline.wallClockMs.toFixed(1)} ms`
|
||||
: 'N/A';
|
||||
const delta = result.baseline
|
||||
? `${result.deltaPercent >= 0 ? '+' : ''}${result.deltaPercent.toFixed(1)}%`
|
||||
: 'N/A';
|
||||
const status = !result.baseline
|
||||
? 'NEW'
|
||||
: result.withinTolerance
|
||||
? '✅'
|
||||
: '❌';
|
||||
|
||||
lines.push(
|
||||
`${result.scenarioName}: ${measured} (Baseline: ${baseline}, Delta: ${delta}) ${status}`,
|
||||
);
|
||||
|
||||
// Show CPU breakdown
|
||||
const cpuMs = `${(result.median.cpuTotalUs / 1000).toFixed(1)} ms`;
|
||||
lines.push(
|
||||
` CPU: ${cpuMs} (user: ${formatUs(result.median.cpuUserUs)}, system: ${formatUs(result.median.cpuSystemUs)})`,
|
||||
);
|
||||
|
||||
if (result.median.eventLoopDelayMaxMs > 0) {
|
||||
lines.push(
|
||||
` Event loop (runner): p50=${result.median.eventLoopDelayP50Ms.toFixed(1)}ms p95=${result.median.eventLoopDelayP95Ms.toFixed(1)}ms max=${result.median.eventLoopDelayMaxMs.toFixed(1)}ms`,
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
result.median.childEventLoopDelayMaxMs !== undefined &&
|
||||
result.median.childEventLoopDelayMaxMs > 0
|
||||
) {
|
||||
lines.push(
|
||||
` Event loop (CLI): p50=${result.median.childEventLoopDelayP50Ms!.toFixed(1)}ms p95=${result.median.childEventLoopDelayP95Ms!.toFixed(1)}ms max=${result.median.childEventLoopDelayMaxMs!.toFixed(1)}ms`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
` Samples: ${result.samples.length} → ${result.filteredSamples.length} after IQR filter`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
// Generate ASCII chart for wall-clock per scenario
|
||||
try {
|
||||
// @ts-expect-error - asciichart may not have types
|
||||
const asciichart = (await import('asciichart')) as {
|
||||
default?: { plot?: PlotFn };
|
||||
plot?: PlotFn;
|
||||
};
|
||||
const plot: PlotFn | undefined =
|
||||
asciichart.default?.plot ?? asciichart.plot;
|
||||
|
||||
for (const result of resultsToReport) {
|
||||
if (result.filteredSamples.length > 2) {
|
||||
lines.push(`📈 Wall-clock trend: ${result.scenarioName}`);
|
||||
lines.push('─'.repeat(60));
|
||||
|
||||
const wallClockData = result.filteredSamples.map(
|
||||
(s) => s.wallClockMs,
|
||||
);
|
||||
|
||||
if (plot) {
|
||||
const chart = plot(wallClockData, {
|
||||
height: 8,
|
||||
format: (x: number) => `${x.toFixed(0)} ms`.padStart(10),
|
||||
});
|
||||
lines.push(chart);
|
||||
}
|
||||
|
||||
const labels = result.filteredSamples.map((s) => s.label);
|
||||
lines.push(' ' + labels.join(' → '));
|
||||
lines.push('');
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
lines.push(
|
||||
'(asciichart not available — install with: npm install --save-dev asciichart)',
|
||||
);
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
lines.push('═══════════════════════════════════════════════════');
|
||||
lines.push('');
|
||||
|
||||
const report = lines.join('\n');
|
||||
console.log(report);
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter outliers using the Interquartile Range (IQR) method.
|
||||
* Removes samples where the given metric falls outside Q1 - 1.5*IQR or Q3 + 1.5*IQR.
|
||||
*/
|
||||
private filterOutliers(
|
||||
snapshots: PerfSnapshot[],
|
||||
metric: keyof PerfSnapshot,
|
||||
): PerfSnapshot[] {
|
||||
if (snapshots.length < 4) {
|
||||
// Not enough data for meaningful IQR filtering
|
||||
return [...snapshots];
|
||||
}
|
||||
|
||||
const sorted = [...snapshots].sort(
|
||||
(a, b) => (a[metric] as number) - (b[metric] as number),
|
||||
);
|
||||
const q1Idx = Math.floor(sorted.length * 0.25);
|
||||
const q3Idx = Math.floor(sorted.length * 0.75);
|
||||
|
||||
const q1 = sorted[q1Idx]![metric] as number;
|
||||
const q3 = sorted[q3Idx]![metric] as number;
|
||||
const iqr = q3 - q1;
|
||||
const lowerBound = q1 - 1.5 * iqr;
|
||||
const upperBound = q3 + 1.5 * iqr;
|
||||
|
||||
return snapshots.filter((s) => {
|
||||
const val = s[metric] as number;
|
||||
return val >= lowerBound && val <= upperBound;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the median snapshot by wall-clock time from a sorted list.
|
||||
*/
|
||||
private getMedianSnapshot(snapshots: PerfSnapshot[]): PerfSnapshot {
|
||||
if (snapshots.length === 0) {
|
||||
throw new Error('Cannot compute median of empty snapshot list');
|
||||
}
|
||||
|
||||
const sorted = [...snapshots].sort((a, b) => a.wallClockMs - b.wallClockMs);
|
||||
const medianIdx = Math.floor(sorted.length / 2);
|
||||
return { ...sorted[medianIdx]! };
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Baseline management ─────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Load perf baselines from a JSON file.
|
||||
*/
|
||||
export function loadPerfBaselines(path: string): PerfBaselineFile {
|
||||
if (!existsSync(path)) {
|
||||
return {
|
||||
version: 1,
|
||||
updatedAt: new Date().toISOString(),
|
||||
scenarios: {},
|
||||
};
|
||||
}
|
||||
|
||||
const content = readFileSync(path, 'utf-8');
|
||||
return JSON.parse(content) as PerfBaselineFile;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save perf baselines to a JSON file.
|
||||
*/
|
||||
export function savePerfBaselines(
|
||||
path: string,
|
||||
baselines: PerfBaselineFile,
|
||||
): void {
|
||||
baselines.updatedAt = new Date().toISOString();
|
||||
writeFileSync(path, JSON.stringify(baselines, null, 2) + '\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Update (or create) a single scenario baseline in the file.
|
||||
*/
|
||||
export function updatePerfBaseline(
|
||||
path: string,
|
||||
scenarioName: string,
|
||||
measured: {
|
||||
wallClockMs: number;
|
||||
cpuTotalUs: number;
|
||||
},
|
||||
): void {
|
||||
const baselines = loadPerfBaselines(path);
|
||||
baselines.scenarios[scenarioName] = {
|
||||
wallClockMs: measured.wallClockMs,
|
||||
cpuTotalUs: measured.cpuTotalUs,
|
||||
timestamp: new Date().toISOString(),
|
||||
};
|
||||
savePerfBaselines(path, baselines);
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Format microseconds as a human-readable string.
|
||||
*/
|
||||
function formatUs(us: number): string {
|
||||
if (us > 1_000_000) {
|
||||
return `${(us / 1_000_000).toFixed(2)} s`;
|
||||
}
|
||||
if (us > 1_000) {
|
||||
return `${(us / 1_000).toFixed(1)} ms`;
|
||||
}
|
||||
return `${us} μs`;
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
||||
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
||||
import {
|
||||
ListToolsRequestSchema,
|
||||
CallToolRequestSchema,
|
||||
} from '@modelcontextprotocol/sdk/types.js';
|
||||
import fs from 'fs';
|
||||
|
||||
const configPath = process.argv[2];
|
||||
if (!configPath) {
|
||||
console.error('Usage: node template.mjs <config-path>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
|
||||
|
||||
const server = new Server(
|
||||
{
|
||||
name: config.name,
|
||||
version: config.version || '1.0.0',
|
||||
},
|
||||
{
|
||||
capabilities: {
|
||||
tools: {},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Add tools handler
|
||||
server.setRequestHandler(ListToolsRequestSchema, async () => {
|
||||
return {
|
||||
tools: (config.tools || []).map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
inputSchema: tool.inputSchema || { type: 'object', properties: {} },
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
// Add call handler
|
||||
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
||||
const toolName = request.params.name;
|
||||
const tool = (config.tools || []).find((t) => t.name === toolName);
|
||||
|
||||
if (!tool) {
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: `Error: Tool ${toolName} not found`,
|
||||
},
|
||||
],
|
||||
isError: true,
|
||||
};
|
||||
}
|
||||
|
||||
return tool.response;
|
||||
});
|
||||
|
||||
const transport = new StdioServerTransport();
|
||||
await server.connect(transport);
|
||||
// server.connect resolves when transport connects, but listening continues
|
||||
console.error(`Test MCP Server '${config.name}' connected and listening.`);
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* Response structure for a test tool call.
|
||||
*/
|
||||
export interface TestToolResponse {
|
||||
content: { type: 'text'; text: string }[];
|
||||
isError?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Definition of a test tool.
|
||||
*/
|
||||
export interface TestTool {
|
||||
name: string;
|
||||
description: string;
|
||||
/** JSON Schema for input arguments */
|
||||
inputSchema?: Record<string, unknown>;
|
||||
response: TestToolResponse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Configuration structure for the generic test MCP server template.
|
||||
*/
|
||||
export interface TestMcpConfig {
|
||||
name: string;
|
||||
version?: string;
|
||||
tools: TestTool[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Builder to easily configure a Test MCP Server in tests.
|
||||
*/
|
||||
export class TestMcpServerBuilder {
|
||||
private config: TestMcpConfig;
|
||||
|
||||
constructor(name: string) {
|
||||
this.config = { name, tools: [] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a tool to the test server configuration.
|
||||
* @param name Tool name
|
||||
* @param description Tool description
|
||||
* @param response The response to return. Can be a string for simple text responses.
|
||||
* @param inputSchema Optional JSON Schema for validation/documentation
|
||||
*/
|
||||
addTool(
|
||||
name: string,
|
||||
description: string,
|
||||
response: TestToolResponse | string,
|
||||
inputSchema?: Record<string, unknown>,
|
||||
): this {
|
||||
const responseObj =
|
||||
typeof response === 'string'
|
||||
? { content: [{ type: 'text' as const, text: response }] }
|
||||
: response;
|
||||
|
||||
this.config.tools.push({
|
||||
name,
|
||||
description,
|
||||
inputSchema,
|
||||
response: responseObj,
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
build(): TestMcpConfig {
|
||||
return this.config;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"extends": "../../tsconfig.json",
|
||||
"compilerOptions": {
|
||||
"outDir": "dist",
|
||||
"lib": ["DOM", "DOM.Iterable", "ES2021"],
|
||||
"composite": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["index.ts", "src/**/*.ts", "src/**/*.json"],
|
||||
"exclude": ["node_modules", "dist"]
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
reporters: ['default', 'junit'],
|
||||
silent: true,
|
||||
outputFile: {
|
||||
junit: 'junit.xml',
|
||||
},
|
||||
poolOptions: {
|
||||
threads: {
|
||||
minThreads: 8,
|
||||
maxThreads: 16,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user