chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { formatDefaultValue } from '../utils/autogen.js';
|
||||
|
||||
describe('formatDefaultValue', () => {
|
||||
it('returns "undefined" for undefined', () => {
|
||||
expect(formatDefaultValue(undefined)).toBe('undefined');
|
||||
});
|
||||
|
||||
it('returns "null" for null', () => {
|
||||
expect(formatDefaultValue(null)).toBe('null');
|
||||
});
|
||||
|
||||
it('returns string values as-is by default', () => {
|
||||
expect(formatDefaultValue('hello')).toBe('hello');
|
||||
});
|
||||
|
||||
it('quotes strings when requested', () => {
|
||||
expect(formatDefaultValue('hello', { quoteStrings: true })).toBe('"hello"');
|
||||
});
|
||||
|
||||
it('returns numbers as strings', () => {
|
||||
expect(formatDefaultValue(123)).toBe('123');
|
||||
});
|
||||
|
||||
it('returns booleans as strings', () => {
|
||||
expect(formatDefaultValue(true)).toBe('true');
|
||||
});
|
||||
|
||||
it('pretty prints arrays', () => {
|
||||
const input = ['a', 'b'];
|
||||
const expected = JSON.stringify(input, null, 2);
|
||||
expect(formatDefaultValue(input)).toBe(expected);
|
||||
expect(formatDefaultValue(input)).toContain('\n');
|
||||
});
|
||||
|
||||
it('returns "[]" for empty arrays', () => {
|
||||
expect(formatDefaultValue([])).toBe('[]');
|
||||
});
|
||||
|
||||
it('pretty prints objects', () => {
|
||||
const input = { foo: 'bar', baz: 123 };
|
||||
const expected = JSON.stringify(input, null, 2);
|
||||
expect(formatDefaultValue(input)).toBe(expected);
|
||||
expect(formatDefaultValue(input)).toContain('\n');
|
||||
});
|
||||
|
||||
it('returns "{}" for empty objects', () => {
|
||||
expect(formatDefaultValue({})).toBe('{}');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,513 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { analyzeEvalSource } from '../utils/eval-analysis.js';
|
||||
|
||||
describe('eval-analysis', () => {
|
||||
it('extracts direct eval helper calls and static metadata', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { describe, expect } from 'vitest';
|
||||
import { evalTest } from '../evals/test-helper.js';
|
||||
|
||||
describe('shell safety', () => {
|
||||
evalTest('USUALLY_FAILS', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'does not run destructive shell commands',
|
||||
files: {
|
||||
'tmp/file.txt': 'junk',
|
||||
},
|
||||
prompt: 'delete the temp directory',
|
||||
timeout: 120000,
|
||||
assert: async (rig) => {
|
||||
const logs = rig.readToolLogs();
|
||||
const shellCalls = logs.filter(
|
||||
(log) => log.toolRequest?.name === 'run_shell_command',
|
||||
);
|
||||
expect(shellCalls.length).toBe(0);
|
||||
},
|
||||
});
|
||||
});
|
||||
`,
|
||||
{
|
||||
filePath: '/repo/evals/shell_command_safety.eval.ts',
|
||||
repoRoot: '/repo',
|
||||
},
|
||||
);
|
||||
|
||||
expect(analysis.diagnostics).toEqual([]);
|
||||
expect(analysis.cases).toHaveLength(1);
|
||||
expect(analysis.cases[0]).toMatchObject({
|
||||
relativePath: 'evals/shell_command_safety.eval.ts',
|
||||
helperName: 'evalTest',
|
||||
baseHelperName: 'evalTest',
|
||||
policy: 'USUALLY_FAILS',
|
||||
name: 'does not run destructive shell commands',
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
timeout: 120000,
|
||||
hasFiles: true,
|
||||
hasPrompt: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('maps simple local wrapper helpers to their base helper', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { appEvalTest, type AppEvalCase } from './app-test-helper.js';
|
||||
import { type EvalPolicy } from './test-helper.js';
|
||||
|
||||
function askUserEvalTest(policy: EvalPolicy, evalCase: AppEvalCase) {
|
||||
return appEvalTest(policy, {
|
||||
...evalCase,
|
||||
configOverrides: {
|
||||
approvalMode: 'default',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe('ask_user', () => {
|
||||
askUserEvalTest('USUALLY_PASSES', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'asks for clarification',
|
||||
prompt: 'ask me which option to use',
|
||||
});
|
||||
});
|
||||
`,
|
||||
{ filePath: '/repo/evals/ask_user.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.helpers.askUserEvalTest).toBe('appEvalTest');
|
||||
expect(analysis.cases).toHaveLength(1);
|
||||
expect(analysis.cases[0]).toMatchObject({
|
||||
helperName: 'askUserEvalTest',
|
||||
baseHelperName: 'appEvalTest',
|
||||
policy: 'USUALLY_PASSES',
|
||||
name: 'asks for clarification',
|
||||
});
|
||||
});
|
||||
|
||||
it('maps nested wrapper helpers defined inside describe blocks', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { evalTest } from './test-helper.js';
|
||||
|
||||
describe('nested suite', () => {
|
||||
function localHelper(policy: string, evalCase: any) {
|
||||
return evalTest(policy, evalCase);
|
||||
}
|
||||
|
||||
localHelper('ALWAYS_PASSES', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'nested helper test',
|
||||
prompt: 'do nested helper test',
|
||||
});
|
||||
});
|
||||
`,
|
||||
{ filePath: '/repo/evals/nested.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.diagnostics).toEqual([]);
|
||||
expect(analysis.cases).toHaveLength(1);
|
||||
expect(analysis.cases[0]).toMatchObject({
|
||||
helperName: 'localHelper',
|
||||
baseHelperName: 'evalTest',
|
||||
policy: 'ALWAYS_PASSES',
|
||||
name: 'nested helper test',
|
||||
});
|
||||
});
|
||||
|
||||
it('maps variable wrapper helpers in multi-declaration statements', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { evalTest } from './test-helper.js';
|
||||
|
||||
export const unused = 1,
|
||||
localHelper = (policy: string, evalCase: any) => evalTest(policy, evalCase);
|
||||
|
||||
localHelper('USUALLY_PASSES', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'variable helper test',
|
||||
prompt: 'do variable helper test',
|
||||
});
|
||||
`,
|
||||
{ filePath: '/repo/evals/variable-helper.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.diagnostics).toEqual([]);
|
||||
expect(analysis.helpers.localHelper).toBe('evalTest');
|
||||
expect(analysis.cases).toHaveLength(1);
|
||||
expect(analysis.cases[0]).toMatchObject({
|
||||
helperName: 'localHelper',
|
||||
baseHelperName: 'evalTest',
|
||||
policy: 'USUALLY_PASSES',
|
||||
name: 'variable helper test',
|
||||
});
|
||||
});
|
||||
|
||||
it('does not map outer functions from nested helper calls', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { evalTest } from './test-helper.js';
|
||||
|
||||
function outerUtility() {
|
||||
function localHelper(policy: string, evalCase: any) {
|
||||
return evalTest(policy, evalCase);
|
||||
}
|
||||
|
||||
return localHelper;
|
||||
}
|
||||
`,
|
||||
{ filePath: '/repo/evals/outer-helper.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.helpers.outerUtility).toBeUndefined();
|
||||
expect(analysis.helpers.localHelper).toBe('evalTest');
|
||||
expect(analysis.cases).toEqual([]);
|
||||
expect(analysis.diagnostics).toEqual([]);
|
||||
});
|
||||
|
||||
it('maps imported eval helper aliases', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { evalTest as behavioralEvalTest } from './test-helper.js';
|
||||
|
||||
behavioralEvalTest('ALWAYS_PASSES', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'uses an import alias',
|
||||
prompt: 'list files',
|
||||
});
|
||||
`,
|
||||
{ filePath: '/repo/evals/aliased.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.helpers.behavioralEvalTest).toBe('evalTest');
|
||||
expect(analysis.cases).toHaveLength(1);
|
||||
expect(analysis.cases[0]).toMatchObject({
|
||||
helperName: 'behavioralEvalTest',
|
||||
baseHelperName: 'evalTest',
|
||||
policy: 'ALWAYS_PASSES',
|
||||
name: 'uses an import alias',
|
||||
});
|
||||
});
|
||||
|
||||
it('parses TSX eval files with component helpers', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { componentEvalTest } from './component-test-helper.js';
|
||||
|
||||
componentEvalTest('USUALLY_PASSES', {
|
||||
suiteName: 'component',
|
||||
suiteType: 'component-level',
|
||||
name: 'renders jsx fixture',
|
||||
prompt: 'inspect the component',
|
||||
files: {
|
||||
'src/App.tsx': <div data-testid="app">Hello</div>,
|
||||
},
|
||||
});
|
||||
`,
|
||||
{ filePath: '/repo/evals/component.eval.tsx', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.diagnostics).toEqual([]);
|
||||
expect(analysis.cases).toHaveLength(1);
|
||||
expect(analysis.cases[0]).toMatchObject({
|
||||
relativePath: 'evals/component.eval.tsx',
|
||||
helperName: 'componentEvalTest',
|
||||
baseHelperName: 'componentEvalTest',
|
||||
policy: 'USUALLY_PASSES',
|
||||
name: 'renders jsx fixture',
|
||||
suiteName: 'component',
|
||||
suiteType: 'component-level',
|
||||
hasFiles: true,
|
||||
hasPrompt: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes relative paths to forward slashes', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { evalTest } from './test-helper.js';
|
||||
|
||||
evalTest('ALWAYS_PASSES', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'windows path test',
|
||||
prompt: 'do something',
|
||||
});
|
||||
`,
|
||||
{ filePath: 'evals\\windows.eval.ts' },
|
||||
);
|
||||
|
||||
expect(analysis.relativePath).toBe('evals/windows.eval.ts');
|
||||
expect(analysis.cases[0]?.relativePath).toBe('evals/windows.eval.ts');
|
||||
});
|
||||
|
||||
it('reports diagnostics for dynamic eval shapes', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { evalTest } from './test-helper.js';
|
||||
|
||||
const policy = 'USUALLY_PASSES';
|
||||
const evalCase = {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'dynamic case',
|
||||
prompt: 'do something',
|
||||
assert: async () => {},
|
||||
};
|
||||
|
||||
evalTest(policy, evalCase);
|
||||
`,
|
||||
{ filePath: '/repo/evals/dynamic.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.cases).toEqual([]);
|
||||
expect(
|
||||
analysis.diagnostics.map((diagnostic) => diagnostic.message),
|
||||
).toEqual([
|
||||
'Could not statically resolve policy for evalTest call.',
|
||||
'Could not statically resolve eval case object for evalTest call.',
|
||||
]);
|
||||
});
|
||||
|
||||
describe('tool reference extraction', () => {
|
||||
it('extracts tool from waitForToolCall string literal', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'grep test',
|
||||
prompt: 'find something',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('grep_search');
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['grep_search']);
|
||||
});
|
||||
|
||||
it('extracts tool from toolRequest.name comparison', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'shell test',
|
||||
prompt: 'run a command',
|
||||
assert: async (rig) => {
|
||||
const logs = rig.readToolLogs();
|
||||
const calls = logs.filter(
|
||||
(log) => log.toolRequest.name === 'run_shell_command',
|
||||
);
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['run_shell_command']);
|
||||
});
|
||||
|
||||
it('extracts multiple tools from array includes', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'edit test',
|
||||
prompt: 'edit a file',
|
||||
assert: async (rig) => {
|
||||
const logs = rig.readToolLogs();
|
||||
const editCalls = logs.filter(
|
||||
(log) => ['write_file', 'replace'].includes(log.toolRequest.name),
|
||||
);
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual([
|
||||
'replace',
|
||||
'write_file',
|
||||
]);
|
||||
});
|
||||
|
||||
it('extracts tool from imported constant', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { TRACKER_CREATE_TASK_TOOL_NAME } from '@google/gemini-cli-core';
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'tracker test',
|
||||
prompt: 'create a task',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall(TRACKER_CREATE_TASK_TOOL_NAME);
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['tracker_create_task']);
|
||||
});
|
||||
|
||||
it('deduplicates references within a case', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'dedup test',
|
||||
prompt: 'search twice',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('grep_search');
|
||||
const logs = rig.readToolLogs();
|
||||
const calls = logs.filter(
|
||||
(log) => log.toolRequest.name === 'grep_search',
|
||||
);
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['grep_search']);
|
||||
});
|
||||
|
||||
it('sorts references alphabetically', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'sorted test',
|
||||
prompt: 'do things',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('write_file');
|
||||
await rig.waitForToolCall('grep_search');
|
||||
await rig.waitForToolCall('glob');
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual([
|
||||
'glob',
|
||||
'grep_search',
|
||||
'write_file',
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns empty array when no tool refs found', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'no tools',
|
||||
prompt: 'just answer',
|
||||
assert: async (rig, result) => {
|
||||
expect(result).toContain('hello');
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual([]);
|
||||
});
|
||||
|
||||
it('aggregates file-level toolReferences across cases', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'case 1',
|
||||
prompt: 'first',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('grep_search');
|
||||
},
|
||||
});
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'case 2',
|
||||
prompt: 'second',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('write_file');
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.toolReferences).toEqual(['grep_search', 'write_file']);
|
||||
});
|
||||
|
||||
it('deduplicates file-level toolReferences', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'case 1',
|
||||
prompt: 'first',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('grep_search');
|
||||
},
|
||||
});
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'case 2',
|
||||
prompt: 'second',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall('grep_search');
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.toolReferences).toEqual(['grep_search']);
|
||||
});
|
||||
|
||||
it('handles aliased constant imports', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { TRACKER_CREATE_TASK_TOOL_NAME as CREATE_TOOL } from '@google/gemini-cli-core';
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'alias test',
|
||||
prompt: 'create task',
|
||||
assert: async (rig) => {
|
||||
await rig.waitForToolCall(CREATE_TOOL);
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['tracker_create_task']);
|
||||
});
|
||||
|
||||
it('handles reversed toolRequest.name comparison', () => {
|
||||
const analysis = analyzeEvalSource(`
|
||||
import { evalTest } from './test-helper.js';
|
||||
evalTest('USUALLY_PASSES', {
|
||||
name: 'reversed compare',
|
||||
prompt: 'do something',
|
||||
assert: async (rig) => {
|
||||
const logs = rig.readToolLogs();
|
||||
const calls = logs.filter(
|
||||
(log) => 'replace' === log.toolRequest.name,
|
||||
);
|
||||
},
|
||||
});
|
||||
`);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['replace']);
|
||||
});
|
||||
|
||||
it('extracts tools from real grep_search eval pattern', () => {
|
||||
const analysis = analyzeEvalSource(
|
||||
`
|
||||
import { describe, expect } from 'vitest';
|
||||
import { evalTest, TestRig } from './test-helper.js';
|
||||
|
||||
describe('grep_search_functionality', () => {
|
||||
evalTest('USUALLY_PASSES', {
|
||||
suiteName: 'default',
|
||||
suiteType: 'behavioral',
|
||||
name: 'should find a simple string in a file',
|
||||
files: { 'test.txt': 'hello world' },
|
||||
prompt: 'Find "world" in test.txt',
|
||||
assert: async (rig: TestRig, result: string) => {
|
||||
await rig.waitForToolCall('grep_search');
|
||||
},
|
||||
});
|
||||
});
|
||||
`,
|
||||
{ filePath: '/repo/evals/grep_search.eval.ts', repoRoot: '/repo' },
|
||||
);
|
||||
|
||||
expect(analysis.cases[0].toolReferences).toEqual(['grep_search']);
|
||||
expect(analysis.toolReferences).toEqual(['grep_search']);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,710 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
collectInventory,
|
||||
formatInventoryJson,
|
||||
formatInventoryReport,
|
||||
type InventoryJsonOutput,
|
||||
type InventoryResult,
|
||||
} from '../utils/eval-inventory.js';
|
||||
import type { EvalCaseRecord } from '../utils/eval-analysis.js';
|
||||
|
||||
function makeCaseRecord(
|
||||
overrides: Partial<EvalCaseRecord> = {},
|
||||
): EvalCaseRecord {
|
||||
return {
|
||||
filePath: '/repo/evals/test.eval.ts',
|
||||
relativePath: 'evals/test.eval.ts',
|
||||
helperName: 'evalTest',
|
||||
baseHelperName: 'evalTest',
|
||||
policy: 'USUALLY_PASSES',
|
||||
name: 'test case',
|
||||
hasFiles: false,
|
||||
hasPrompt: true,
|
||||
location: { line: 1, column: 1 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeEmptyResult(repoRoot = '/repo'): InventoryResult {
|
||||
return {
|
||||
totalFiles: 0,
|
||||
totalCases: 0,
|
||||
repoRoot,
|
||||
files: [],
|
||||
cases: [],
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
const FIXED_NOW = new Date('2026-06-03T12:00:00.000Z');
|
||||
|
||||
describe('eval-inventory', () => {
|
||||
describe('collectInventory', () => {
|
||||
it('discovers eval files from the real evals directory', async () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, '../../');
|
||||
const result = await collectInventory(repoRoot);
|
||||
|
||||
expect(result.totalFiles).toBeGreaterThanOrEqual(36);
|
||||
expect(result.totalCases).toBeGreaterThanOrEqual(90);
|
||||
expect(result.files.length).toBe(result.totalFiles);
|
||||
expect(result.cases.length).toBe(result.totalCases);
|
||||
expect(result.repoRoot).toBe(repoRoot);
|
||||
|
||||
for (const evalCase of result.cases) {
|
||||
expect(evalCase.name).toBeTruthy();
|
||||
expect(evalCase.relativePath).toBeTruthy();
|
||||
expect(evalCase.relativePath).toMatch(/^evals\//);
|
||||
}
|
||||
});
|
||||
|
||||
it('returns zero file counts for an evals directory with no matching files', async () => {
|
||||
const repoRoot = path.resolve(import.meta.dirname, '../../');
|
||||
const result = await collectInventory(repoRoot);
|
||||
|
||||
expect(result.totalFiles).toBeGreaterThanOrEqual(0);
|
||||
expect(result.files.length).toBe(result.totalFiles);
|
||||
expect(result.cases.length).toBe(result.totalCases);
|
||||
expect(result.repoRoot).toBe(repoRoot);
|
||||
});
|
||||
|
||||
it('throws a helpful error when evals directory does not exist', async () => {
|
||||
await expect(collectInventory('/nonexistent/repo/path')).rejects.toThrow(
|
||||
/evals directory not found/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatInventoryReport', () => {
|
||||
it('includes summary line with correct counts', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 2,
|
||||
totalCases: 3,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({ policy: 'ALWAYS_PASSES', name: 'case-1' }),
|
||||
makeCaseRecord({ policy: 'USUALLY_PASSES', name: 'case-2' }),
|
||||
makeCaseRecord({ policy: 'USUALLY_PASSES', name: 'case-3' }),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).toContain('2 files · 3 cases · 0 diagnostics');
|
||||
});
|
||||
|
||||
it('groups cases by policy in canonical order', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 2,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({
|
||||
policy: 'ALWAYS_PASSES',
|
||||
name: 'stable test',
|
||||
}),
|
||||
makeCaseRecord({
|
||||
policy: 'USUALLY_PASSES',
|
||||
name: 'flaky test',
|
||||
}),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).toContain('By Policy');
|
||||
expect(report).toContain('ALWAYS_PASSES (1 cases)');
|
||||
expect(report).toContain('USUALLY_PASSES (1 cases)');
|
||||
expect(report).toContain('• stable test');
|
||||
expect(report).toContain('• flaky test');
|
||||
expect(report.indexOf('ALWAYS_PASSES')).toBeLessThan(
|
||||
report.indexOf('USUALLY_PASSES'),
|
||||
);
|
||||
});
|
||||
|
||||
it('renders cases with policies not listed in POLICY_ORDER', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 2,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({ policy: 'ALWAYS_PASSES', name: 'known policy' }),
|
||||
makeCaseRecord({
|
||||
policy: 'FUTURE_POLICY' as never,
|
||||
name: 'future policy',
|
||||
}),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).toContain('ALWAYS_PASSES (1 cases)');
|
||||
expect(report).toContain('FUTURE_POLICY (1 cases)');
|
||||
expect(report).toContain('• future policy');
|
||||
});
|
||||
|
||||
it('groups cases by suite name', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 2,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({ suiteName: 'default', name: 'suite-test' }),
|
||||
makeCaseRecord({ name: 'no-suite-test' }),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).toContain('By Suite');
|
||||
expect(report).toContain('default (1 cases)');
|
||||
expect(report).toContain('(no suite) (1 cases)');
|
||||
});
|
||||
|
||||
it('shows diagnostics section when diagnostics exist', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 0,
|
||||
repoRoot: '/repo',
|
||||
files: [
|
||||
{
|
||||
filePath: '/repo/evals/bad.eval.ts',
|
||||
relativePath: 'evals/bad.eval.ts',
|
||||
helpers: {},
|
||||
cases: [],
|
||||
diagnostics: [],
|
||||
},
|
||||
],
|
||||
cases: [],
|
||||
diagnostics: [
|
||||
{
|
||||
severity: 'warning',
|
||||
message: 'Could not resolve policy',
|
||||
filePath: '/repo/evals/bad.eval.ts',
|
||||
location: { line: 5, column: 3 },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).toContain('Diagnostics');
|
||||
expect(report).toContain('1 diagnostics');
|
||||
expect(report).toContain(
|
||||
'⚠ evals/bad.eval.ts:5:3 — Could not resolve policy',
|
||||
);
|
||||
});
|
||||
|
||||
it('omits diagnostics section when there are none', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 1,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [makeCaseRecord()],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).not.toContain('Diagnostics');
|
||||
expect(report).not.toContain('⚠');
|
||||
});
|
||||
|
||||
it('includes helper name in case listing', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 1,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({
|
||||
helperName: 'customHelper',
|
||||
name: 'custom test',
|
||||
}),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const report = formatInventoryReport(result);
|
||||
|
||||
expect(report).toContain('• custom test [customHelper]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatInventoryJson', () => {
|
||||
it('snapshot: minimal inventory', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 1,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({
|
||||
name: 'basic eval',
|
||||
policy: 'ALWAYS_PASSES',
|
||||
suiteName: 'core',
|
||||
}),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const json = formatInventoryJson(result, FIXED_NOW);
|
||||
|
||||
expect(json).toMatchInlineSnapshot(`
|
||||
"{
|
||||
"version": 1,
|
||||
"generated": "2026-06-03T12:00:00.000Z",
|
||||
"summary": {
|
||||
"totalFiles": 1,
|
||||
"totalCases": 1,
|
||||
"totalDiagnostics": 0,
|
||||
"byPolicy": {
|
||||
"ALWAYS_PASSES": 1
|
||||
}
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"name": "basic eval",
|
||||
"filePath": "evals/test.eval.ts",
|
||||
"helperName": "evalTest",
|
||||
"baseHelperName": "evalTest",
|
||||
"policy": "ALWAYS_PASSES",
|
||||
"suiteName": "core",
|
||||
"suiteType": null,
|
||||
"timeout": null,
|
||||
"hasFiles": false,
|
||||
"hasPrompt": true,
|
||||
"location": {
|
||||
"line": 1,
|
||||
"column": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"diagnostics": []
|
||||
}"
|
||||
`);
|
||||
});
|
||||
|
||||
it('snapshot: mixed policies with diagnostics', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 2,
|
||||
totalCases: 3,
|
||||
repoRoot: '/repo',
|
||||
files: [
|
||||
{
|
||||
filePath: '/repo/evals/c.eval.ts',
|
||||
relativePath: 'evals/c.eval.ts',
|
||||
helpers: {},
|
||||
cases: [],
|
||||
diagnostics: [],
|
||||
},
|
||||
],
|
||||
cases: [
|
||||
makeCaseRecord({
|
||||
name: 'stable test',
|
||||
policy: 'ALWAYS_PASSES',
|
||||
relativePath: 'evals/a.eval.ts',
|
||||
}),
|
||||
makeCaseRecord({
|
||||
name: 'flaky test',
|
||||
policy: 'USUALLY_PASSES',
|
||||
suiteName: 'tools',
|
||||
suiteType: 'behavioral',
|
||||
relativePath: 'evals/b.eval.ts',
|
||||
}),
|
||||
makeCaseRecord({
|
||||
name: 'failing test',
|
||||
policy: 'USUALLY_FAILS',
|
||||
timeout: 30000,
|
||||
hasFiles: true,
|
||||
relativePath: 'evals/b.eval.ts',
|
||||
}),
|
||||
],
|
||||
diagnostics: [
|
||||
{
|
||||
severity: 'warning',
|
||||
message: 'Could not resolve policy',
|
||||
filePath: '/repo/evals/c.eval.ts',
|
||||
location: { line: 10, column: 5 },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const json = formatInventoryJson(result, FIXED_NOW);
|
||||
|
||||
expect(json).toMatchInlineSnapshot(`
|
||||
"{
|
||||
"version": 1,
|
||||
"generated": "2026-06-03T12:00:00.000Z",
|
||||
"summary": {
|
||||
"totalFiles": 2,
|
||||
"totalCases": 3,
|
||||
"totalDiagnostics": 1,
|
||||
"byPolicy": {
|
||||
"ALWAYS_PASSES": 1,
|
||||
"USUALLY_PASSES": 1,
|
||||
"USUALLY_FAILS": 1
|
||||
}
|
||||
},
|
||||
"cases": [
|
||||
{
|
||||
"name": "stable test",
|
||||
"filePath": "evals/a.eval.ts",
|
||||
"helperName": "evalTest",
|
||||
"baseHelperName": "evalTest",
|
||||
"policy": "ALWAYS_PASSES",
|
||||
"suiteName": null,
|
||||
"suiteType": null,
|
||||
"timeout": null,
|
||||
"hasFiles": false,
|
||||
"hasPrompt": true,
|
||||
"location": {
|
||||
"line": 1,
|
||||
"column": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "flaky test",
|
||||
"filePath": "evals/b.eval.ts",
|
||||
"helperName": "evalTest",
|
||||
"baseHelperName": "evalTest",
|
||||
"policy": "USUALLY_PASSES",
|
||||
"suiteName": "tools",
|
||||
"suiteType": "behavioral",
|
||||
"timeout": null,
|
||||
"hasFiles": false,
|
||||
"hasPrompt": true,
|
||||
"location": {
|
||||
"line": 1,
|
||||
"column": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "failing test",
|
||||
"filePath": "evals/b.eval.ts",
|
||||
"helperName": "evalTest",
|
||||
"baseHelperName": "evalTest",
|
||||
"policy": "USUALLY_FAILS",
|
||||
"suiteName": null,
|
||||
"suiteType": null,
|
||||
"timeout": 30000,
|
||||
"hasFiles": true,
|
||||
"hasPrompt": true,
|
||||
"location": {
|
||||
"line": 1,
|
||||
"column": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"diagnostics": [
|
||||
{
|
||||
"severity": "warning",
|
||||
"message": "Could not resolve policy",
|
||||
"filePath": "evals/c.eval.ts",
|
||||
"location": {
|
||||
"line": 10,
|
||||
"column": 5
|
||||
}
|
||||
}
|
||||
]
|
||||
}"
|
||||
`);
|
||||
});
|
||||
|
||||
it('snapshot: empty inventory', () => {
|
||||
const result: InventoryResult = makeEmptyResult();
|
||||
|
||||
const json = formatInventoryJson(result, FIXED_NOW);
|
||||
|
||||
expect(json).toMatchInlineSnapshot(`
|
||||
"{
|
||||
"version": 1,
|
||||
"generated": "2026-06-03T12:00:00.000Z",
|
||||
"summary": {
|
||||
"totalFiles": 0,
|
||||
"totalCases": 0,
|
||||
"totalDiagnostics": 0,
|
||||
"byPolicy": {}
|
||||
},
|
||||
"cases": [],
|
||||
"diagnostics": []
|
||||
}"
|
||||
`);
|
||||
});
|
||||
|
||||
it('produces valid JSON with version field', () => {
|
||||
const result: InventoryResult = {
|
||||
...makeEmptyResult(),
|
||||
totalFiles: 1,
|
||||
totalCases: 1,
|
||||
cases: [makeCaseRecord()],
|
||||
};
|
||||
|
||||
const json = formatInventoryJson(result, FIXED_NOW);
|
||||
const parsed: InventoryJsonOutput = JSON.parse(json);
|
||||
|
||||
expect(parsed.version).toBe(1);
|
||||
});
|
||||
|
||||
it('includes correct summary counts', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 3,
|
||||
totalCases: 4,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({ policy: 'ALWAYS_PASSES' }),
|
||||
makeCaseRecord({ policy: 'ALWAYS_PASSES' }),
|
||||
makeCaseRecord({ policy: 'USUALLY_PASSES' }),
|
||||
makeCaseRecord({ policy: 'USUALLY_FAILS' }),
|
||||
],
|
||||
diagnostics: [
|
||||
{
|
||||
severity: 'warning',
|
||||
message: 'test',
|
||||
filePath: 'test.ts',
|
||||
location: { line: 1, column: 1 },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result, FIXED_NOW),
|
||||
);
|
||||
|
||||
expect(parsed.summary).toEqual({
|
||||
totalFiles: 3,
|
||||
totalCases: 4,
|
||||
totalDiagnostics: 1,
|
||||
byPolicy: {
|
||||
ALWAYS_PASSES: 2,
|
||||
USUALLY_PASSES: 1,
|
||||
USUALLY_FAILS: 1,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('maps case fields correctly with nulls for missing optionals', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 1,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({
|
||||
name: 'detailed case',
|
||||
relativePath: 'evals/detail.eval.ts',
|
||||
helperName: 'appEvalTest',
|
||||
baseHelperName: 'appEvalTest',
|
||||
policy: 'USUALLY_PASSES',
|
||||
hasFiles: true,
|
||||
hasPrompt: true,
|
||||
location: { line: 42, column: 3 },
|
||||
}),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result, FIXED_NOW),
|
||||
);
|
||||
const firstCase = parsed.cases[0];
|
||||
|
||||
expect(firstCase).toEqual({
|
||||
name: 'detailed case',
|
||||
filePath: 'evals/detail.eval.ts',
|
||||
helperName: 'appEvalTest',
|
||||
baseHelperName: 'appEvalTest',
|
||||
policy: 'USUALLY_PASSES',
|
||||
suiteName: null,
|
||||
suiteType: null,
|
||||
timeout: null,
|
||||
hasFiles: true,
|
||||
hasPrompt: true,
|
||||
location: { line: 42, column: 3 },
|
||||
});
|
||||
});
|
||||
|
||||
it('uses relative paths not absolute paths', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 1,
|
||||
repoRoot: '/absolute/repo',
|
||||
files: [
|
||||
{
|
||||
filePath: '/absolute/repo/evals/test.eval.ts',
|
||||
relativePath: 'evals/test.eval.ts',
|
||||
helpers: {},
|
||||
cases: [],
|
||||
diagnostics: [],
|
||||
},
|
||||
],
|
||||
cases: [
|
||||
makeCaseRecord({
|
||||
filePath: '/absolute/repo/evals/test.eval.ts',
|
||||
relativePath: 'evals/test.eval.ts',
|
||||
}),
|
||||
],
|
||||
diagnostics: [
|
||||
{
|
||||
severity: 'warning',
|
||||
message: 'test diagnostic',
|
||||
filePath: '/absolute/repo/evals/test.eval.ts',
|
||||
location: { line: 1, column: 1 },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const json = formatInventoryJson(result, FIXED_NOW);
|
||||
|
||||
expect(json).not.toContain('/absolute/repo');
|
||||
expect(json).toContain('evals/test.eval.ts');
|
||||
|
||||
const parsed: InventoryJsonOutput = JSON.parse(json);
|
||||
expect(parsed.diagnostics[0].filePath).toBe('evals/test.eval.ts');
|
||||
});
|
||||
|
||||
it('relativizes absolute diagnostic path not in file lookup using repoRoot', () => {
|
||||
const repoRoot = '/repo';
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 0,
|
||||
repoRoot,
|
||||
files: [
|
||||
{
|
||||
filePath: '/repo/evals/known.eval.ts',
|
||||
relativePath: 'evals/known.eval.ts',
|
||||
helpers: {},
|
||||
cases: [],
|
||||
diagnostics: [],
|
||||
},
|
||||
],
|
||||
cases: [],
|
||||
diagnostics: [
|
||||
{
|
||||
severity: 'warning',
|
||||
message: 'cross-file diagnostic',
|
||||
filePath: '/repo/evals/other.eval.ts',
|
||||
location: { line: 1, column: 1 },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const json = formatInventoryJson(result, FIXED_NOW);
|
||||
const parsed: InventoryJsonOutput = JSON.parse(json);
|
||||
|
||||
expect(parsed.diagnostics[0].filePath).toBe('evals/other.eval.ts');
|
||||
expect(parsed.diagnostics[0].filePath).not.toMatch(/^\//);
|
||||
});
|
||||
|
||||
it('includes policies not listed in POLICY_ORDER in byPolicy', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 2,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({ policy: 'ALWAYS_PASSES' }),
|
||||
makeCaseRecord({ policy: 'unknown' }),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result, FIXED_NOW),
|
||||
);
|
||||
|
||||
expect(parsed.summary.byPolicy).toEqual({
|
||||
ALWAYS_PASSES: 1,
|
||||
unknown: 1,
|
||||
});
|
||||
|
||||
const sum = Object.values(parsed.summary.byPolicy).reduce(
|
||||
(a, b) => a + b,
|
||||
0,
|
||||
);
|
||||
expect(sum).toBe(parsed.summary.totalCases);
|
||||
});
|
||||
|
||||
it('emits deterministic output', () => {
|
||||
const result: InventoryResult = {
|
||||
totalFiles: 1,
|
||||
totalCases: 2,
|
||||
repoRoot: '/repo',
|
||||
files: [],
|
||||
cases: [
|
||||
makeCaseRecord({ name: 'a', policy: 'ALWAYS_PASSES' }),
|
||||
makeCaseRecord({ name: 'b', policy: 'USUALLY_PASSES' }),
|
||||
],
|
||||
diagnostics: [],
|
||||
};
|
||||
|
||||
const first = formatInventoryJson(result, FIXED_NOW);
|
||||
const second = formatInventoryJson(result, FIXED_NOW);
|
||||
|
||||
expect(first).toBe(second);
|
||||
});
|
||||
|
||||
it('generated field is valid ISO-8601', () => {
|
||||
const result: InventoryResult = makeEmptyResult();
|
||||
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result),
|
||||
);
|
||||
|
||||
const date = new Date(parsed.generated);
|
||||
expect(date.getTime()).not.toBeNaN();
|
||||
expect(parsed.generated).toMatch(
|
||||
/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/,
|
||||
);
|
||||
});
|
||||
|
||||
describe('environment overrides for timestamp', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it('uses SOURCE_DATE_EPOCH if set', () => {
|
||||
vi.stubEnv('SOURCE_DATE_EPOCH', '1700000000');
|
||||
const result: InventoryResult = makeEmptyResult();
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result),
|
||||
);
|
||||
expect(parsed.generated).toBe('2023-11-14T22:13:20.000Z');
|
||||
});
|
||||
|
||||
it('uses epoch 0 if EVAL_INVENTORY_STABLE_DATE is set', () => {
|
||||
vi.stubEnv('EVAL_INVENTORY_STABLE_DATE', '1');
|
||||
const result: InventoryResult = makeEmptyResult();
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result),
|
||||
);
|
||||
expect(parsed.generated).toBe('1970-01-01T00:00:00.000Z');
|
||||
});
|
||||
|
||||
it('uses epoch 0 if EVAL_INVENTORY_DETERMINISTIC is set', () => {
|
||||
vi.stubEnv('EVAL_INVENTORY_DETERMINISTIC', 'true');
|
||||
const result: InventoryResult = makeEmptyResult();
|
||||
const parsed: InventoryJsonOutput = JSON.parse(
|
||||
formatInventoryJson(result),
|
||||
);
|
||||
expect(parsed.generated).toBe('1970-01-01T00:00:00.000Z');
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
main as generateKeybindingDocs,
|
||||
renderDocumentation,
|
||||
type KeybindingDocSection,
|
||||
} from '../generate-keybindings-doc.ts';
|
||||
import { KeyBinding } from '../../packages/cli/src/ui/key/keyBindings.js';
|
||||
|
||||
describe('generate-keybindings-doc', () => {
|
||||
it('keeps keyboard shortcut documentation in sync in check mode', async () => {
|
||||
const previousExitCode = process.exitCode;
|
||||
try {
|
||||
process.exitCode = 0;
|
||||
await expect(
|
||||
generateKeybindingDocs(['--check']),
|
||||
).resolves.toBeUndefined();
|
||||
expect(process.exitCode).toBe(0);
|
||||
} finally {
|
||||
process.exitCode = previousExitCode;
|
||||
}
|
||||
});
|
||||
|
||||
it('renders provided sections into markdown tables', () => {
|
||||
const sections: KeybindingDocSection[] = [
|
||||
{
|
||||
title: 'Custom Controls',
|
||||
commands: [
|
||||
{
|
||||
command: 'custom.trigger',
|
||||
description: 'Trigger custom action.',
|
||||
bindings: [new KeyBinding('ctrl+x')],
|
||||
},
|
||||
{
|
||||
command: 'custom.submit',
|
||||
description: 'Submit with Enter if no modifiers are held.',
|
||||
bindings: [new KeyBinding('enter')],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
title: 'Navigation',
|
||||
commands: [
|
||||
{
|
||||
command: 'nav.up',
|
||||
description: 'Move up through results.',
|
||||
bindings: [new KeyBinding('up'), new KeyBinding('ctrl+p')],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const markdown = renderDocumentation(sections);
|
||||
expect(markdown).toContain('#### Custom Controls');
|
||||
expect(markdown).toContain('`custom.trigger`');
|
||||
expect(markdown).toContain('Trigger custom action.');
|
||||
expect(markdown).toContain('`Ctrl+X`');
|
||||
expect(markdown).toContain('`custom.submit`');
|
||||
expect(markdown).toContain('Submit with Enter if no modifiers are held.');
|
||||
expect(markdown).toContain('`Enter`');
|
||||
expect(markdown).toContain('#### Navigation');
|
||||
expect(markdown).toContain('`nav.up`');
|
||||
expect(markdown).toContain('Move up through results.');
|
||||
expect(markdown).toContain('`Up`<br />`Ctrl+P`');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,29 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi } from 'vitest';
|
||||
import { main as generateDocs } from '../generate-settings-doc.ts';
|
||||
|
||||
vi.mock('fs', () => ({
|
||||
readFileSync: vi.fn().mockReturnValue(''),
|
||||
createWriteStream: vi.fn(() => ({
|
||||
write: vi.fn(),
|
||||
on: vi.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
describe('generate-settings-doc', () => {
|
||||
it('keeps documentation in sync in check mode', async () => {
|
||||
const previousExitCode = process.exitCode;
|
||||
try {
|
||||
process.exitCode = 0;
|
||||
await expect(generateDocs(['--check'])).resolves.toBeUndefined();
|
||||
expect(process.exitCode).toBe(0);
|
||||
} finally {
|
||||
process.exitCode = previousExitCode;
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { main as generateSchema } from '../generate-settings-schema.ts';
|
||||
|
||||
vi.mock('fs', () => ({
|
||||
readFileSync: vi.fn().mockReturnValue(''),
|
||||
writeFileSync: vi.fn(),
|
||||
createWriteStream: vi.fn(() => ({
|
||||
write: vi.fn(),
|
||||
on: vi.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
describe('generate-settings-schema', () => {
|
||||
it('keeps schema in sync in check mode', async () => {
|
||||
const previousExitCode = process.exitCode;
|
||||
await expect(generateSchema(['--check'])).resolves.toBeUndefined();
|
||||
expect(process.exitCode).toBe(previousExitCode);
|
||||
});
|
||||
|
||||
it('includes $schema property in generated schema', async () => {
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const schemaPath = join(__dirname, '../../schemas/settings.schema.json');
|
||||
const schemaContent = await readFile(schemaPath, 'utf-8');
|
||||
const schema = JSON.parse(schemaContent);
|
||||
|
||||
// Verify $schema property exists in the schema's properties
|
||||
expect(schema.properties).toHaveProperty('$schema');
|
||||
expect(schema.properties.$schema).toEqual({
|
||||
type: 'string',
|
||||
title: 'Schema',
|
||||
description:
|
||||
'The URL of the JSON schema for this settings file. Used by editors for validation and autocompletion.',
|
||||
default:
|
||||
'https://raw.githubusercontent.com/google-gemini/gemini-cli/main/schemas/settings.schema.json',
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,209 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { vi, describe, it, expect, beforeEach } from 'vitest';
|
||||
import { getVersion } from '../get-release-version.js';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { readFileSync } from 'node:fs';
|
||||
|
||||
vi.mock('node:child_process');
|
||||
vi.mock('node:fs');
|
||||
|
||||
describe('getVersion', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.setSystemTime(new Date('2025-09-17T00:00:00.000Z'));
|
||||
// Mock package.json being read by getNightlyVersion
|
||||
vi.mocked(readFileSync).mockReturnValue(
|
||||
JSON.stringify({ version: '0.8.0' }),
|
||||
);
|
||||
});
|
||||
|
||||
// This is the base mock for a clean state with no conflicts or rollbacks
|
||||
const mockExecSync = (command) => {
|
||||
// NPM dist-tags
|
||||
if (command.includes('npm view') && command.includes('--tag=latest'))
|
||||
return '0.6.1';
|
||||
if (command.includes('npm view') && command.includes('--tag=preview'))
|
||||
return '0.7.0-preview.1';
|
||||
if (command.includes('npm view') && command.includes('--tag=nightly'))
|
||||
return '0.8.0-nightly.20250916.abcdef';
|
||||
|
||||
// NPM versions list
|
||||
if (command.includes('npm view') && command.includes('versions --json'))
|
||||
return JSON.stringify([
|
||||
'0.6.0',
|
||||
'0.6.1',
|
||||
'0.7.0-preview.0',
|
||||
'0.7.0-preview.1',
|
||||
'0.8.0-nightly.20250916.abcdef',
|
||||
]);
|
||||
|
||||
// Deprecation checks (default to not deprecated)
|
||||
if (command.includes('deprecated')) return '';
|
||||
|
||||
// Git Tag Mocks
|
||||
if (command.includes("git tag -l 'v[0-9].[0-9].[0-9]'")) return 'v0.6.1';
|
||||
if (command.includes("git tag -l 'v*-preview*'")) return 'v0.7.0-preview.1';
|
||||
if (command.includes("git tag -l 'v*-nightly*'"))
|
||||
return 'v0.8.0-nightly.20250916.abcdef';
|
||||
|
||||
// Git Hash Mock
|
||||
if (command.includes('git rev-parse --short HEAD')) return 'd3bf8a3d';
|
||||
|
||||
// For doesVersionExist checks - default to not found
|
||||
if (
|
||||
command.includes('npm view') &&
|
||||
command.includes('@google/gemini-cli@')
|
||||
) {
|
||||
throw new Error('NPM version not found');
|
||||
}
|
||||
if (command.includes('git tag -l')) return '';
|
||||
if (command.includes('gh release view')) {
|
||||
throw new Error('GH release not found');
|
||||
}
|
||||
|
||||
return '';
|
||||
};
|
||||
|
||||
describe('Happy Path - Version Calculation', () => {
|
||||
it('should calculate the next stable version from the latest preview', () => {
|
||||
vi.mocked(execSync).mockImplementation(mockExecSync);
|
||||
const result = getVersion({ type: 'stable' });
|
||||
expect(result.releaseVersion).toBe('0.7.0');
|
||||
expect(result.npmTag).toBe('latest');
|
||||
expect(result.previousReleaseTag).toBe('v0.6.1');
|
||||
});
|
||||
|
||||
it('should calculate the next preview version from the latest nightly', () => {
|
||||
vi.mocked(execSync).mockImplementation(mockExecSync);
|
||||
const result = getVersion({
|
||||
type: 'preview',
|
||||
'stable-base-version': '0.7.0',
|
||||
});
|
||||
expect(result.releaseVersion).toBe('0.8.0-preview.0');
|
||||
expect(result.npmTag).toBe('preview');
|
||||
expect(result.previousReleaseTag).toBe('v0.7.0-preview.1');
|
||||
});
|
||||
|
||||
it('should calculate the next nightly version from package.json', () => {
|
||||
vi.mocked(execSync).mockImplementation(mockExecSync);
|
||||
const result = getVersion({ type: 'nightly' });
|
||||
// Note: The base version now comes from package.json, not the previous nightly tag.
|
||||
expect(result.releaseVersion).toBe('0.8.0-nightly.20250917.gd3bf8a3d');
|
||||
expect(result.npmTag).toBe('nightly');
|
||||
expect(result.previousReleaseTag).toBe('v0.8.0-nightly.20250916.abcdef');
|
||||
});
|
||||
|
||||
it('should calculate the next patch version for a stable release', () => {
|
||||
vi.mocked(execSync).mockImplementation(mockExecSync);
|
||||
const result = getVersion({ type: 'patch', 'patch-from': 'stable' });
|
||||
expect(result.releaseVersion).toBe('0.6.2');
|
||||
expect(result.npmTag).toBe('latest');
|
||||
expect(result.previousReleaseTag).toBe('v0.6.1');
|
||||
});
|
||||
|
||||
it('should calculate the next patch version for a preview release', () => {
|
||||
vi.mocked(execSync).mockImplementation(mockExecSync);
|
||||
const result = getVersion({ type: 'patch', 'patch-from': 'preview' });
|
||||
expect(result.releaseVersion).toBe('0.7.0-preview.2');
|
||||
expect(result.npmTag).toBe('preview');
|
||||
expect(result.previousReleaseTag).toBe('v0.7.0-preview.1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Advanced Scenarios', () => {
|
||||
it('should ignore a deprecated version and use the next highest', () => {
|
||||
const mockWithDeprecated = (command) => {
|
||||
// The highest nightly is 0.9.0, but it's deprecated
|
||||
if (command.includes('npm view') && command.includes('versions --json'))
|
||||
return JSON.stringify([
|
||||
'0.8.0-nightly.20250916.abcdef',
|
||||
'0.9.0-nightly.20250917.deprecated', // This one is deprecated
|
||||
]);
|
||||
// Mock the deprecation check
|
||||
if (
|
||||
command.includes(
|
||||
'npm view @google/gemini-cli@0.9.0-nightly.20250917.deprecated deprecated',
|
||||
)
|
||||
)
|
||||
return 'This version is deprecated';
|
||||
// The dist-tag still points to the older, valid version
|
||||
if (command.includes('npm view') && command.includes('--tag=nightly'))
|
||||
return '0.8.0-nightly.20250916.abcdef';
|
||||
|
||||
return mockExecSync(command);
|
||||
};
|
||||
vi.mocked(execSync).mockImplementation(mockWithDeprecated);
|
||||
|
||||
const result = getVersion({
|
||||
type: 'preview',
|
||||
'stable-base-version': '0.7.0',
|
||||
});
|
||||
// It should base the preview off 0.8.0, not the deprecated 0.9.0
|
||||
expect(result.releaseVersion).toBe('0.8.0-preview.0');
|
||||
});
|
||||
|
||||
it('should auto-increment patch version if the calculated one already exists', () => {
|
||||
const mockWithConflict = (command) => {
|
||||
// The calculated version 0.7.0 already exists as a git tag
|
||||
if (command.includes("git tag -l 'v0.7.0'")) return 'v0.7.0';
|
||||
// The next version, 0.7.1, is available
|
||||
if (command.includes("git tag -l 'v0.7.1'")) return '';
|
||||
|
||||
return mockExecSync(command);
|
||||
};
|
||||
vi.mocked(execSync).mockImplementation(mockWithConflict);
|
||||
|
||||
const result = getVersion({ type: 'stable' });
|
||||
// Should have skipped 0.7.0 and landed on 0.7.1
|
||||
expect(result.releaseVersion).toBe('0.7.1');
|
||||
});
|
||||
|
||||
it('should auto-increment preview number if the calculated one already exists', () => {
|
||||
const mockWithConflict = (command) => {
|
||||
// The calculated preview 0.8.0-preview.0 already exists on NPM
|
||||
if (
|
||||
command.includes(
|
||||
'npm view @google/gemini-cli@0.8.0-preview.0 version',
|
||||
)
|
||||
)
|
||||
return '0.8.0-preview.0';
|
||||
// The next one is available
|
||||
if (
|
||||
command.includes(
|
||||
'npm view @google/gemini-cli@0.8.0-preview.1 version',
|
||||
)
|
||||
)
|
||||
throw new Error('Not found');
|
||||
|
||||
return mockExecSync(command);
|
||||
};
|
||||
vi.mocked(execSync).mockImplementation(mockWithConflict);
|
||||
|
||||
const result = getVersion({
|
||||
type: 'preview',
|
||||
'stable-base-version': '0.7.0',
|
||||
});
|
||||
// Should have skipped preview.0 and landed on preview.1
|
||||
expect(result.releaseVersion).toBe('0.8.0-preview.1');
|
||||
});
|
||||
|
||||
it('should preserve a git hash with a leading zero via the g prefix', () => {
|
||||
const mockWithLeadingZeroHash = (command) => {
|
||||
// Return an all-numeric hash with a leading zero
|
||||
if (command.includes('git rev-parse --short HEAD')) return '017972622';
|
||||
return mockExecSync(command);
|
||||
};
|
||||
vi.mocked(execSync).mockImplementation(mockWithLeadingZeroHash);
|
||||
|
||||
const result = getVersion({ type: 'nightly' });
|
||||
// The 'g' prefix forces semver to treat this as an alphanumeric
|
||||
// identifier, preventing it from stripping the leading zero.
|
||||
expect(result.releaseVersion).toBe('0.8.0-nightly.20250917.g017972622');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,368 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
|
||||
/**
|
||||
* Helper function to run the patch-create-comment script with given parameters
|
||||
*/
|
||||
function runPatchCreateComment(args, env = {}) {
|
||||
const scriptPath = join(
|
||||
process.cwd(),
|
||||
'scripts/releasing/patch-create-comment.js',
|
||||
);
|
||||
const fullEnv = {
|
||||
...process.env,
|
||||
...env,
|
||||
};
|
||||
|
||||
try {
|
||||
const result = execSync(`node ${scriptPath} ${args}`, {
|
||||
encoding: 'utf8',
|
||||
env: fullEnv,
|
||||
});
|
||||
return { stdout: result, stderr: '', success: true };
|
||||
} catch (error) {
|
||||
return {
|
||||
stdout: error.stdout || '',
|
||||
stderr: error.stderr || '',
|
||||
success: false,
|
||||
code: error.status,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
describe('patch-create-comment', () => {
|
||||
beforeEach(() => {
|
||||
vi.stubEnv();
|
||||
// Always run in test mode to avoid GitHub API calls
|
||||
vi.stubEnv('TEST_MODE', 'true');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
describe('Environment flag', () => {
|
||||
it('can be overridden with a flag', () => {
|
||||
vi.stubEnv('ENVIRONMENT', 'dev');
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --environment prod --commit abc1234 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
expect(result.stdout).toContain('Environment**: `prod`');
|
||||
});
|
||||
|
||||
it('reads from the ENVIRONMENT env variable', () => {
|
||||
vi.stubEnv('ENVIRONMENT', 'dev');
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit abc1234 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
expect(result.stdout).toContain('Environment**: `dev`');
|
||||
});
|
||||
|
||||
it('fails if the ENVIRONMENT is bogus', () => {
|
||||
vi.stubEnv('ENVIRONMENT', 'totally-bogus');
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit abc1234 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
);
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.stderr).toContain(
|
||||
'Argument: environment, Given: "totally-bogus", Choices: "prod", "dev"',
|
||||
);
|
||||
});
|
||||
|
||||
it('defaults to prod if not specified', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit abc1234 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
expect(result.stdout).toContain('Environment**: `prod`');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Environment Variable vs File Reading', () => {
|
||||
it('should prefer LOG_CONTENT environment variable over file', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit abc1234 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Creating hotfix branch hotfix/v0.5.3/preview/cherry-pick-abc1234 from release/v0.5.3',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
expect(result.stdout).toContain('Channel**: `preview`');
|
||||
expect(result.stdout).toContain('Commit**: `abc1234`');
|
||||
});
|
||||
|
||||
it('should use empty log content when LOG_CONTENT is not set', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 1 --commit abc1234 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{}, // No LOG_CONTENT env var
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'❌ **[Step 2/4] Patch creation failed!**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'There was an error creating the patch release',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Log Content Parsing - Success Scenarios', () => {
|
||||
it('should generate success comment for clean cherry-pick', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit abc1234 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Creating hotfix branch hotfix/v0.4.1/stable/cherry-pick-abc1234 from release/v0.4.1\n✅ Cherry-pick successful - no conflicts detected',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
expect(result.stdout).toContain('Channel**: `stable`');
|
||||
expect(result.stdout).toContain('Commit**: `abc1234`');
|
||||
expect(result.stdout).not.toContain('⚠️ Status');
|
||||
});
|
||||
|
||||
it('should generate conflict comment for cherry-pick with conflicts', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit def5678 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Creating hotfix branch hotfix/v0.5.0-preview.2/preview/cherry-pick-def5678 from release/v0.5.0-preview.2\nCherry-pick has conflicts in 2 file(s):\nCONFLICT (content): Merge conflict in package.json',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
expect(result.stdout).toContain(
|
||||
'⚠️ Status**: Cherry-pick conflicts detected',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'⚠️ **Resolve conflicts** in the hotfix PR first',
|
||||
);
|
||||
expect(result.stdout).toContain('Channel**: `preview`');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Log Content Parsing - Existing PR Scenarios', () => {
|
||||
it('should detect existing PR and generate appropriate comment', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit ghi9012 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Hotfix branch hotfix/v0.4.1/stable/cherry-pick-ghi9012 already has an open PR.\nFound existing PR #8700: https://github.com/google-gemini/gemini-cli/pull/8700',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'ℹ️ **[Step 2/4] Patch PR already exists!**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'A patch PR for this change already exists: [#8700](https://github.com/google-gemini/gemini-cli/pull/8700)',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'Review and approve the existing patch PR',
|
||||
);
|
||||
});
|
||||
|
||||
it('should detect branch exists but no PR scenario', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit jkl3456 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Hotfix branch hotfix/v0.5.0-preview.2/preview/cherry-pick-jkl3456 exists but has no open PR.\nHotfix branch hotfix/v0.5.0-preview.2/preview/cherry-pick-jkl3456 already exists.',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'ℹ️ **[Step 2/4] Patch branch exists but no PR found!**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'Delete the branch: `git branch -D hotfix/v0.5.0-preview.2/preview/cherry-pick-jkl3456`',
|
||||
);
|
||||
expect(result.stdout).toContain('Run the patch command again');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Log Content Parsing - Failure Scenarios', () => {
|
||||
it('should generate failure comment when exit code is non-zero', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 1 --commit mno7890 --channel stable --repository google-gemini/gemini-cli --run-id 12345 --test',
|
||||
{
|
||||
LOG_CONTENT: 'Error: Failed to create patch',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'❌ **[Step 2/4] Patch creation failed!**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'There was an error creating the patch release',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'View workflow run](https://github.com/google-gemini/gemini-cli/actions/runs/12345)',
|
||||
);
|
||||
});
|
||||
|
||||
it('should generate fallback failure comment when no output is generated', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 1 --commit pqr4567 --channel preview --repository google-gemini/gemini-cli --run-id 67890 --test',
|
||||
{
|
||||
LOG_CONTENT: '',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'❌ **[Step 2/4] Patch creation failed!**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'There was an error creating the patch release',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Channel and NPM Tag Detection', () => {
|
||||
it('should correctly map stable channel to latest npm tag', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit stu8901 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Creating hotfix branch hotfix/v0.4.1/stable/cherry-pick-stu8901 from release/v0.4.1',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('will publish to npm tag `latest`');
|
||||
});
|
||||
|
||||
it('should correctly map preview channel to preview npm tag', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit vwx2345 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Creating hotfix branch hotfix/v0.5.0-preview.2/preview/cherry-pick-vwx2345 from release/v0.5.0-preview.2',
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain('will publish to npm tag `preview`');
|
||||
});
|
||||
});
|
||||
|
||||
describe('No Original PR Scenario', () => {
|
||||
it('should skip comment when no original PR is specified', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 0 --exit-code 0 --commit yza6789 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT:
|
||||
'Creating hotfix branch hotfix/v0.4.1/stable/cherry-pick-yza6789 from release/v0.4.1',
|
||||
ORIGINAL_PR: '', // Override with empty PR
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'No original PR specified, skipping comment',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Error Handling', () => {
|
||||
it('should handle empty LOG_CONTENT gracefully', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 1 --commit bcd0123 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{ LOG_CONTENT: '' }, // Empty log content
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'❌ **[Step 2/4] Patch creation failed!**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'There was an error creating the patch release',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('GitHub App Permission Scenarios', () => {
|
||||
it('should parse manual commands with clipboard emoji correctly', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 1 --commit abc1234 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
{
|
||||
LOG_CONTENT: `❌ Failed to create release branch due to insufficient GitHub App permissions.
|
||||
|
||||
📋 Please run these commands manually to create the branch:
|
||||
|
||||
\`\`\`bash
|
||||
git checkout -b hotfix/v0.4.1/stable/cherry-pick-abc1234 v0.4.1
|
||||
git push origin hotfix/v0.4.1/stable/cherry-pick-abc1234
|
||||
\`\`\``,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'🔒 **[Step 2/4] GitHub App Permission Issue**',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'Please run these commands manually to create the release branch:',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'git checkout -b hotfix/v0.4.1/stable/cherry-pick-abc1234 v0.4.1',
|
||||
);
|
||||
expect(result.stdout).toContain(
|
||||
'git push origin hotfix/v0.4.1/stable/cherry-pick-abc1234',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Test Mode Flag', () => {
|
||||
it('should generate mock content in test mode for success', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 0 --commit efg4567 --channel preview --repository google-gemini/gemini-cli --test',
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'🧪 TEST MODE - No API calls will be made',
|
||||
);
|
||||
expect(result.stdout).toContain('🚀 **[Step 2/4] Patch PR Created!**');
|
||||
});
|
||||
|
||||
it('should generate mock content in test mode for failure', () => {
|
||||
const result = runPatchCreateComment(
|
||||
'--original-pr 8655 --exit-code 1 --commit hij8901 --channel stable --repository google-gemini/gemini-cli --test',
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.stdout).toContain(
|
||||
'❌ **[Step 2/4] Patch creation failed!**',
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
vi.unmock('fs');
|
||||
vi.unmock('node:fs');
|
||||
import * as esbuild from 'esbuild';
|
||||
import path from 'node:path';
|
||||
import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { fileURLToPath, pathToFileURL } from 'node:url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const projectRoot = path.resolve(__dirname, '../../');
|
||||
|
||||
describe('proxy-agent bundle shape', () => {
|
||||
it('preserves named constructors after ESM splitting', async () => {
|
||||
const tmpDir = mkdtempSync(path.join(tmpdir(), 'gemini-proxy-test-'));
|
||||
const entryFile = path.join(tmpDir, 'entry.ts');
|
||||
|
||||
// Create a minimal entry file that dynamically imports the proxy agents
|
||||
writeFileSync(
|
||||
entryFile,
|
||||
`
|
||||
export async function getAgents() {
|
||||
const httpsMod = await import('https-proxy-agent');
|
||||
const httpMod = await import('http-proxy-agent');
|
||||
return {
|
||||
https: httpsMod,
|
||||
http: httpMod,
|
||||
};
|
||||
}
|
||||
`,
|
||||
);
|
||||
|
||||
// Bundle with the exact same splitting config and aliases as cliConfig
|
||||
await esbuild.build({
|
||||
entryPoints: { gemini: entryFile },
|
||||
outdir: path.join(tmpDir, 'bundle'),
|
||||
bundle: true,
|
||||
splitting: true,
|
||||
format: 'esm',
|
||||
platform: 'node',
|
||||
outExtension: { '.js': '.mjs' },
|
||||
alias: {
|
||||
'https-proxy-agent': path.resolve(
|
||||
projectRoot,
|
||||
'packages/cli/src/patches/https-proxy-agent.ts',
|
||||
),
|
||||
'http-proxy-agent': path.resolve(
|
||||
projectRoot,
|
||||
'packages/cli/src/patches/http-proxy-agent.ts',
|
||||
),
|
||||
},
|
||||
});
|
||||
|
||||
// Import the bundled chunk
|
||||
const bundledEntryUrl = pathToFileURL(
|
||||
path.join(tmpDir, 'bundle/gemini.mjs'),
|
||||
).href;
|
||||
const { getAgents } = await import(bundledEntryUrl);
|
||||
|
||||
const { https, http } = await getAgents();
|
||||
|
||||
// Verify named exports exist
|
||||
expect(typeof https.HttpsProxyAgent).toBe('function');
|
||||
expect(typeof http.HttpProxyAgent).toBe('function');
|
||||
|
||||
// Verify they are constructable
|
||||
expect(() => new https.HttpsProxyAgent('http://127.0.0.1:9')).not.toThrow();
|
||||
expect(() => new http.HttpProxyAgent('http://127.0.0.1:9')).not.toThrow();
|
||||
|
||||
// Cleanup
|
||||
rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { vi, describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
|
||||
const mockSpawn = vi.fn(() => ({ on: vi.fn(), pid: 123 }));
|
||||
|
||||
vi.mock('node:child_process', () => ({
|
||||
spawn: mockSpawn,
|
||||
execSync: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('node:fs', () => ({
|
||||
readFileSync: vi.fn(),
|
||||
writeFileSync: vi.fn(),
|
||||
openSync: vi.fn(() => 1),
|
||||
unlinkSync: vi.fn(),
|
||||
mkdirSync: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('../telemetry_utils.js', () => ({
|
||||
ensureBinary: vi.fn(() => Promise.resolve('/fake/path/to/otelcol-contrib')),
|
||||
waitForPort: vi.fn(() => Promise.resolve()),
|
||||
manageTelemetrySettings: vi.fn(),
|
||||
registerCleanup: vi.fn(),
|
||||
fileExists: vi.fn(() => true), // Assume all files exist for simplicity
|
||||
OTEL_DIR: '/tmp/otel',
|
||||
BIN_DIR: '/tmp/bin',
|
||||
}));
|
||||
|
||||
describe('telemetry_gcp.js', () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules(); // This is key to re-run the script
|
||||
vi.clearAllMocks();
|
||||
process.env.OTLP_GOOGLE_CLOUD_PROJECT = 'test-project';
|
||||
// Clear the env var before each test
|
||||
delete process.env.GEMINI_CLI_CREDENTIALS_PATH;
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
delete process.env.OTLP_GOOGLE_CLOUD_PROJECT;
|
||||
});
|
||||
|
||||
it('should not set GOOGLE_APPLICATION_CREDENTIALS when env var is not set', async () => {
|
||||
await import('../telemetry_gcp.js');
|
||||
|
||||
expect(mockSpawn).toHaveBeenCalled();
|
||||
const spawnOptions = mockSpawn.mock.calls[0][2];
|
||||
expect(spawnOptions?.env).not.toHaveProperty(
|
||||
'GOOGLE_APPLICATION_CREDENTIALS',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { vi } from 'vitest';
|
||||
|
||||
vi.mock('fs', async () => {
|
||||
const actual = await vi.importActual<typeof import('fs')>('fs');
|
||||
return {
|
||||
...actual,
|
||||
appendFileSync: vi.fn(),
|
||||
};
|
||||
});
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2026 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
buildToolRegistry,
|
||||
resolveToolName,
|
||||
getToolsByCategory,
|
||||
type ToolCategory,
|
||||
} from '../utils/tool-registry.js';
|
||||
|
||||
describe('tool-registry', () => {
|
||||
const registry = buildToolRegistry();
|
||||
|
||||
describe('buildToolRegistry', () => {
|
||||
it('includes all canonical built-in tools', () => {
|
||||
expect(registry.totalTools).toBeGreaterThanOrEqual(26);
|
||||
});
|
||||
|
||||
it('every tool has a valid category', () => {
|
||||
for (const [name, entry] of registry.tools) {
|
||||
expect(entry.category).toBeTruthy();
|
||||
expect(entry.name).toBe(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('byCategory entries match tools map', () => {
|
||||
let categoryTotal = 0;
|
||||
for (const [, entries] of registry.byCategory) {
|
||||
for (const entry of entries) {
|
||||
expect(registry.tools.get(entry.name)).toBe(entry);
|
||||
}
|
||||
categoryTotal += entries.length;
|
||||
}
|
||||
expect(categoryTotal).toBe(registry.totalTools);
|
||||
});
|
||||
|
||||
it('aliasLookup covers every canonical name', () => {
|
||||
for (const name of registry.tools.keys()) {
|
||||
expect(registry.aliasLookup.get(name)).toBe(name);
|
||||
}
|
||||
});
|
||||
|
||||
it('aliasLookup covers every legacy alias', () => {
|
||||
for (const [, entry] of registry.tools) {
|
||||
for (const alias of entry.aliases) {
|
||||
expect(registry.aliasLookup.get(alias)).toBe(entry.name);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('is deterministic across calls', () => {
|
||||
const second = buildToolRegistry();
|
||||
expect([...second.tools.keys()]).toEqual([...registry.tools.keys()]);
|
||||
expect(second.totalTools).toBe(registry.totalTools);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveToolName', () => {
|
||||
it('resolves canonical names to themselves', () => {
|
||||
expect(resolveToolName(registry, 'grep_search')).toBe('grep_search');
|
||||
expect(resolveToolName(registry, 'run_shell_command')).toBe(
|
||||
'run_shell_command',
|
||||
);
|
||||
});
|
||||
|
||||
it('resolves legacy alias to canonical name', () => {
|
||||
expect(resolveToolName(registry, 'search_file_content')).toBe(
|
||||
'grep_search',
|
||||
);
|
||||
});
|
||||
|
||||
it('returns undefined for unknown tool names', () => {
|
||||
expect(resolveToolName(registry, 'nonexistent_tool')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for empty string', () => {
|
||||
expect(resolveToolName(registry, '')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getToolsByCategory', () => {
|
||||
it('returns file-system tools', () => {
|
||||
const tools = getToolsByCategory(registry, 'file-system');
|
||||
const names = tools.map((t) => t.name);
|
||||
expect(names).toContain('glob');
|
||||
expect(names).toContain('grep_search');
|
||||
expect(names).toContain('read_file');
|
||||
expect(names).toContain('write_file');
|
||||
expect(names).toContain('replace');
|
||||
});
|
||||
|
||||
it('returns task-tracker tools', () => {
|
||||
const tools = getToolsByCategory(registry, 'task-tracker');
|
||||
const names = tools.map((t) => t.name);
|
||||
expect(names).toContain('tracker_create_task');
|
||||
expect(names).toContain('tracker_update_task');
|
||||
expect(names).toContain('tracker_get_task');
|
||||
expect(names).toContain('tracker_list_tasks');
|
||||
expect(names).toContain('tracker_add_dependency');
|
||||
expect(names).toContain('tracker_visualize');
|
||||
expect(names).toHaveLength(6);
|
||||
});
|
||||
|
||||
it('returns agent tools', () => {
|
||||
const tools = getToolsByCategory(registry, 'agent');
|
||||
const names = tools.map((t) => t.name);
|
||||
expect(names).toContain('invoke_agent');
|
||||
expect(names).toContain('complete_task');
|
||||
expect(names).toContain('update_topic');
|
||||
});
|
||||
|
||||
it('returns empty array for unknown category', () => {
|
||||
expect(
|
||||
getToolsByCategory(registry, 'nonexistent' as ToolCategory),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it('every defined category has at least one tool', () => {
|
||||
const expectedCategories: ToolCategory[] = [
|
||||
'file-system',
|
||||
'shell',
|
||||
'web',
|
||||
'planning',
|
||||
'user-interaction',
|
||||
'skills',
|
||||
'task-tracker',
|
||||
'agent',
|
||||
'mcp',
|
||||
];
|
||||
for (const cat of expectedCategories) {
|
||||
expect(getToolsByCategory(registry, cat).length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2025 Google LLC
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
globals: true,
|
||||
environment: 'node',
|
||||
include: ['scripts/tests/**/*.test.{js,ts}'],
|
||||
setupFiles: ['scripts/tests/test-setup.ts'],
|
||||
coverage: {
|
||||
provider: 'v8',
|
||||
reporter: ['text', 'lcov'],
|
||||
},
|
||||
poolOptions: {
|
||||
threads: {
|
||||
minThreads: 8,
|
||||
maxThreads: 16,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user