import { describe, expect, it } from 'vitest'; import { handleTrajectoryStepCount, handleTrajectoryToolArgsMatch, handleTrajectoryToolSequence, handleTrajectoryToolUsed, } from '../../src/assertions/trajectory'; import { extractTrajectorySteps, summarizeTrajectoryForJudge, } from '../../src/assertions/trajectoryUtils'; import { createMockProvider, createProviderResponse } from '../factories/provider'; import type { AssertionParams, AtomicTestCase } from '../../src/types/index'; import type { TraceData } from '../../src/types/tracing'; const mockProvider = createMockProvider({ id: 'mock', response: createProviderResponse({ output: 'mock' }), }); const mockTraceData: TraceData = { traceId: 'test-trace-id', evaluationId: 'test-evaluation-id', testCaseId: 'test-test-case-id', metadata: { test: 'value' }, spans: [ { spanId: 'span-1', name: 'chat gpt-5', startTime: 1000, endTime: 1800, attributes: { 'promptfoo.provider.id': 'openai:gpt-5', }, }, { spanId: 'span-2', name: 'tool.call', startTime: 1100, endTime: 1200, attributes: { 'tool.name': 'search_orders', 'tool.arguments': '{"order_id":"123","include_history":false}', }, }, { spanId: 'span-3', name: 'mcp inventory/search_inventory', startTime: 1250, endTime: 1350, attributes: { 'codex.item.type': 'mcp_tool_call', 'codex.mcp.server': 'inventory', 'codex.mcp.tool': 'search_inventory', 'codex.mcp.input': '{"query":"quantum computing","limit":3}', }, }, { spanId: 'span-4', name: 'tool.call', startTime: 1400, endTime: 1500, attributes: { 'tool.name': 'compose_reply', 'tool.args': { tone: 'friendly', citations: ['doc_1', 'doc_2'], }, }, }, { spanId: 'span-5', name: 'exec ls', startTime: 1550, endTime: 1600, attributes: { 'codex.item.type': 'command_execution', 'codex.command': 'ls -la', }, }, { spanId: 'span-6', name: 'reasoning_plan', startTime: 1650, endTime: 1700, attributes: { 'reasoning.step': 'plan', }, }, ], }; const defaultParams = { assertionValueContext: { vars: {}, test: {} as AtomicTestCase, prompt: 'test prompt', logProbs: undefined, provider: mockProvider, providerResponse: { output: 'test output' }, trace: mockTraceData, }, output: 'test output', outputString: 'test output', providerResponse: { output: 'test output' }, test: {} as AtomicTestCase, inverse: false, }; describe('trajectory utilities', () => { it('treats configured shell tool names as command steps', () => { const customShellTrace: TraceData = { traceId: 'custom-shell', evaluationId: 'eval-1', testCaseId: 'tc-1', metadata: {}, spans: [ { spanId: 'span-1', name: 'tool.call', startTime: 100, endTime: 200, attributes: { 'tool.name': 'bash', 'tool.arguments': JSON.stringify({ cmd: 'pytest tests/' }), }, }, ], }; // By default, a tool named 'bash' normalizes to type:tool, not type:command. expect(extractTrajectorySteps(customShellTrace)[0].type).toBe('tool'); const steps = extractTrajectorySteps({ ...customShellTrace, metadata: { commandToolNames: ['bash'] }, }); expect(steps[0].type).toBe('command'); // The command argument should also be reflected in the step name. expect(steps[0].name).toBe('pytest tests/'); }); it('matches configured command tool names case-insensitively and ignores invalid entries', () => { const trace: TraceData = { traceId: 'custom-shell-2', evaluationId: 'eval-1', testCaseId: 'tc-1', metadata: {}, spans: [ { spanId: 'span-1', name: 'tool.call', startTime: 100, endTime: 200, attributes: { 'tool.name': 'bash', 'tool.arguments': JSON.stringify({ cmd: 'ls -la' }), }, }, ], }; // An uppercase override matches the lowercase tool name. const upper = extractTrajectorySteps({ ...trace, metadata: { commandToolNames: ['BASH'] } }); expect(upper[0].type).toBe('command'); expect(upper[0].name).toBe('ls -la'); // Non-string / blank entries are ignored without throwing; the valid name still applies. const mixed = extractTrajectorySteps({ ...trace, metadata: { commandToolNames: ['bash', 123, '', ' '] as any }, }); expect(mixed[0].type).toBe('command'); }); it('extracts normalized trajectory steps from trace spans', () => { const steps = extractTrajectorySteps(mockTraceData); expect(steps.map((step) => ({ type: step.type, name: step.name }))).toEqual([ { type: 'span', name: 'chat gpt-5' }, { type: 'tool', name: 'search_orders' }, { type: 'tool', name: 'search_inventory' }, { type: 'tool', name: 'compose_reply' }, { type: 'command', name: 'ls -la' }, { type: 'reasoning', name: 'reasoning_plan' }, ]); expect(steps[4].aliases).toContain('ls'); expect(steps[2].aliases).toContain('mcp inventory/search_inventory'); expect(steps[1].args).toEqual({ order_id: '123', include_history: false }); expect(steps[2].args).toEqual({ query: 'quantum computing', limit: 3 }); expect(steps[3].args).toEqual({ tone: 'friendly', citations: ['doc_1', 'doc_2'], }); }); it('does not classify a chat span carrying gen_ai.tool.definitions as a tool', () => { // pydantic-ai (and other GenAI-convention tracers) put the list of *available* tools on // each chat span as a JSON array under gen_ai.tool.definitions; that must not be read as // a tool call. Real tool calls use gen_ai.tool.name. Regression for #9523. const toolDefinitions = JSON.stringify([ { type: 'function', name: 'orders', description: 'List orders' }, { type: 'function', name: 'order_rows', description: 'List order rows' }, ]); const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'chat-1', name: 'chat gpt-4o-mini', startTime: 1000, endTime: 1100, attributes: { 'gen_ai.tool.definitions': toolDefinitions }, }, { spanId: 'exec-1', name: 'execute_tool', startTime: 1200, endTime: 1300, attributes: { 'gen_ai.tool.name': 'order_rows' }, }, ], }); expect(steps.map((step) => ({ type: step.type, name: step.name }))).toEqual([ { type: 'span', name: 'chat gpt-4o-mini' }, { type: 'tool', name: 'order_rows' }, ]); }); it('still reads a plain string tool name from a generic tool attribute', () => { // The structured-value guard must not break scalar tool-name fallbacks. const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'custom-tool', name: 'invoke', startTime: 1000, endTime: 1100, attributes: { 'app.tool': 'lookup_customer' }, }, ], }); expect(steps.map((step) => ({ type: step.type, name: step.name }))).toEqual([ { type: 'tool', name: 'lookup_customer' }, ]); }); it('only treats a generic query attribute as search when the span looks search-like', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'span-sql', name: 'sql.query', startTime: 1000, endTime: 1100, attributes: { query: 'SELECT * FROM users', }, }, { spanId: 'span-search', name: 'document_search', startTime: 1200, endTime: 1300, attributes: { query: 'refund policy', }, }, ], }); expect(steps.map((step) => ({ type: step.type, name: step.name }))).toEqual([ { type: 'span', name: 'sql.query' }, { type: 'search', name: 'refund policy' }, ]); }); it('normalizes OpenAI Agents sandbox exec_command tool spans as commands', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'sandbox-command', name: 'tool exec_command', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'exec_command', 'tool.arguments': '{"cmd":"cat repo/tickets/TICKET-014.md","workdir":"/tmp/ws"}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('command'); expect(steps[0].name).toBe('cat repo/tickets/TICKET-014.md'); expect(steps[0].args).toEqual({ cmd: 'cat repo/tickets/TICKET-014.md', workdir: '/tmp/ws', }); expect(steps[0].aliases).toEqual( expect.arrayContaining(['tool exec_command', 'exec_command', 'cat']), ); }); it('normalizes local_shell tool spans as commands using the command arg', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'local-shell-span', name: 'tool local_shell', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'local_shell', 'tool.arguments': '{"command":"ls -la /tmp","timeout":30}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('command'); expect(steps[0].name).toBe('ls -la /tmp'); expect(steps[0].aliases).toEqual( expect.arrayContaining(['tool local_shell', 'local_shell', 'ls']), ); }); it('normalizes shell tool spans as commands', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'shell-span', name: 'tool shell', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'shell', 'tool.arguments': '{"cmd":"echo hello"}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('command'); expect(steps[0].name).toBe('echo hello'); expect(steps[0].aliases).toEqual(expect.arrayContaining(['shell', 'echo'])); }); it('normalizes shell tool command arrays as one command step', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'shell-command-array', name: 'tool shell', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'shell', 'tool.arguments': '{"commands":["cat skills/discount-review/SKILL.md","python3 skills/discount-review/scripts/analyze_discount_policy.py skill_fixture/repo"]}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('command'); expect(steps[0].name).toBe( 'cat skills/discount-review/SKILL.md; python3 skills/discount-review/scripts/analyze_discount_policy.py skill_fixture/repo', ); expect(steps[0].aliases).toEqual(expect.arrayContaining(['shell', 'cat'])); }); it('normalizes array-format command arguments from exec_command', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'array-cmd', name: 'tool exec_command', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'exec_command', 'tool.arguments': '{"cmd":["python","-m","pytest"],"workdir":"/ws"}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('command'); expect(steps[0].name).toBe('python -m pytest'); expect(steps[0].aliases).toEqual( expect.arrayContaining(['exec_command', 'python -m pytest', 'python']), ); }); it('falls back to tool type when exec_command has no cmd or command arg', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'no-cmd', name: 'tool exec_command', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'exec_command', 'tool.arguments': '{"workdir":"/tmp"}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('tool'); expect(steps[0].name).toBe('exec_command'); }); it('matches command tool names case-insensitively with whitespace trimming', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'cased-shell', name: 'tool Shell', startTime: 1000, endTime: 1100, attributes: { 'tool.name': ' Shell ', 'tool.arguments': '{"cmd":"pwd"}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('command'); expect(steps[0].name).toBe('pwd'); }); it.each([ ['ai.toolCall.args', '{"order_id":"123","include_history":false}'], ['ai.toolCall.arguments', '{"order_id":"123","include_history":false}'], ['ai.toolCall.input', '{"order_id":"123","include_history":false}'], ])('normalizes Vercel AI SDK tool spans with %s', (argKey, argValue) => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'vercel-tool-call', name: 'ai.toolCall', startTime: 1000, endTime: 1100, attributes: { 'ai.toolCall.name': 'search_orders', [argKey]: argValue, 'ai.toolCall.result': '{"status":"ok"}', }, }, ], }); expect(steps).toHaveLength(1); expect(steps[0].type).toBe('tool'); expect(steps[0].name).toBe('search_orders'); expect(steps[0].aliases).toEqual(expect.arrayContaining(['ai.toolCall', 'search_orders'])); expect(steps[0].args).toEqual({ order_id: '123', include_history: false, }); }); it('preserves original span order when timestamps are tied', () => { const steps = extractTrajectorySteps({ ...mockTraceData, spans: [ { spanId: 'tied-1', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'compose_reply', }, }, { spanId: 'tied-2', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'search_orders', }, }, { spanId: 'tied-3', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'finalize', }, }, ], }); expect(steps.map((step) => step.name)).toEqual(['compose_reply', 'search_orders', 'finalize']); }); it('compacts repeated steps and truncates long judge summaries', () => { const trace = { ...mockTraceData, spans: [ ...Array.from({ length: 4 }, (_, index) => ({ spanId: `search-${index}`, name: 'document_search', startTime: 1000 + index, endTime: 1100 + index, attributes: { query: 'refund policy', }, })), ...Array.from({ length: 25 }, (_, index) => ({ spanId: `tool-${index}`, name: 'tool.call', startTime: 2000 + index, endTime: 2100 + index, attributes: { 'tool.name': `tool_${index + 1}`, }, })), ], } satisfies TraceData; const summary = JSON.parse(summarizeTrajectoryForJudge(trace)) as { compactedStepCount: number; stepCount: number; steps: Array< | { collapsedCount?: number; index?: number; name?: string; spanName?: string; type?: string; } | { omittedCount: number } >; }; expect(summary.stepCount).toBe(29); expect(summary.compactedStepCount).toBe(26); expect(summary.steps).toHaveLength(25); expect(summary.steps[0]).toMatchObject({ index: 1, type: 'search', name: 'refund policy', spanName: 'document_search', collapsedCount: 4, }); expect(summary.steps[12]).toEqual({ omittedCount: 2 }); expect(summary.steps.at(-1)).toMatchObject({ index: 29, type: 'tool', name: 'tool_25', }); }); }); describe('trajectory assertions', () => { describe('trajectory:tool-used', () => { it('throws when trace data is not available', () => { const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: undefined, }, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: 'search_orders', }, renderedValue: 'search_orders', }; expect(() => handleTrajectoryToolUsed(params)).toThrow( 'No trace data available for trajectory:tool-used assertion', ); }); it('passes when a required tool is present', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: 'search_orders', }, renderedValue: 'search_orders', }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Observed required tool(s): search_orders. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('supports count-based matching with patterns', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: { pattern: 'search*', min: 2, max: 2, }, }, renderedValue: { pattern: 'search*', min: 2, max: 2, }, }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Matched tool "search*" 2 time(s) (expected 2-2). Matches: tool:search_orders, tool:search_inventory', assertion: params.assertion, }); }); it('supports minimum-only count matching with patterns', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: { pattern: 'search*', min: 1, }, }, renderedValue: { pattern: 'search*', min: 1, }, }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Matched tool "search*" 2 time(s) (expected at least 1). Matches: tool:search_orders, tool:search_inventory', assertion: params.assertion, }); }); it('supports inverse assertions', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', inverse: true, assertion: { type: 'not-trajectory:tool-used', value: 'compose_reply', }, renderedValue: 'compose_reply', }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Forbidden tool(s) were used: compose_reply. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('fails inverse list assertions when any forbidden tool is present', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', inverse: true, assertion: { type: 'not-trajectory:tool-used', value: ['missing_tool', 'compose_reply'], }, renderedValue: ['missing_tool', 'compose_reply'], }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Forbidden tool(s) were used: compose_reply. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('passes inverse list assertions when forbidden tools are absent', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', inverse: true, assertion: { type: 'not-trajectory:tool-used', value: ['missing_tool'], }, renderedValue: ['missing_tool'], }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Forbidden tool(s) were not used: missing_tool', assertion: params.assertion, }); }); it('passes inverse count assertions when the forbidden count is not satisfied', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', inverse: true, assertion: { type: 'not-trajectory:tool-used', value: { name: 'missing_tool', min: 1, }, }, renderedValue: { name: 'missing_tool', min: 1, }, }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Tool "missing_tool" did not satisfy the forbidden match condition', assertion: params.assertion, }); }); it('fails when required tools are missing', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: ['search_orders', 'missing_tool'], }, renderedValue: ['search_orders', 'missing_tool'], }; const result = handleTrajectoryToolUsed(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Missing required tool(s): missing_tool. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('rejects invalid matcher values', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: 123, }, renderedValue: 123, }; expect(() => handleTrajectoryToolUsed(params)).toThrow( 'trajectory:tool-used assertion must have a string, string array, or object value', ); }); it('rejects count objects without a name or pattern', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-used', assertion: { type: 'trajectory:tool-used', value: { min: 1, }, }, renderedValue: { min: 1, }, }; expect(() => handleTrajectoryToolUsed(params)).toThrow( 'trajectory:tool-used assertion object must include a name or pattern property', ); }); }); describe('trajectory:tool-sequence', () => { it('passes when tools are observed in order', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: ['search_orders', 'compose_reply'], }, renderedValue: ['search_orders', 'compose_reply'], }; const result = handleTrajectoryToolSequence(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Observed tool sequence in order: tool:search_orders, tool:compose_reply. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('fails exact mode when there are additional tools', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: { mode: 'exact', steps: ['search_orders', 'compose_reply'], }, }, renderedValue: { mode: 'exact', steps: ['search_orders', 'compose_reply'], }, }; const result = handleTrajectoryToolSequence(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Expected exact tool sequence of search_orders, compose_reply, but actual tools were tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('passes exact mode when the sequence exactly matches', () => { const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: { ...mockTraceData, spans: [ { spanId: 'tool-1', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'search_orders' }, }, { spanId: 'tool-2', name: 'tool.call', startTime: 1200, endTime: 1300, attributes: { 'tool.name': 'compose_reply' }, }, ], }, }, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: { mode: 'exact', steps: ['search_orders', 'compose_reply'], }, }, renderedValue: { mode: 'exact', steps: ['search_orders', 'compose_reply'], }, }; const result = handleTrajectoryToolSequence(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Observed exact tool sequence: tool:search_orders, tool:compose_reply', assertion: params.assertion, }); }); it('fails when an in-order sequence is not observed', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: ['compose_reply', 'search_orders'], }, renderedValue: ['compose_reply', 'search_orders'], }; const result = handleTrajectoryToolSequence(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Expected tool "search_orders" was not observed in order. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('fails inverse sequence assertions when the forbidden sequence is present', () => { const params: AssertionParams = { ...defaultParams, inverse: true, baseType: 'trajectory:tool-sequence', assertion: { type: 'not-trajectory:tool-sequence', value: ['search_orders', 'compose_reply'], }, renderedValue: ['search_orders', 'compose_reply'], }; const result = handleTrajectoryToolSequence(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Forbidden tool sequence was observed. Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('passes inverse sequence assertions when the sequence is absent', () => { const params: AssertionParams = { ...defaultParams, inverse: true, baseType: 'trajectory:tool-sequence', assertion: { type: 'not-trajectory:tool-sequence', value: ['compose_reply', 'search_orders'], }, renderedValue: ['compose_reply', 'search_orders'], }; const result = handleTrajectoryToolSequence(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Forbidden tool sequence was not observed', assertion: params.assertion, }); }); it('rejects object steps without a name or pattern', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: [{ type: 'tool' }, 'compose_reply'], }, renderedValue: [{ type: 'tool' }, 'compose_reply'], }; expect(() => handleTrajectoryToolSequence(params)).toThrow( 'trajectory:tool-sequence assertion step 1 must include a name or pattern property', ); }); it('rejects invalid sequence values', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: 'search_orders', }, renderedValue: 'search_orders', }; expect(() => handleTrajectoryToolSequence(params)).toThrow( 'trajectory:tool-sequence assertion must have an array or object value', ); }); it('rejects empty sequence objects', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-sequence', assertion: { type: 'trajectory:tool-sequence', value: { steps: [], }, }, renderedValue: { steps: [], }, }; expect(() => handleTrajectoryToolSequence(params)).toThrow( 'trajectory:tool-sequence assertion requires at least one expected step', ); }); }); describe('trajectory:tool-args-match', () => { it('passes when a tool call contains the expected argument subset', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '123', }, }, }, renderedValue: { name: 'search_orders', args: { order_id: '123', }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Tool "search_orders" matched expected arguments (partial) on tool:search_orders. Args: {"order_id":"123","include_history":false}', assertion: params.assertion, }); }); it('supports partial array matching for argument subsets', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'compose_reply', args: { citations: ['doc_1', 'doc_2'], }, }, }, renderedValue: { name: 'compose_reply', args: { citations: ['doc_1', 'doc_2'], }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Tool "compose_reply" matched expected arguments (partial) on tool:compose_reply. Args: {"tone":"friendly","citations":["doc_1","doc_2"]}', assertion: params.assertion, }); }); it('supports exact mode for argument matching', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { pattern: 'compose_*', mode: 'exact', arguments: { tone: 'friendly', citations: ['doc_1', 'doc_2'], }, }, }, renderedValue: { pattern: 'compose_*', mode: 'exact', arguments: { tone: 'friendly', citations: ['doc_1', 'doc_2'], }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Tool "compose_*" matched expected arguments (exact) on tool:compose_reply. Args: {"tone":"friendly","citations":["doc_1","doc_2"]}', assertion: params.assertion, }); }); it('matches Vercel AI SDK spans end-to-end against expected args', () => { const vercelTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'vercel-span', name: 'ai.toolCall', startTime: 1000, endTime: 1100, attributes: { 'ai.toolCall.name': 'lookup_customer', 'ai.toolCall.args': '{"customer_id":"cust_1234","include_history":true}', 'ai.toolCall.result': '{"plan":"pro"}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: vercelTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'lookup_customer', args: { customer_id: 'cust_1234' }, }, }, renderedValue: { name: 'lookup_customer', args: { customer_id: 'cust_1234' }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); expect(result.score).toBe(1); }); it('fails when no matching tool arguments satisfy the expectation', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '999', }, }, }, renderedValue: { name: 'search_orders', args: { order_id: '999', }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: false, score: 0, reason: 'No call to tool "search_orders" matched expected arguments (partial): {"order_id":"999"}. Observed args: {"order_id":"123","include_history":false}', assertion: params.assertion, }); }); it('fails when no tool call matches the requested tool', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'missing_tool', args: { order_id: '123', }, }, }, renderedValue: { name: 'missing_tool', args: { order_id: '123', }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: false, score: 0, reason: 'No tool call matched "missing_tool". Actual tools: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('fails when the tool is present but no arguments were captured', () => { const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: { ...mockTraceData, spans: [ { spanId: 'tool-without-args', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'search_orders', }, }, ], }, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '123', }, }, }, renderedValue: { name: 'search_orders', args: { order_id: '123', }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Tool "search_orders" was observed but no arguments were captured. Actual tools: tool:search_orders', assertion: params.assertion, }); }); it('supports inverse assertions', () => { const params: AssertionParams = { ...defaultParams, inverse: true, baseType: 'trajectory:tool-args-match', assertion: { type: 'not-trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '123', }, }, }, renderedValue: { name: 'search_orders', args: { order_id: '123', }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: false, score: 0, reason: 'Forbidden argument match for tool "search_orders" was observed on tool:search_orders. Args: {"order_id":"123","include_history":false}', assertion: params.assertion, }); }); it('passes inverse assertions when no tool call matches the requested tool', () => { const params: AssertionParams = { ...defaultParams, inverse: true, baseType: 'trajectory:tool-args-match', assertion: { type: 'not-trajectory:tool-args-match', value: { name: 'missing_tool', args: { order_id: '123', }, }, }, renderedValue: { name: 'missing_tool', args: { order_id: '123', }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Forbidden argument match for tool "missing_tool" was not observed because no tool call matched it', assertion: params.assertion, }); }); it('rejects values without args', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', }, }, renderedValue: { name: 'search_orders', }, }; expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion must include an args or arguments property', ); }); it('rejects invalid mode values', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', mode: 'excat', args: { order_id: '123', }, }, }, renderedValue: { name: 'search_orders', mode: 'excat', args: { order_id: '123', }, }, }; expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion mode must be "partial" or "exact"', ); }); it('rejects non-object argument match values', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: 'search_orders', }, renderedValue: 'search_orders', }; expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion must have an object value', ); }); it('rejects argument matchers without a name or pattern', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { args: { order_id: '123', }, }, }, renderedValue: { args: { order_id: '123', }, }, }; expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion object must include a name or pattern property', ); }); describe('defaults', () => { it('passes in exact mode when an unexpected actual key matches a declared default', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: { include_history: false }, }, }, renderedValue: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: { include_history: false }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Tool "search_orders" matched expected arguments (exact) on tool:search_orders. Args: {"order_id":"123","include_history":false}. Ignored default argument(s): include_history', assertion: params.assertion, }); }); it('fails in exact mode when an actual key value differs from the declared default', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: { include_history: true }, }, }, renderedValue: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: { include_history: true }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); expect(result.score).toBe(0); expect(result.reason).toContain( 'No call to tool "search_orders" matched expected arguments', ); }); it('passes in exact mode when actual contains multiple keys all matching declared defaults', () => { const allDefaultsTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-all-defaults', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":1,"page_size":5}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: allDefaultsTrace, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('passes in exact mode when actual omits a key listed in defaults', () => { const traceWithoutDefault: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-no-default', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q"}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: traceWithoutDefault, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('fails in exact mode when actual contains a hallucinated key not in args or defaults', () => { const hallucinatedTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-hallucinated', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":1,"delete_database":true}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: hallucinatedTrace, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); expect(result.score).toBe(0); expect(result.reason).toContain('delete_database'); }); it('treats empty defaults the same as no defaults', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: {}, }, }, renderedValue: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: {}, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); }); it('supports defaults with structured values via deep equality', () => { const structuredTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-structured', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","sort":{"field":"created","order":"desc"}}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: structuredTrace, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { sort: { field: 'created', order: 'desc' } }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { sort: { field: 'created', order: 'desc' } }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('still passes in partial mode when defaults are set', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '123' }, defaults: { include_history: false }, }, }, renderedValue: { name: 'search_orders', args: { order_id: '123' }, defaults: { include_history: false }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('rejects malformed defaults (array)', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '123' }, defaults: ['include_history'] as unknown as Record, }, }, renderedValue: { name: 'search_orders', args: { order_id: '123' }, defaults: ['include_history'] as unknown as Record, }, }; expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion defaults must be an object mapping argument names to default values', ); }); it('matches the documented outcome table row "status:Q, page:1" using the example config', () => { const docsTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-docs-row-2', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":1}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: docsTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('matches the documented outcome table row "status:Q, page:2" using the example config', () => { const docsTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-docs-row-4', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":2}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: docsTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); }); it('strips a null default value when actual emits null for that key', () => { const nullTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-null-default', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","cursor":null}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: nullTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { cursor: null }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { cursor: null }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('strips a key listed in defaults even when args also declares it (defaults applied first)', () => { const collisionTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-collision', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":1}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: collisionTrace, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q', page: 2 }, defaults: { page: 1 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q', page: 2 }, defaults: { page: 1 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); }); it('does not recurse into nested objects when stripping defaults (top-level only)', () => { const nestedTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-nested-default', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","pagination":{"page":1}}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: nestedTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); }); it('inverts correctly when defaults allow an otherwise-forbidden match', () => { const params: AssertionParams = { ...defaultParams, inverse: true, baseType: 'trajectory:tool-args-match', assertion: { type: 'not-trajectory:tool-args-match', value: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: { include_history: false }, }, }, renderedValue: { name: 'search_orders', mode: 'exact', args: { order_id: '123' }, defaults: { include_history: false }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); expect(result.reason).toContain('Forbidden argument match'); }); it('surfaces the stripped keys in the pass reason when multiple defaults are tolerated', () => { const allDefaultsTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-reason-defaults', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":1,"page_size":5}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: allDefaultsTrace, }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1, page_size: 5 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); expect(result.reason).toContain('Ignored default argument(s): page, page_size'); }); it('does not append the ignored-defaults note when nothing was stripped', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', mode: 'partial', args: { order_id: '123' }, defaults: { include_history: true }, }, }, renderedValue: { name: 'search_orders', mode: 'partial', args: { order_id: '123' }, defaults: { include_history: true }, }, }; // The observed call has include_history:false, which does not equal the // declared default (true), so nothing is stripped and no note is added. const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); expect(result.reason).not.toContain('Ignored default'); }); it('does not let a hallucinated __proto__ argument escape exact matching when defaults are set', () => { const protoTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-proto-default', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","__proto__":{"polluted":true}}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: protoTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, }, }; // A non-default __proto__ key must be preserved through stripping so exact // mode still rejects it as a hallucinated extra (matching no-defaults behavior). const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); // The prototype must not have been mutated by the stripping rebuild. expect(({} as Record).polluted).toBeUndefined(); }); it('fails in partial mode when a key in both args and defaults is stripped to nothing', () => { const collisionTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-partial-collision', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '{"status":"Q","page":1}', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: collisionTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', // partial (default) mode; page is listed in BOTH args and defaults args: { status: 'Q', page: 1 }, defaults: { page: 1 }, }, }, renderedValue: { name: 'orders', args: { status: 'Q', page: 1 }, defaults: { page: 1 }, }, }; // Documented footgun: stripping removes page before the subset check, so the // explicitly-asserted page:1 can no longer be satisfied. const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(false); }); it('applies defaults when the expected payload uses the arguments alias', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', mode: 'exact', arguments: { order_id: '123' }, defaults: { include_history: false }, }, }, renderedValue: { name: 'search_orders', mode: 'exact', arguments: { order_id: '123' }, defaults: { include_history: false }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it('treats defaults as a no-op when the actual payload is a top-level array', () => { const arrayTrace: TraceData = { ...mockTraceData, spans: [ { spanId: 'span-array-args', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'orders', 'tool.arguments': '[1,2,3]', }, }, ], }; const params: AssertionParams = { ...defaultParams, assertionValueContext: { ...defaultParams.assertionValueContext, trace: arrayTrace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'orders', mode: 'exact', args: [1, 2, 3], defaults: { page: 1 }, }, }, renderedValue: { name: 'orders', mode: 'exact', args: [1, 2, 3], defaults: { page: 1 }, }, }; const result = handleTrajectoryToolArgsMatch(params); expect(result.pass).toBe(true); }); it.each([ ['null', null], ['string', 'include_history'], ['number', 1], ])('rejects malformed defaults (%s)', (_label, malformed) => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value: { name: 'search_orders', args: { order_id: '123' }, defaults: malformed as unknown as Record, }, }, renderedValue: { name: 'search_orders', args: { order_id: '123' }, defaults: malformed as unknown as Record, }, }; expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion defaults must be an object mapping argument names to default values', ); }); }); describe('ignore', () => { const makeIgnoreParams = ( toolArgs: Record | unknown[] | string, value: Record, inverse = false, ): AssertionParams => { const argsJson = typeof toolArgs === 'string' ? toolArgs : JSON.stringify(toolArgs); const trace: TraceData = { ...mockTraceData, spans: [ { spanId: 'ignore-tool-call', name: 'tool.call', startTime: 1000, endTime: 1100, attributes: { 'tool.name': 'search_orders', 'tool.arguments': argsJson }, }, ], }; return { ...defaultParams, inverse, assertionValueContext: { ...defaultParams.assertionValueContext, trace }, baseType: 'trajectory:tool-args-match', assertion: { type: 'trajectory:tool-args-match', value }, renderedValue: value, } as AssertionParams; }; it('drops an ignored key regardless of value in exact mode', () => { const value = { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['request_id'], }; expect( handleTrajectoryToolArgsMatch(makeIgnoreParams({ status: 'Q', request_id: 'a' }, value)) .pass, ).toBe(true); expect( handleTrajectoryToolArgsMatch(makeIgnoreParams({ status: 'Q', request_id: 'z' }, value)) .pass, ).toBe(true); }); it('accepts a bare string ignore value', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'abc' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: 'request_id' }, ), ); expect(result.pass).toBe(true); }); it('still fails on a non-ignored hallucinated argument', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'abc', delete_database: true }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['request_id'] }, ), ); expect(result.pass).toBe(false); }); it('strips the ignored key from the expected payload too', () => { // request_id is declared in args but ignored, so the agent omitting it still passes. const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q', request_id: 'expected' }, ignore: ['request_id'], }, ), ); expect(result.pass).toBe(true); }); it('composes with defaults and reports both in the pass reason', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', page: 1, request_id: 'abc' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, defaults: { page: 1 }, ignore: ['request_id'], }, ), ); expect(result.pass).toBe(true); expect(result.reason).toContain('Ignored argument(s): request_id'); expect(result.reason).toContain('Ignored default argument(s): page'); }); it('works in partial mode', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'abc' }, { name: 'search_orders', mode: 'partial', args: { status: 'Q' }, ignore: ['request_id'], }, ), ); expect(result.pass).toBe(true); }); it('inverts correctly', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'abc' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['request_id'] }, true, ), ); expect(result.pass).toBe(false); }); it('does not let a hallucinated __proto__ argument escape exact matching', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams('{"status":"Q","__proto__":{"polluted":true},"request_id":"abc"}', { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['request_id'], }), ); expect(result.pass).toBe(false); }); it('treats ignore as a no-op when the actual payload is a top-level array', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams('["a","b"]', { name: 'search_orders', mode: 'exact', args: ['a', 'b'], ignore: ['request_id'], }), ); expect(result.pass).toBe(true); }); it('treats entries containing glob characters as key patterns', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'a1', order_id: 'b2' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['*_id'] }, ), ); expect(result.pass).toBe(true); expect(result.reason).toContain('Ignored argument(s): request_id, order_id'); }); it('still fails when a glob pattern does not cover a hallucinated key', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'a1', delete_database: true }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['*_id'] }, ), ); expect(result.pass).toBe(false); }); it('supports single-character ? glob patterns', () => { const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', v1: 'x' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['v?'] }, ), ); expect(result.pass).toBe(true); }); it('treats a plain entry as an exact match, not a substring or pattern', () => { // "request" must not strip "request_id"; only an exact key or glob does. const result = handleTrajectoryToolArgsMatch( makeIgnoreParams( { status: 'Q', request_id: 'a1' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: ['request'] }, ), ); expect(result.pass).toBe(false); }); it.each([ ['number', [5]], ['empty string', ['']], ['null', [null]], ['object', [{ request_id: true }]], ])('rejects malformed ignore entries (%s)', (_label, malformed) => { const params = makeIgnoreParams( { status: 'Q' }, { name: 'search_orders', mode: 'exact', args: { status: 'Q' }, ignore: malformed as unknown as string[], }, ); expect(() => handleTrajectoryToolArgsMatch(params)).toThrow( 'trajectory:tool-args-match assertion ignore must be a non-empty string or an array of non-empty strings', ); }); }); }); describe('trajectory:step-count', () => { it('counts steps by type', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:step-count', assertion: { type: 'trajectory:step-count', value: { type: 'command', min: 1, max: 1, }, }, renderedValue: { type: 'command', min: 1, max: 1, }, }; const result = handleTrajectoryStepCount(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Matched 1 trajectory step(s) for type=command (expected 1-1). Matches: command:ls -la', assertion: params.assertion, }); }); it('counts steps by pattern', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:step-count', assertion: { type: 'trajectory:step-count', value: { pattern: 'reasoning*', min: 1, }, }, renderedValue: { pattern: 'reasoning*', min: 1, }, }; const result = handleTrajectoryStepCount(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Matched 1 trajectory step(s) for pattern=reasoning* (expected at least 1). Matches: reasoning:reasoning_plan', assertion: params.assertion, }); }); it('supports max-only count assertions', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:step-count', assertion: { type: 'trajectory:step-count', value: { type: 'tool', max: 3, }, }, renderedValue: { type: 'tool', max: 3, }, }; const result = handleTrajectoryStepCount(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Matched 3 trajectory step(s) for type=tool (expected at most 3). Matches: tool:search_orders, tool:search_inventory, tool:compose_reply', assertion: params.assertion, }); }); it('supports inverse count assertions', () => { const params: AssertionParams = { ...defaultParams, inverse: true, baseType: 'trajectory:step-count', assertion: { type: 'not-trajectory:step-count', value: { type: 'tool', min: 4, }, }, renderedValue: { type: 'tool', min: 4, }, }; const result = handleTrajectoryStepCount(params); expect(result).toEqual({ pass: true, score: 1, reason: 'Trajectory step count did not satisfy the forbidden range', assertion: params.assertion, }); }); it('rejects count assertions without min or max', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:step-count', assertion: { type: 'trajectory:step-count', value: { type: 'tool', }, }, renderedValue: { type: 'tool', }, }; expect(() => handleTrajectoryStepCount(params)).toThrow( 'trajectory:step-count assertion must include a min or max property', ); }); it('rejects non-object count assertions', () => { const params: AssertionParams = { ...defaultParams, baseType: 'trajectory:step-count', assertion: { type: 'trajectory:step-count', value: 'tool', }, renderedValue: 'tool', }; expect(() => handleTrajectoryStepCount(params)).toThrow( 'trajectory:step-count assertion must have an object value', ); }); }); });