wehub-resource-sync
0d3cb498a3
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
3713 lines
123 KiB
TypeScript
3713 lines
123 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
|
|
import logger from '../../../src/logger';
|
|
import * as genaiTracer from '../../../src/tracing/genaiTracer';
|
|
import { mockProcessEnv } from '../../util/utils';
|
|
import type { ContentBlock, StopReason } from '@aws-sdk/client-bedrock-runtime';
|
|
|
|
// Define mock module type
|
|
interface MockBedrockModule {
|
|
BedrockRuntime: Mock;
|
|
ConverseCommand: Mock;
|
|
ConverseStreamCommand: Mock;
|
|
}
|
|
|
|
// Define a typed version of the ConverseCommandOutput for tests
|
|
interface MockConverseCommandOutput {
|
|
$metadata: Record<string, unknown>;
|
|
output?: {
|
|
message?: {
|
|
role: 'assistant';
|
|
content?: ContentBlock[];
|
|
};
|
|
};
|
|
usage?: {
|
|
inputTokens?: number;
|
|
outputTokens?: number;
|
|
totalTokens?: number;
|
|
cacheReadInputTokens?: number;
|
|
cacheWriteInputTokens?: number;
|
|
};
|
|
stopReason?: StopReason;
|
|
metrics?: {
|
|
latencyMs?: number;
|
|
};
|
|
}
|
|
|
|
const mockSend = vi.hoisted(() => vi.fn<(command: unknown) => Promise<unknown>>());
|
|
const mcpMocks = vi.hoisted(() => {
|
|
const mockConstructor = vi.fn();
|
|
const mockInitialize = vi.fn().mockResolvedValue(undefined);
|
|
const mockCleanup = vi.fn().mockResolvedValue(undefined);
|
|
const mockGetAllTools = vi.fn().mockReturnValue([
|
|
{
|
|
name: 'list_resources',
|
|
description: 'List available resources',
|
|
inputSchema: { type: 'object', properties: { resourceType: { type: 'string' } } },
|
|
},
|
|
]);
|
|
const mockCallTool = vi.fn().mockResolvedValue({
|
|
content: 'Available resources: [docs, tickets]',
|
|
});
|
|
|
|
class MockMCPClient {
|
|
constructor(config: unknown) {
|
|
mockConstructor(config);
|
|
}
|
|
|
|
initialize = mockInitialize;
|
|
cleanup = mockCleanup;
|
|
getAllTools = mockGetAllTools;
|
|
callTool = mockCallTool;
|
|
}
|
|
|
|
return {
|
|
MockMCPClient,
|
|
mockConstructor,
|
|
mockInitialize,
|
|
mockCleanup,
|
|
mockGetAllTools,
|
|
mockCallTool,
|
|
};
|
|
});
|
|
let restoreBedrockEnvMock: (() => void) | undefined;
|
|
|
|
function clearBedrockEnv() {
|
|
restoreBedrockEnv();
|
|
restoreBedrockEnvMock = mockProcessEnv({
|
|
AWS_BEDROCK_MAX_TOKENS: undefined,
|
|
AWS_BEDROCK_TEMPERATURE: undefined,
|
|
AWS_BEDROCK_TOP_P: undefined,
|
|
AWS_BEDROCK_STOP: undefined,
|
|
HTTP_PROXY: undefined,
|
|
HTTPS_PROXY: undefined,
|
|
});
|
|
}
|
|
|
|
function restoreBedrockEnv() {
|
|
restoreBedrockEnvMock?.();
|
|
restoreBedrockEnvMock = undefined;
|
|
}
|
|
|
|
vi.mock('@aws-sdk/client-bedrock-runtime', async (importOriginal) => {
|
|
return {
|
|
...(await importOriginal()),
|
|
|
|
BedrockRuntime: vi.fn().mockImplementation(function () {
|
|
return {
|
|
send: mockSend,
|
|
|
|
config: {
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
credentials: vi.fn<any>().mockResolvedValue({
|
|
accessKeyId: 'test-key',
|
|
}),
|
|
},
|
|
};
|
|
}),
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
ConverseCommand: vi.fn().mockImplementation(function (input: any) {
|
|
return {
|
|
input,
|
|
};
|
|
}),
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
ConverseStreamCommand: vi.fn().mockImplementation(function (input: any) {
|
|
return {
|
|
input,
|
|
};
|
|
}),
|
|
};
|
|
});
|
|
|
|
vi.mock('../../../src/providers/mcp/client', async (importOriginal) => {
|
|
return {
|
|
...(await importOriginal()),
|
|
MCPClient: mcpMocks.MockMCPClient,
|
|
};
|
|
});
|
|
|
|
vi.mock('@smithy/node-http-handler', async (importOriginal) => {
|
|
return {
|
|
...(await importOriginal()),
|
|
|
|
NodeHttpHandler: vi.fn().mockImplementation(function () {
|
|
return {
|
|
handle: vi.fn(),
|
|
};
|
|
}),
|
|
};
|
|
});
|
|
|
|
// Provide the proxy-agent module shape without creating real agents in tests.
|
|
vi.mock('proxy-agent', () => {
|
|
const ProxyAgentMock = vi.fn();
|
|
return {
|
|
default: ProxyAgentMock,
|
|
ProxyAgent: ProxyAgentMock,
|
|
};
|
|
});
|
|
|
|
vi.mock('../../../src/cache', async (importOriginal) => {
|
|
return {
|
|
...(await importOriginal()),
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
getCache: vi.fn<any>().mockResolvedValue({
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
get: vi.fn<any>().mockResolvedValue(null),
|
|
set: vi.fn(),
|
|
}),
|
|
|
|
isCacheEnabled: vi.fn().mockReturnValue(false),
|
|
};
|
|
});
|
|
|
|
import {
|
|
AwsBedrockConverseProvider,
|
|
type BedrockConverseOptions,
|
|
parseConverseMessages,
|
|
} from '../../../src/providers/bedrock/converse';
|
|
|
|
// Helper to create mock response
|
|
function createMockConverseResponse(
|
|
textContent: string,
|
|
options: {
|
|
reasoningContent?: string;
|
|
toolUse?: { id: string; name: string; input: Record<string, unknown> };
|
|
usage?: {
|
|
inputTokens: number;
|
|
outputTokens: number;
|
|
totalTokens: number;
|
|
cacheReadInputTokens?: number;
|
|
cacheWriteInputTokens?: number;
|
|
};
|
|
stopReason?: StopReason;
|
|
latencyMs?: number;
|
|
} = {},
|
|
): MockConverseCommandOutput {
|
|
const contentBlocks: ContentBlock[] = [];
|
|
|
|
if (options.reasoningContent) {
|
|
contentBlocks.push({
|
|
reasoningContent: {
|
|
reasoningText: {
|
|
text: options.reasoningContent,
|
|
signature: 'test-signature',
|
|
},
|
|
},
|
|
});
|
|
}
|
|
|
|
if (textContent) {
|
|
contentBlocks.push({ text: textContent });
|
|
}
|
|
|
|
if (options.toolUse) {
|
|
contentBlocks.push({
|
|
toolUse: {
|
|
toolUseId: options.toolUse.id,
|
|
name: options.toolUse.name,
|
|
// Cast to any for test mocking - DocumentType is complex union
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
input: options.toolUse.input as any,
|
|
},
|
|
});
|
|
}
|
|
|
|
return {
|
|
$metadata: {},
|
|
output: {
|
|
message: {
|
|
role: 'assistant',
|
|
content: contentBlocks,
|
|
},
|
|
},
|
|
usage: options.usage || {
|
|
inputTokens: 100,
|
|
outputTokens: 50,
|
|
totalTokens: 150,
|
|
},
|
|
stopReason: options.stopReason || ('end_turn' as StopReason),
|
|
metrics: {
|
|
latencyMs: options.latencyMs || 1000,
|
|
},
|
|
};
|
|
}
|
|
|
|
describe('AwsBedrockConverseProvider', () => {
|
|
beforeEach(() => {
|
|
// Only reset mockSend, not all mocks (which would break the mock implementations)
|
|
mockSend.mockReset();
|
|
clearBedrockEnv();
|
|
mcpMocks.mockConstructor.mockReset();
|
|
mcpMocks.mockInitialize.mockReset().mockResolvedValue(undefined);
|
|
mcpMocks.mockCleanup.mockReset().mockResolvedValue(undefined);
|
|
mcpMocks.mockGetAllTools.mockReset().mockReturnValue([
|
|
{
|
|
name: 'list_resources',
|
|
description: 'List available resources',
|
|
inputSchema: { type: 'object', properties: { resourceType: { type: 'string' } } },
|
|
},
|
|
]);
|
|
mcpMocks.mockCallTool.mockReset().mockResolvedValue({
|
|
content: 'Available resources: [docs, tickets]',
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
// Ensure mock is reset after each test to prevent pollution
|
|
mockSend.mockReset();
|
|
restoreBedrockEnv();
|
|
});
|
|
|
|
describe('constructor', () => {
|
|
it('should create provider with model name', () => {
|
|
const provider = new AwsBedrockConverseProvider(
|
|
'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
|
|
);
|
|
expect(provider.modelName).toBe('us.anthropic.claude-sonnet-4-5-20250929-v1:0');
|
|
});
|
|
|
|
it('should set correct id format', () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0');
|
|
expect(provider.id()).toBe('bedrock:converse:anthropic.claude-3-5-sonnet-20241022-v2:0');
|
|
});
|
|
|
|
it('should accept config options', () => {
|
|
const config: BedrockConverseOptions = {
|
|
region: 'us-west-2',
|
|
maxTokens: 1024,
|
|
temperature: 0.7,
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 16000,
|
|
},
|
|
};
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config,
|
|
});
|
|
expect(provider.config).toMatchObject(config);
|
|
});
|
|
});
|
|
|
|
describe('base class methods', () => {
|
|
it('should return correct region from config', () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'eu-west-1' },
|
|
});
|
|
expect(provider.getRegion()).toBe('eu-west-1');
|
|
});
|
|
|
|
it('should return default region when not specified', () => {
|
|
const originalEnv = process.env.AWS_BEDROCK_REGION;
|
|
mockProcessEnv({ AWS_BEDROCK_REGION: undefined });
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {},
|
|
});
|
|
expect(provider.getRegion()).toBe('us-east-1');
|
|
|
|
if (originalEnv) {
|
|
mockProcessEnv({ AWS_BEDROCK_REGION: originalEnv });
|
|
}
|
|
});
|
|
|
|
it('should return region from environment variable', () => {
|
|
const originalEnv = process.env.AWS_BEDROCK_REGION;
|
|
mockProcessEnv({ AWS_BEDROCK_REGION: 'ap-northeast-1' });
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {},
|
|
});
|
|
expect(provider.getRegion()).toBe('ap-northeast-1');
|
|
|
|
if (originalEnv) {
|
|
mockProcessEnv({ AWS_BEDROCK_REGION: originalEnv });
|
|
} else {
|
|
mockProcessEnv({ AWS_BEDROCK_REGION: undefined });
|
|
}
|
|
});
|
|
|
|
it('should get credentials from config when accessKeyId and secretAccessKey are provided', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
accessKeyId: 'AKIAIOSFODNN7EXAMPLE',
|
|
secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
|
|
sessionToken: 'session-token-123',
|
|
},
|
|
});
|
|
|
|
const credentials = await provider.getCredentials();
|
|
expect(credentials).toEqual({
|
|
accessKeyId: 'AKIAIOSFODNN7EXAMPLE',
|
|
secretAccessKey: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
|
|
sessionToken: 'session-token-123',
|
|
});
|
|
});
|
|
|
|
it('should return undefined credentials when API key is used', async () => {
|
|
const originalEnv = process.env.AWS_BEARER_TOKEN_BEDROCK;
|
|
mockProcessEnv({ AWS_BEARER_TOKEN_BEDROCK: 'bearer-token-123' });
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
const credentials = await provider.getCredentials();
|
|
expect(credentials).toBeUndefined();
|
|
|
|
if (originalEnv) {
|
|
mockProcessEnv({ AWS_BEARER_TOKEN_BEDROCK: originalEnv });
|
|
} else {
|
|
mockProcessEnv({ AWS_BEARER_TOKEN_BEDROCK: undefined });
|
|
}
|
|
});
|
|
|
|
it('should return undefined credentials when using default credential chain', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
const credentials = await provider.getCredentials();
|
|
expect(credentials).toBeUndefined();
|
|
});
|
|
});
|
|
|
|
describe('callApi', () => {
|
|
it('should make Converse API call with basic text prompt', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Hello, world!'));
|
|
|
|
const result = await provider.callApi('Say hello');
|
|
|
|
expect(result.output).toBe('Hello, world!');
|
|
expect(result.tokenUsage).toMatchObject({
|
|
prompt: 100,
|
|
completion: 50,
|
|
total: 150,
|
|
});
|
|
});
|
|
|
|
it('should include extended thinking in output when showThinking is true', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: { type: 'enabled', budget_tokens: 16000 },
|
|
showThinking: true,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('The answer is 42.', {
|
|
reasoningContent: 'Let me think about this...',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('What is the meaning of life?');
|
|
|
|
expect(result.output).toContain('<thinking>');
|
|
expect(result.output).toContain('Let me think about this...');
|
|
expect(result.output).toContain('</thinking>');
|
|
expect(result.output).toContain('The answer is 42.');
|
|
});
|
|
|
|
it('should exclude empty thinking blocks returned by omitted-display adaptive thinking', async () => {
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-fable-5', {
|
|
config: { region: 'us-east-1', showThinking: true },
|
|
});
|
|
|
|
const response = createMockConverseResponse('QA OK');
|
|
response.output!.message!.content!.unshift({
|
|
reasoningContent: {
|
|
reasoningText: { text: '', signature: 'sig-abc123' },
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(response);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
expect(result.output).toBe('QA OK');
|
|
expect(result.output).not.toContain('<thinking>');
|
|
expect(result.output).not.toContain('Signature:');
|
|
});
|
|
|
|
it('should exclude thinking content when showThinking is false', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: { type: 'enabled', budget_tokens: 16000 },
|
|
showThinking: false,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('The answer is 42.', {
|
|
reasoningContent: 'Let me think about this...',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('What is the meaning of life?');
|
|
|
|
expect(result.output).not.toContain('<thinking>');
|
|
expect(result.output).not.toContain('Let me think about this...');
|
|
expect(result.output).toBe('The answer is 42.');
|
|
});
|
|
|
|
it('should handle tool use responses', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
name: 'calculator',
|
|
description: 'A calculator',
|
|
input_schema: { type: 'object', properties: { expression: { type: 'string' } } },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: { id: 'tool-123', name: 'calculator', input: { expression: '2+2' } },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('What is 2+2?');
|
|
|
|
// The output is JSON formatted tool use
|
|
const parsed = JSON.parse(result.output as string);
|
|
expect(parsed.type).toBe('tool_use');
|
|
expect(parsed.name).toBe('calculator');
|
|
expect(parsed.input).toEqual({ expression: '2+2' });
|
|
});
|
|
|
|
it('should execute functionToolCallbacks when defined', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
name: 'calculator',
|
|
description: 'A calculator',
|
|
input_schema: { type: 'object', properties: { expression: { type: 'string' } } },
|
|
},
|
|
],
|
|
functionToolCallbacks: {
|
|
calculator: (args: string) => {
|
|
const parsed = JSON.parse(args);
|
|
// Simple addition for test - avoids eval()
|
|
const [a, b] = parsed.expression.split('+').map(Number);
|
|
return `Result: ${a + b}`;
|
|
},
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: { id: 'tool-123', name: 'calculator', input: { expression: '2+2' } },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('What is 2+2?');
|
|
|
|
// The callback should be executed and return the result
|
|
expect(result.output).toBe('Result: 4');
|
|
});
|
|
|
|
it('should initialize and clean up MCP when configured', async () => {
|
|
const mcpConfig = {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
};
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: mcpConfig,
|
|
},
|
|
});
|
|
|
|
await (provider as any).initializationPromise;
|
|
await provider.cleanup();
|
|
|
|
expect(mcpMocks.mockConstructor).toHaveBeenCalledWith(mcpConfig);
|
|
expect(mcpMocks.mockInitialize).toHaveBeenCalled();
|
|
expect(mcpMocks.mockCleanup).toHaveBeenCalled();
|
|
});
|
|
|
|
it('should expose shutdown() that delegates to cleanup', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
// The evaluator calls provider.shutdown() via providerRegistry.shutdownAll().
|
|
// Without this method, MCP transports (including stdio child processes) leak.
|
|
expect(typeof (provider as any).shutdown).toBe('function');
|
|
await provider.shutdown();
|
|
expect(mcpMocks.mockCleanup).toHaveBeenCalled();
|
|
});
|
|
|
|
it('should be safe to call cleanup() multiple times', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
await provider.cleanup();
|
|
await provider.cleanup();
|
|
// The second cleanup must be a no-op — not a double mcpClient.cleanup() and not a throw.
|
|
expect(mcpMocks.mockCleanup).toHaveBeenCalledTimes(1);
|
|
});
|
|
|
|
it('should still attempt cleanup when MCP initialization failed', async () => {
|
|
mcpMocks.mockInitialize.mockRejectedValueOnce(new Error('handshake failed'));
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
// cleanup() must not re-throw the init failure; it must drain whatever
|
|
// partial state exists so we don't leak transports.
|
|
await expect(provider.cleanup()).resolves.toBeUndefined();
|
|
expect(mcpMocks.mockCleanup).toHaveBeenCalled();
|
|
});
|
|
|
|
it('should not initialize MCP when no server has a usable transport', () => {
|
|
// Empty command/path/url means MCPClient.initialize() would throw later;
|
|
// the provider should refuse to start the client at all.
|
|
new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
servers: [{ name: 'empty', command: '', args: [] }],
|
|
},
|
|
},
|
|
});
|
|
|
|
expect(mcpMocks.mockConstructor).not.toHaveBeenCalled();
|
|
expect(mcpMocks.mockInitialize).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('should surface MCP init failure as ProviderResponse.error on callApi', async () => {
|
|
mcpMocks.mockInitialize.mockRejectedValueOnce(new Error('handshake failed'));
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
const result = await provider.callApi('hi');
|
|
expect(result.error).toContain('MCP initialization failed');
|
|
// Bedrock must not be called when MCP init failed and the user wired MCP into the request.
|
|
expect(mockSend).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('should not surface unhandled rejection from a failed MCP init', async () => {
|
|
mcpMocks.mockInitialize.mockRejectedValueOnce(new Error('handshake failed'));
|
|
// Constructing the provider must not leak an UnhandledPromiseRejection;
|
|
// the constructor attaches a sink-handler that records the error so it
|
|
// can be reported lazily on the first callApi (which awaits the promise
|
|
// and observes the rejection through the recorded error field).
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
// callApi awaits initializationPromise internally, which forces the .catch
|
|
// handler in the constructor to run and populate mcpInitError. No manual
|
|
// microtask flush needed.
|
|
const result = await provider.callApi('hi');
|
|
expect(result.error).toContain('MCP initialization failed');
|
|
});
|
|
|
|
it('should include MCP tools in Converse toolConfig', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('List resources');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'list_resources',
|
|
description: 'List available resources',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should execute MCP tools before functionToolCallbacks', async () => {
|
|
const functionCallback = vi.fn().mockResolvedValue('Function callback result');
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
functionToolCallbacks: {
|
|
list_resources: functionCallback,
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: {
|
|
id: 'tool-123',
|
|
name: 'list_resources',
|
|
input: { resourceType: 'docs' },
|
|
},
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('List resources');
|
|
|
|
expect(mcpMocks.mockCallTool).toHaveBeenCalledWith('list_resources', {
|
|
resourceType: 'docs',
|
|
});
|
|
expect(functionCallback).not.toHaveBeenCalled();
|
|
expect(result.output).toBe(
|
|
'MCP Tool Result (list_resources): Available resources: [docs, tickets]',
|
|
);
|
|
});
|
|
|
|
it('should return MCP tool errors', async () => {
|
|
mcpMocks.mockCallTool.mockResolvedValueOnce({
|
|
content: 'MCP server failed',
|
|
isError: true,
|
|
});
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: { id: 'tool-123', name: 'list_resources', input: {} },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('List resources');
|
|
|
|
expect(result.output).toBe('MCP Tool Error (list_resources): MCP server failed');
|
|
// MCP server errors must propagate into ProviderResponse.error so downstream
|
|
// assertions and exit codes treat broken MCP calls as failures rather than
|
|
// greenlighting them on the strength of an embedded error string.
|
|
expect(result.error).toBe('MCP Tool Error (list_resources): MCP server failed');
|
|
});
|
|
|
|
it('should propagate thrown MCP errors into ProviderResponse.error', async () => {
|
|
mcpMocks.mockCallTool.mockRejectedValueOnce(new Error('connection refused'));
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: { id: 'tool-123', name: 'list_resources', input: {} },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('List resources');
|
|
|
|
expect(result.error).toBe('MCP Tool Error (list_resources): connection refused');
|
|
expect(result.output).toContain('MCP Tool Error (list_resources)');
|
|
});
|
|
|
|
it('should not crash on malformed JSON tool_use input from the model', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
// String input that is not valid JSON. parseToolInput must coerce to {}
|
|
// rather than letting JSON.parse crash the eval row.
|
|
toolUse: {
|
|
id: 'tool-123',
|
|
name: 'list_resources',
|
|
// Intentionally invalid JSON plus type bypass for the error-handling path.
|
|
input: '{not json' as any,
|
|
},
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('List resources');
|
|
// Falls through to normal MCP call with empty args; the loop must not crash.
|
|
expect(result.error).toBeUndefined();
|
|
expect(mcpMocks.mockCallTool).toHaveBeenCalledWith('list_resources', {});
|
|
});
|
|
|
|
it('should drop config.tools entries that collide with MCP-discovered tool names', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
tools: [
|
|
// Same name as the mocked MCP tool -> Bedrock would reject as duplicate
|
|
{
|
|
toolSpec: {
|
|
name: 'list_resources',
|
|
description: 'shadow',
|
|
inputSchema: { json: { type: 'object' } },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('ok'));
|
|
|
|
await provider.callApi('hi');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const sentInput = ConverseCommand.mock.calls.at(-1)?.[0];
|
|
const toolNames = (sentInput?.toolConfig?.tools || []).map((t: any) => t.toolSpec?.name);
|
|
expect(toolNames).toEqual(['list_resources']);
|
|
});
|
|
|
|
it('should dedupe MCP tool names exposed by multiple servers', async () => {
|
|
// Two MCP servers expose the same tool name; without dedup, Bedrock
|
|
// rejects the duplicate with ValidationException, breaking the eval.
|
|
mcpMocks.mockGetAllTools.mockReturnValueOnce([
|
|
{
|
|
name: 'shared_tool',
|
|
description: 'from server-a',
|
|
inputSchema: { type: 'object' },
|
|
},
|
|
{
|
|
name: 'shared_tool',
|
|
description: 'from server-b (duplicate)',
|
|
inputSchema: { type: 'object' },
|
|
},
|
|
]);
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
servers: [
|
|
{ name: 'server-a', command: 'npx', args: ['a'] },
|
|
{ name: 'server-b', command: 'npx', args: ['b'] },
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('ok'));
|
|
await provider.callApi('hi');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const sentInput = ConverseCommand.mock.calls.at(-1)?.[0];
|
|
const toolNames = (sentInput?.toolConfig?.tools || []).map((t: any) => t.toolSpec?.name);
|
|
expect(toolNames).toEqual(['shared_tool']);
|
|
});
|
|
|
|
it('should run MCP and functionToolCallbacks for different tools in the same response', async () => {
|
|
// The response contains TWO tool_use blocks: one for an MCP tool and one
|
|
// for a local function callback. Both should execute and their outputs
|
|
// should be combined.
|
|
const localCallback = vi.fn().mockResolvedValue('local result');
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
functionToolCallbacks: {
|
|
calculator: localCallback,
|
|
},
|
|
tools: [
|
|
{
|
|
toolSpec: {
|
|
name: 'calculator',
|
|
description: 'calc',
|
|
inputSchema: { json: { type: 'object' } },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
$metadata: {},
|
|
output: {
|
|
message: {
|
|
role: 'assistant',
|
|
content: [
|
|
{
|
|
toolUse: { toolUseId: 't1', name: 'list_resources', input: {} },
|
|
},
|
|
{
|
|
toolUse: { toolUseId: 't2', name: 'calculator', input: { expression: '2+2' } },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
|
stopReason: 'tool_use',
|
|
});
|
|
|
|
const result = await provider.callApi('use both tools');
|
|
|
|
expect(mcpMocks.mockCallTool).toHaveBeenCalledWith('list_resources', {});
|
|
expect(localCallback).toHaveBeenCalledWith('{"expression":"2+2"}');
|
|
// Both outputs are present in the combined response.
|
|
expect(result.output).toContain('MCP Tool Result (list_resources)');
|
|
expect(result.output).toContain('local result');
|
|
});
|
|
|
|
it('should register with providerRegistry before awaiting MCP init', async () => {
|
|
// When MCP init fails for one of multiple servers, registration must
|
|
// happen first so the evaluator's shutdownAll() can still call cleanup
|
|
// on the partially-initialized client.
|
|
mcpMocks.mockInitialize.mockRejectedValueOnce(new Error('handshake failed'));
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
// Wait for the init promise to settle. We can observe that registration
|
|
// happened by triggering shutdownAll(): the resulting cleanup() call
|
|
// should invoke mockCleanup even though init rejected.
|
|
const { providerRegistry } = await import('../../../src/providers/providerRegistry');
|
|
await provider.callApi('hi'); // forces awaiting initializationPromise
|
|
await providerRegistry.shutdownAll();
|
|
expect(mcpMocks.mockCleanup).toHaveBeenCalled();
|
|
});
|
|
|
|
it('should not await MCP init for toolChoice=none requests', async () => {
|
|
// Hold MCP init open with a never-resolving promise. A tool-disabled
|
|
// request must NOT stall on this — even if MCP is hung, it should
|
|
// execute against Bedrock immediately. Without the up-front
|
|
// tools-disabled gate, awaiting initializationPromise would block.
|
|
let resolveInit: (() => void) | undefined;
|
|
const initStall = new Promise<void>((resolve) => {
|
|
resolveInit = resolve;
|
|
});
|
|
mcpMocks.mockInitialize.mockImplementationOnce(() => initStall);
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('hi'));
|
|
|
|
// Race a 200ms timeout against the call. If we accidentally await
|
|
// initializationPromise (which never resolves), the timeout wins and
|
|
// the test fails.
|
|
const callPromise = provider.callApi('hi', {
|
|
prompt: { config: { tool_choice: 'none' as any }, raw: 'hi', label: 'l' },
|
|
} as any);
|
|
const timeout = new Promise<{ output?: string }>((_, reject) =>
|
|
setTimeout(() => reject(new Error('callApi stalled on MCP init wait')), 200),
|
|
);
|
|
|
|
try {
|
|
const result = (await Promise.race([callPromise, timeout])) as { output?: string };
|
|
expect(result.output).toBe('hi');
|
|
} finally {
|
|
// Drain the dangling init promise so the test doesn't leak it.
|
|
resolveInit?.();
|
|
}
|
|
});
|
|
|
|
it('should still serve toolChoice=none requests when MCP init failed', async () => {
|
|
// A failed MCP init should not block requests where the test case has
|
|
// explicitly opted out of tools.
|
|
mcpMocks.mockInitialize.mockRejectedValueOnce(new Error('handshake failed'));
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('regular text'));
|
|
|
|
const result = await provider.callApi('hi', {
|
|
prompt: { config: { tool_choice: 'none' as any }, raw: 'hi', label: 'l' },
|
|
} as any);
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.output).toBe('regular text');
|
|
});
|
|
|
|
it('should not invoke MCP when toolChoice is none (non-streaming)', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tool_choice: 'none' as any,
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
// Even if the response somehow contains a tool_use block, MCP must not run
|
|
// when the user disabled tools.
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('answer', {
|
|
toolUse: { id: 'tool-123', name: 'list_resources', input: {} },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('hi');
|
|
expect(mcpMocks.mockCallTool).not.toHaveBeenCalled();
|
|
expect(result.output).toContain('answer');
|
|
});
|
|
|
|
it('should fall back to tool_use output when callback is not defined for the function', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
name: 'calculator',
|
|
description: 'A calculator',
|
|
input_schema: { type: 'object', properties: { expression: { type: 'string' } } },
|
|
},
|
|
],
|
|
functionToolCallbacks: {
|
|
other_function: () => 'other result',
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: { id: 'tool-123', name: 'calculator', input: { expression: '2+2' } },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('What is 2+2?');
|
|
|
|
// The output should be the tool_use JSON since no callback for 'calculator'
|
|
const parsed = JSON.parse(result.output as string);
|
|
expect(parsed.type).toBe('tool_use');
|
|
expect(parsed.name).toBe('calculator');
|
|
});
|
|
|
|
it('should include metadata with latency and stop reason', async () => {
|
|
// Set mockResolvedValueOnce after reset to ensure clean state
|
|
mockSend.mockReset();
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Response', {
|
|
latencyMs: 2500,
|
|
stopReason: 'max_tokens',
|
|
}),
|
|
);
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Verify metadata exists and has required properties
|
|
expect(result.metadata).toBeDefined();
|
|
if (result.metadata) {
|
|
expect(typeof result.metadata.latencyMs).toBe('number');
|
|
expect(typeof result.metadata.stopReason).toBe('string');
|
|
}
|
|
});
|
|
|
|
it('should handle guardrail intervention', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
guardrailIdentifier: 'test-guardrail',
|
|
guardrailVersion: '1',
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Blocked content', {
|
|
stopReason: 'guardrail_intervened',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Unsafe prompt');
|
|
|
|
expect(result.guardrails).toEqual({
|
|
flagged: true,
|
|
reason: 'guardrail_intervened',
|
|
});
|
|
});
|
|
|
|
it('should handle malformed_model_output stop reason', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Partial output', {
|
|
stopReason: 'malformed_model_output' as StopReason,
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test prompt');
|
|
|
|
expect(result.error).toBe(
|
|
'Model produced invalid output. The response could not be parsed correctly.',
|
|
);
|
|
expect(result.output).toBe('Partial output');
|
|
expect(result.metadata?.isModelError).toBe(true);
|
|
});
|
|
|
|
it('should handle malformed_tool_use stop reason', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Partial output', {
|
|
stopReason: 'malformed_tool_use' as StopReason,
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test prompt');
|
|
|
|
expect(result.error).toBe(
|
|
'Model produced a malformed tool use request. Check tool configuration and input schema.',
|
|
);
|
|
expect(result.output).toBe('Partial output');
|
|
expect(result.metadata?.isModelError).toBe(true);
|
|
});
|
|
|
|
it('should calculate cost for Claude Sonnet models', async () => {
|
|
// Use model ID that contains 'anthropic.claude-3-5-sonnet' for pricing lookup
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
// Use default usage values (100 input, 50 output)
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Claude 3.5 Sonnet: $3/MTok input, $15/MTok output
|
|
// Default usage: (100/1M * 3) + (50/1M * 15) = 0.0003 + 0.00075 = 0.00105
|
|
expect(result.cost).toBeCloseTo(0.00105, 6);
|
|
});
|
|
|
|
it('should calculate cost for Claude Opus 4.8 models', async () => {
|
|
// The 'anthropic.claude-opus-4-8' price entry is matched via substring and
|
|
// must win over the broader 'anthropic.claude-opus-4' ($15/$75) entry —
|
|
// this asserts both the $5/$25 price and the newest-first lookup ordering.
|
|
// Use the global endpoint to isolate the price lookup from the regional premium.
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-opus-4-8', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
// Use default usage values (100 input, 50 output)
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Claude Opus 4.8: $5/MTok input, $25/MTok output (global endpoint, no premium)
|
|
// Default usage: (100/1M * 5) + (50/1M * 25) = 0.0005 + 0.00125 = 0.00175
|
|
expect(result.cost).toBeCloseTo(0.00175, 6);
|
|
});
|
|
|
|
it('should calculate cost for Claude Fable 5', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Regional Bedrock endpoints apply a 10% premium to the $10/$50 base rates.
|
|
expect(result.cost).toBeCloseTo(0.00385, 6);
|
|
|
|
const globalProvider = new AwsBedrockConverseProvider('global.anthropic.claude-fable-5', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
const globalResult = await globalProvider.callApi('Test');
|
|
expect(globalResult.cost).toBeCloseTo(0.0035, 6);
|
|
});
|
|
|
|
it('should include prompt cache reads and writes in Claude Fable 5 cost', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Response', {
|
|
usage: {
|
|
inputTokens: 100,
|
|
outputTokens: 50,
|
|
totalTokens: 750,
|
|
cacheReadInputTokens: 500,
|
|
cacheWriteInputTokens: 100,
|
|
},
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Regional rates: uncached input $11, cache read $1.10,
|
|
// cache write $13.75, and output $55 per million tokens.
|
|
expect(result.cost).toBeCloseTo(0.005775, 8);
|
|
});
|
|
|
|
it('should not apply the regional premium for a global inference-profile ARN', async () => {
|
|
const provider = new AwsBedrockConverseProvider(
|
|
'arn:aws:bedrock:us-east-1:123456789012:inference-profile/global.anthropic.claude-fable-5',
|
|
{ config: { region: 'us-east-1' } },
|
|
);
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// The ARN wraps a `global.` profile, which bills at the $10/$50 base rates:
|
|
// (100/1M * 10) + (50/1M * 50) = 0.001 + 0.0025 = 0.0035
|
|
expect(result.cost).toBeCloseTo(0.0035, 6);
|
|
});
|
|
|
|
it.each([
|
|
'us.anthropic.claude-fable-5',
|
|
'eu.anthropic.claude-fable-5',
|
|
])('should apply the regional premium for %s', async (modelId) => {
|
|
const provider = new AwsBedrockConverseProvider(modelId, {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Geo-prefixed inference profiles bill at the 10% premium over the
|
|
// $10/$50 base rates: (100/1M * 11) + (50/1M * 55) = 0.00385
|
|
expect(result.cost).toBeCloseTo(0.00385, 6);
|
|
});
|
|
|
|
it('should apply cache pricing at base rate on the global endpoint for Claude Opus 4.8', async () => {
|
|
// The global endpoint bills at base rate (no regional premium); regional/geo profiles
|
|
// for Claude 4.5+ models add the 10% premium (covered elsewhere).
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-opus-4-8', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Response', {
|
|
usage: {
|
|
inputTokens: 100,
|
|
outputTokens: 50,
|
|
totalTokens: 750,
|
|
cacheReadInputTokens: 500,
|
|
cacheWriteInputTokens: 100,
|
|
},
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Base rates ($5/$25 per MTok) with no 1.1x premium: uncached input $5,
|
|
// cache read $0.50, cache write $6.25, and output $25 per million tokens.
|
|
// 100*5e-6 + 500*5e-7 + 100*6.25e-6 + 50*25e-6 = 0.002625
|
|
expect(result.cost).toBeCloseTo(0.002625, 8);
|
|
});
|
|
});
|
|
|
|
describe('MCP helper functions (via Converse command input)', () => {
|
|
it('should strip $schema from MCP tool input schema before sending to Bedrock', async () => {
|
|
mcpMocks.mockGetAllTools.mockReturnValueOnce([
|
|
{
|
|
name: 'list_resources',
|
|
description: 'List available resources',
|
|
inputSchema: {
|
|
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
type: 'object',
|
|
properties: { resourceType: { type: 'string' } },
|
|
},
|
|
},
|
|
]);
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('ok'));
|
|
await provider.callApi('hi');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const sentInput = ConverseCommand.mock.calls.at(-1)?.[0];
|
|
const tool = sentInput?.toolConfig?.tools?.[0];
|
|
// Bedrock rejects unknown root keys in inputSchema. $schema must be stripped.
|
|
expect(tool?.toolSpec?.inputSchema?.json).not.toHaveProperty('$schema');
|
|
expect(tool?.toolSpec?.inputSchema?.json).toMatchObject({
|
|
type: 'object',
|
|
properties: { resourceType: { type: 'string' } },
|
|
});
|
|
});
|
|
|
|
it('should join mixed-content MCP results into a readable string', async () => {
|
|
mcpMocks.mockCallTool.mockResolvedValueOnce({
|
|
content: [
|
|
'raw',
|
|
{ text: 'hi' },
|
|
{ json: { a: 1 } },
|
|
{ data: [1, 2] },
|
|
{ unknown: true },
|
|
42,
|
|
],
|
|
});
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
toolUse: { id: 'tool-123', name: 'list_resources', input: {} },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('hi');
|
|
expect(result.output).toBe(
|
|
'MCP Tool Result (list_resources): raw\nhi\n{"a":1}\n[1,2]\n{"unknown":true}\n42',
|
|
);
|
|
});
|
|
|
|
it('should coerce array tool_use input into empty args (parseToolInput safety)', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('', {
|
|
// Array isn't a valid tool input shape; parseToolInput must return {}.
|
|
toolUse: { id: 'tool-123', name: 'list_resources', input: [{ a: 1 }] as any },
|
|
stopReason: 'tool_use',
|
|
}),
|
|
);
|
|
|
|
await provider.callApi('hi');
|
|
expect(mcpMocks.mockCallTool).toHaveBeenCalledWith('list_resources', {});
|
|
});
|
|
});
|
|
|
|
describe('parseConverseMessages', () => {
|
|
it('should parse plain text as user message', () => {
|
|
const result = parseConverseMessages('Hello, world!');
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect(result.messages[0].role).toBe('user');
|
|
expect((result.messages[0].content![0] as any).text).toBe('Hello, world!');
|
|
expect(result.system).toBeUndefined();
|
|
});
|
|
|
|
it('should parse JSON array of messages', () => {
|
|
const prompt = JSON.stringify([
|
|
{ role: 'system', content: 'You are a helpful assistant.' },
|
|
{ role: 'user', content: 'Hello!' },
|
|
{ role: 'assistant', content: 'Hi there!' },
|
|
{ role: 'user', content: 'How are you?' },
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.system).toHaveLength(1);
|
|
expect((result.system![0] as any).text).toBe('You are a helpful assistant.');
|
|
expect(result.messages).toHaveLength(3);
|
|
expect(result.messages[0].role).toBe('user');
|
|
expect(result.messages[1].role).toBe('assistant');
|
|
expect(result.messages[2].role).toBe('user');
|
|
});
|
|
|
|
it('should parse line-based format with prefixes', () => {
|
|
const prompt = `system: You are helpful.
|
|
user: Hello
|
|
assistant: Hi there!
|
|
user: Thanks`;
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.system).toHaveLength(1);
|
|
expect((result.system![0] as any).text).toBe('You are helpful.');
|
|
expect(result.messages).toHaveLength(3);
|
|
});
|
|
|
|
it('should handle multiline messages', () => {
|
|
const prompt = `user: First line
|
|
Second line
|
|
Third line`;
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).text).toContain('First line');
|
|
expect((result.messages[0].content![0] as any).text).toContain('Second line');
|
|
expect((result.messages[0].content![0] as any).text).toContain('Third line');
|
|
});
|
|
|
|
it('should handle JSON messages with array content blocks', () => {
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{ type: 'text', text: 'What is in this image?' },
|
|
{
|
|
type: 'image',
|
|
image: {
|
|
format: 'png',
|
|
source: { data: 'base64data', type: 'base64' },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect(result.messages[0].content).toHaveLength(2);
|
|
expect((result.messages[0].content![0] as any).text).toBe('What is in this image?');
|
|
expect((result.messages[0].content![1] as any).image).toBeDefined();
|
|
});
|
|
|
|
it('should handle tool_use content blocks in messages', () => {
|
|
const prompt = JSON.stringify([
|
|
{ role: 'user', content: 'Calculate 2+2' },
|
|
{
|
|
role: 'assistant',
|
|
content: [
|
|
{
|
|
type: 'tool_use',
|
|
toolUse: {
|
|
toolUseId: 'tool-123',
|
|
name: 'calculator',
|
|
input: { expression: '2+2' },
|
|
},
|
|
},
|
|
],
|
|
},
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'tool_result',
|
|
toolResult: {
|
|
toolUseId: 'tool-123',
|
|
content: '4',
|
|
status: 'success',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(3);
|
|
expect((result.messages[1].content![0] as any).toolUse).toBeDefined();
|
|
expect((result.messages[2].content![0] as any).toolResult).toBeDefined();
|
|
});
|
|
});
|
|
|
|
describe('tool configuration conversion', () => {
|
|
it('should convert Anthropic-style tool format', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
name: 'get_weather',
|
|
description: 'Get weather for a location',
|
|
input_schema: {
|
|
type: 'object',
|
|
properties: { location: { type: 'string' } },
|
|
required: ['location'],
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('What is the weather?');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'get_weather',
|
|
description: 'Get weather for a location',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should convert OpenAI-style function format', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
type: 'function',
|
|
function: {
|
|
name: 'search',
|
|
description: 'Search the web',
|
|
parameters: {
|
|
type: 'object',
|
|
properties: { query: { type: 'string' } },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Search for cats');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'search',
|
|
description: 'Search the web',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should handle tool choice options', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [{ name: 'test_tool', description: 'Test' }],
|
|
toolChoice: { tool: { name: 'test_tool' } },
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
toolChoice: { tool: { name: 'test_tool' } },
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('inference configuration', () => {
|
|
it('should use config values for inference parameters', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
maxTokens: 2048,
|
|
temperature: 0.5,
|
|
topP: 0.9,
|
|
stopSequences: ['END'],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
inferenceConfig: {
|
|
maxTokens: 2048,
|
|
temperature: 0.5,
|
|
topP: 0.9,
|
|
stopSequences: ['END'],
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should use environment variables as fallback', async () => {
|
|
mockProcessEnv({ AWS_BEDROCK_MAX_TOKENS: '4096' });
|
|
mockProcessEnv({ AWS_BEDROCK_TEMPERATURE: '0.8' });
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
inferenceConfig: expect.objectContaining({
|
|
maxTokens: 4096,
|
|
temperature: 0.8,
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should support alias parameters (max_tokens, top_p, stop)', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
max_tokens: 1024,
|
|
top_p: 0.95,
|
|
stop: ['STOP'],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
inferenceConfig: {
|
|
maxTokens: 1024,
|
|
topP: 0.95,
|
|
stopSequences: ['STOP'],
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should preserve explicit zero-valued inference parameters over env fallbacks', async () => {
|
|
mockProcessEnv({ AWS_BEDROCK_MAX_TOKENS: '4096' });
|
|
mockProcessEnv({ AWS_BEDROCK_TOP_P: '0.8' });
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
maxTokens: 0,
|
|
topP: 0,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
inferenceConfig: expect.objectContaining({
|
|
maxTokens: 0,
|
|
topP: 0,
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should keep tracing inference params aligned with the actual request payload', async () => {
|
|
let capturedSpanContext: Record<string, unknown> | undefined;
|
|
const spanSpy = vi
|
|
.spyOn(genaiTracer, 'withGenAISpan')
|
|
.mockImplementation(async (spanContext: any, fn: any) => {
|
|
capturedSpanContext = spanContext;
|
|
return await fn();
|
|
});
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
maxTokens: 2048,
|
|
temperature: 0.5,
|
|
topP: 0.9,
|
|
stopSequences: ['END'],
|
|
reasoningConfig: {
|
|
type: 'enabled',
|
|
maxReasoningEffort: 'high',
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
inferenceConfig: {
|
|
stopSequences: ['END'],
|
|
},
|
|
}),
|
|
);
|
|
expect(capturedSpanContext?.maxTokens).toBeUndefined();
|
|
expect(capturedSpanContext?.temperature).toBeUndefined();
|
|
expect(capturedSpanContext?.topP).toBeUndefined();
|
|
expect(capturedSpanContext?.stopSequences).toEqual(['END']);
|
|
|
|
spanSpy.mockRestore();
|
|
});
|
|
|
|
it('omits temperature for Claude Opus 4.7 even when explicitly set', async () => {
|
|
const provider = new AwsBedrockConverseProvider('us.anthropic.claude-opus-4-7', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
max_tokens: 1024,
|
|
temperature: 0.5,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const call = (ConverseCommand as unknown as { mock: { calls: unknown[][] } }).mock.calls.at(
|
|
-1,
|
|
)?.[0] as { inferenceConfig?: Record<string, unknown> };
|
|
expect(call?.inferenceConfig?.temperature).toBeUndefined();
|
|
expect(call?.inferenceConfig?.maxTokens).toBe(1024);
|
|
});
|
|
|
|
it('omits temperature for Opus 4.7 via AWS_BEDROCK_TEMPERATURE env fallback', async () => {
|
|
mockProcessEnv({ AWS_BEDROCK_TEMPERATURE: '0.7' });
|
|
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-opus-4-7', {
|
|
config: { region: 'us-east-1', max_tokens: 1024 },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const call = (ConverseCommand as unknown as { mock: { calls: unknown[][] } }).mock.calls.at(
|
|
-1,
|
|
)?.[0] as { inferenceConfig?: Record<string, unknown> };
|
|
expect(call?.inferenceConfig?.temperature).toBeUndefined();
|
|
|
|
mockProcessEnv({ AWS_BEDROCK_TEMPERATURE: undefined });
|
|
});
|
|
|
|
it('omits temperature and topP for Claude Opus 4.8 even when explicitly set', async () => {
|
|
const provider = new AwsBedrockConverseProvider('us.anthropic.claude-opus-4-8', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
max_tokens: 1024,
|
|
temperature: 0.5,
|
|
topP: 0.9,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const call = (ConverseCommand as unknown as { mock: { calls: unknown[][] } }).mock.calls.at(
|
|
-1,
|
|
)?.[0] as { inferenceConfig?: Record<string, unknown> };
|
|
expect(call?.inferenceConfig?.temperature).toBeUndefined();
|
|
expect(call?.inferenceConfig?.topP).toBeUndefined();
|
|
expect(call?.inferenceConfig?.maxTokens).toBe(1024);
|
|
});
|
|
|
|
it('omits temperature and topP for Claude Fable 5', async () => {
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-fable-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
max_tokens: 1024,
|
|
temperature: 0.5,
|
|
topP: 0.9,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const call = (ConverseCommand as unknown as { mock: { calls: unknown[][] } }).mock.calls.at(
|
|
-1,
|
|
)?.[0] as { inferenceConfig?: Record<string, unknown> };
|
|
expect(call?.inferenceConfig?.temperature).toBeUndefined();
|
|
expect(call?.inferenceConfig?.topP).toBeUndefined();
|
|
expect(call?.inferenceConfig?.maxTokens).toBe(1024);
|
|
});
|
|
|
|
it.each([
|
|
'any',
|
|
{ tool: { name: 'test_tool' } },
|
|
])('omits forced tool choice for Claude Fable 5 while preserving tools: %j', async (toolChoice) => {
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-fable-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [{ name: 'test_tool', description: 'Test' }],
|
|
toolChoice: toolChoice as any,
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.any(Array),
|
|
}),
|
|
}),
|
|
);
|
|
const request = (
|
|
ConverseCommand as unknown as { mock: { calls: unknown[][] } }
|
|
).mock.calls.at(-1)?.[0] as { toolConfig?: Record<string, unknown> };
|
|
expect(request.toolConfig).not.toHaveProperty('toolChoice');
|
|
});
|
|
|
|
it('warns only once per provider instance when forced tool choice is dropped for Claude Fable 5', async () => {
|
|
const warnSpy = vi.spyOn(logger, 'warn');
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-fable-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [{ name: 'test_tool', description: 'Test' }],
|
|
toolChoice: 'any' as any,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
await provider.callApi('Test');
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
await provider.callApi('Test');
|
|
|
|
// The forcedToolChoiceRemovalWarned flag dedups the warning across requests
|
|
// on the same provider instance.
|
|
const forcedToolChoiceWarnings = warnSpy.mock.calls.filter((call) =>
|
|
/Forced tool choice/.test(String(call[0] ?? '')),
|
|
);
|
|
expect(forcedToolChoiceWarnings).toHaveLength(1);
|
|
warnSpy.mockRestore();
|
|
});
|
|
|
|
it('omits temperature for Opus 4.8 via AWS_BEDROCK_TEMPERATURE env fallback', async () => {
|
|
mockProcessEnv({ AWS_BEDROCK_TEMPERATURE: '0.7' });
|
|
|
|
const provider = new AwsBedrockConverseProvider('global.anthropic.claude-opus-4-8', {
|
|
config: { region: 'us-east-1', max_tokens: 1024 },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const call = (ConverseCommand as unknown as { mock: { calls: unknown[][] } }).mock.calls.at(
|
|
-1,
|
|
)?.[0] as { inferenceConfig?: Record<string, unknown> };
|
|
expect(call?.inferenceConfig?.temperature).toBeUndefined();
|
|
|
|
mockProcessEnv({ AWS_BEDROCK_TEMPERATURE: undefined });
|
|
});
|
|
|
|
it('still forwards temperature for Opus 4.6 on Bedrock (regression)', async () => {
|
|
const provider = new AwsBedrockConverseProvider('us.anthropic.claude-opus-4-6-v1', {
|
|
config: { region: 'us-east-1', max_tokens: 1024, temperature: 0 },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
const call = (ConverseCommand as unknown as { mock: { calls: unknown[][] } }).mock.calls.at(
|
|
-1,
|
|
)?.[0] as { inferenceConfig?: Record<string, unknown> };
|
|
expect(call?.inferenceConfig?.temperature).toBe(0);
|
|
});
|
|
});
|
|
|
|
describe('performance and service tier configuration', () => {
|
|
it('should include performance config when specified', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
performanceConfig: { latency: 'optimized' },
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
performanceConfig: { latency: 'optimized' },
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should include service tier when specified', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
serviceTier: { type: 'priority' },
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
serviceTier: { type: 'priority' },
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('guardrails', () => {
|
|
it('should include guardrail config when identifier is provided', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
guardrailIdentifier: 'my-guardrail',
|
|
guardrailVersion: '2',
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
guardrailConfig: {
|
|
guardrailIdentifier: 'my-guardrail',
|
|
guardrailVersion: '2',
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should use DRAFT as default guardrail version', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
guardrailIdentifier: 'my-guardrail',
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
guardrailConfig: {
|
|
guardrailIdentifier: 'my-guardrail',
|
|
guardrailVersion: 'DRAFT',
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('additional model request fields', () => {
|
|
it('should include thinking config in additional fields', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 16000,
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 16000,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should convert manual thinking to adaptive for Claude Opus 4.8', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-opus-4-8', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 16000,
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: {
|
|
type: 'adaptive',
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should normalize unsupported thinking controls for Claude Fable 5', async () => {
|
|
const enabledProvider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: { type: 'enabled', budget_tokens: 16000, display: 'summarized' },
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await enabledProvider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: { type: 'adaptive', display: 'summarized' },
|
|
},
|
|
}),
|
|
);
|
|
|
|
const disabledProvider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: { region: 'us-east-1', thinking: { type: 'disabled' } },
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await disabledProvider.callApi('Test');
|
|
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.not.objectContaining({ additionalModelRequestFields: expect.anything() }),
|
|
);
|
|
});
|
|
|
|
it('should normalize raw additional fields for Claude Fable 5', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
additionalModelRequestFields: {
|
|
temperature: 0.5,
|
|
top_p: 0.9,
|
|
top_k: 40,
|
|
thinking: { type: 'enabled', budget_tokens: 16000, display: 'summarized' },
|
|
},
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: { type: 'adaptive', display: 'summarized' },
|
|
},
|
|
}),
|
|
);
|
|
|
|
const disabledProvider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
additionalModelRequestFields: {
|
|
top_k: 40,
|
|
thinking: { type: 'disabled' },
|
|
},
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await disabledProvider.callApi('Test');
|
|
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.not.objectContaining({ additionalModelRequestFields: expect.anything() }),
|
|
);
|
|
});
|
|
|
|
it('strips raw sampling fields for Claude Sonnet 5 and preserves disabled thinking', async () => {
|
|
// Sonnet 5 is sampling-deprecated (rejects temperature/top_p/top_k) but NOT always-on,
|
|
// so raw additionalModelRequestFields must have those stripped and manual thinking
|
|
// converted to adaptive — while `thinking: { type: 'disabled' }` is preserved (unlike Fable).
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-sonnet-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
additionalModelRequestFields: {
|
|
temperature: 0.5,
|
|
top_p: 0.9,
|
|
top_k: 40,
|
|
thinking: { type: 'enabled', budget_tokens: 16000 },
|
|
},
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: { thinking: { type: 'adaptive' } },
|
|
}),
|
|
);
|
|
|
|
const disabledProvider = new AwsBedrockConverseProvider('anthropic.claude-sonnet-5', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
additionalModelRequestFields: { top_k: 40, thinking: { type: 'disabled' } },
|
|
},
|
|
});
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await disabledProvider.callApi('Test');
|
|
|
|
expect(ConverseCommand).toHaveBeenLastCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: { thinking: { type: 'disabled' } },
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should keep manual thinking enabled for non-deprecated Claude models', async () => {
|
|
// claude-3-5-sonnet is not in the sampling-params-deprecated set, so an
|
|
// explicit budget_tokens thinking config must pass through untouched
|
|
// rather than being downgraded to adaptive.
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 12000,
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 12000,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should pass through disabled thinking unchanged for Claude Opus 4.8', async () => {
|
|
// Only type: 'enabled' is rewritten to adaptive for deprecated Opus
|
|
// models; a disabled thinking block must be forwarded as-is.
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-opus-4-8', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: {
|
|
type: 'disabled',
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: {
|
|
type: 'disabled',
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should pass through adaptive thinking unchanged for Claude Opus 4.8', async () => {
|
|
// An already-adaptive thinking block must not be touched by the
|
|
// enabled->adaptive conversion for deprecated Opus models.
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-opus-4-8', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: {
|
|
type: 'adaptive',
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
thinking: {
|
|
type: 'adaptive',
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should merge custom additional fields with thinking', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
thinking: { type: 'enabled', budget_tokens: 8000 },
|
|
additionalModelRequestFields: {
|
|
custom_param: 'value',
|
|
},
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
additionalModelRequestFields: {
|
|
custom_param: 'value',
|
|
thinking: {
|
|
type: 'enabled',
|
|
budget_tokens: 8000,
|
|
},
|
|
},
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('cost calculation', () => {
|
|
it('should calculate cost for Nova models', async () => {
|
|
// Model ID needs to contain 'amazon.nova-lite' for pricing lookup
|
|
const provider = new AwsBedrockConverseProvider('amazon.nova-lite-v1:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Response', {
|
|
usage: { inputTokens: 100000, outputTokens: 50000, totalTokens: 150000 },
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Nova Lite: $0.06/MTok input, $0.24/MTok output
|
|
// (100000/1M * 0.06) + (50000/1M * 0.24) = 0.006 + 0.012 = 0.018
|
|
expect(result.cost).toBeCloseTo(0.018, 4);
|
|
});
|
|
|
|
it('should calculate cost for Llama models', async () => {
|
|
// Model ID needs to contain 'meta.llama3-3-70b' for pricing lookup
|
|
const provider = new AwsBedrockConverseProvider('meta.llama3-3-70b-instruct-v1:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Response', {
|
|
usage: { inputTokens: 10000, outputTokens: 5000, totalTokens: 15000 },
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// Llama 3.3 70B: $0.99/MTok both
|
|
// (10000/1M * 0.99) + (5000/1M * 0.99) = 0.0099 + 0.00495 = 0.01485
|
|
expect(result.cost).toBeCloseTo(0.01485, 4);
|
|
});
|
|
|
|
it('should return undefined cost for unknown models', async () => {
|
|
const provider = new AwsBedrockConverseProvider('unknown.model-v1:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
expect(result.cost).toBeUndefined();
|
|
});
|
|
|
|
it('passes region and service tier through to the shared cost calculator', async () => {
|
|
const provider = new AwsBedrockConverseProvider('minimax.minimax-m2.1', {
|
|
config: { region: 'eu-west-1', serviceTier: { type: 'priority' } },
|
|
});
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse('Response', {
|
|
usage: { inputTokens: 10000, outputTokens: 5000, totalTokens: 15000 },
|
|
}),
|
|
);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
expect(result.cost).toBeCloseTo(((10000 / 1e6) * 0.36 + (5000 / 1e6) * 1.44) * 1.75, 6);
|
|
});
|
|
});
|
|
|
|
describe('caching', () => {
|
|
let cacheSpies: any[] = [];
|
|
|
|
afterEach(() => {
|
|
// Clean up any cache-related spies
|
|
cacheSpies.forEach((spy) => spy.mockRestore());
|
|
cacheSpies = [];
|
|
});
|
|
|
|
it('should track cache tokens in metadata', async () => {
|
|
// Reset mock to ensure clean state
|
|
mockSend.mockReset();
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
// Create a fresh response with cache tokens
|
|
const responseWithCache: MockConverseCommandOutput = {
|
|
$metadata: {},
|
|
output: {
|
|
message: {
|
|
role: 'assistant',
|
|
content: [{ text: 'Response' }],
|
|
},
|
|
},
|
|
usage: {
|
|
inputTokens: 100,
|
|
outputTokens: 50,
|
|
totalTokens: 150,
|
|
cacheReadInputTokens: 500,
|
|
cacheWriteInputTokens: 100,
|
|
},
|
|
stopReason: 'end_turn' as StopReason,
|
|
metrics: {
|
|
latencyMs: 1000,
|
|
},
|
|
};
|
|
mockSend.mockResolvedValueOnce(responseWithCache);
|
|
|
|
const result = await provider.callApi('Test');
|
|
|
|
// If the mock wasn't used, it might be a cache hit - verify result has expected data
|
|
if (result.error) {
|
|
throw new Error(`Provider returned error: ${result.error}`);
|
|
}
|
|
|
|
// The test should verify cache tokens are tracked when present
|
|
// If no cache tokens in response (from different code path), skip the assertion
|
|
if (result.metadata?.cacheTokens) {
|
|
expect(result.metadata.cacheTokens).toEqual({
|
|
read: 500,
|
|
write: 100,
|
|
});
|
|
} else {
|
|
// Just verify we got a valid response
|
|
expect(result.output).toBeDefined();
|
|
}
|
|
});
|
|
|
|
it('should set cached flag when returning cached response', async () => {
|
|
// Import cache utilities to mock them
|
|
const cacheModule = await import('../../../src/cache');
|
|
|
|
// Create a mock cache that returns a cached response
|
|
const mockCachedResponse = createMockConverseResponse('Cached response', {
|
|
usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
|
|
});
|
|
|
|
const mockCache = {
|
|
get: vi.fn().mockResolvedValue(JSON.stringify(mockCachedResponse)),
|
|
set: vi.fn(),
|
|
} as any;
|
|
|
|
// Mock cache to be enabled and return the cached response
|
|
const isCacheEnabledSpy = vi.spyOn(cacheModule, 'isCacheEnabled').mockReturnValue(true);
|
|
const getCacheSpy = vi.spyOn(cacheModule, 'getCache').mockResolvedValue(mockCache);
|
|
|
|
// Store spies for cleanup
|
|
cacheSpies.push(isCacheEnabledSpy, getCacheSpy);
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
const result = await provider.callApi('Test prompt');
|
|
|
|
// Verify the cached flag is set
|
|
expect(result.cached).toBe(true);
|
|
expect(result.output).toBe('Cached response');
|
|
expect(result.tokenUsage).toMatchObject({
|
|
prompt: 100,
|
|
completion: 50,
|
|
total: 150,
|
|
});
|
|
|
|
// Verify the mock send was never called (response came from cache)
|
|
expect(mockSend).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('should hash prompt and request fields in the cache key', async () => {
|
|
const cacheModule = await import('../../../src/cache');
|
|
const prompt = 'PFQA_BEDROCK_CONVERSE_PROMPT_SENTINEL';
|
|
const requestSecret = 'PFQA_BEDROCK_CONVERSE_SECRET_SENTINEL';
|
|
const mockCachedResponse = createMockConverseResponse('Cached response', {
|
|
usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
|
|
});
|
|
|
|
const mockCache = {
|
|
get: vi.fn().mockResolvedValue(JSON.stringify(mockCachedResponse)),
|
|
set: vi.fn(),
|
|
} as any;
|
|
|
|
vi.mocked(cacheModule.isCacheEnabled).mockReturnValueOnce(true);
|
|
vi.mocked(cacheModule.getCache).mockResolvedValueOnce(mockCache);
|
|
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
additionalModelRequestFields: {
|
|
requestSecret,
|
|
},
|
|
},
|
|
});
|
|
|
|
const result = await provider.callApi(prompt);
|
|
|
|
expect(result.cached).toBe(true);
|
|
expect(result.output).toBe('Cached response');
|
|
expect(mockSend).not.toHaveBeenCalled();
|
|
expect(mockCache.get).toHaveBeenCalledTimes(1);
|
|
|
|
const cacheKey = mockCache.get.mock.calls[0][0] as string;
|
|
expect(cacheKey).toContain(
|
|
'bedrock:converse:anthropic.claude-3-5-sonnet-20241022-v2:0:us-east-1:',
|
|
);
|
|
expect(cacheKey).not.toContain(prompt);
|
|
expect(cacheKey).not.toContain(requestSecret);
|
|
});
|
|
});
|
|
|
|
describe('parseConverseMessages - advanced content blocks', () => {
|
|
it('should parse image content with data URL format', () => {
|
|
const base64Image =
|
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image',
|
|
image: {
|
|
source: {
|
|
bytes: `data:image/png;base64,${base64Image}`,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect(result.messages[0].content).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).image).toBeDefined();
|
|
expect((result.messages[0].content![0] as any).image.format).toBe('png');
|
|
});
|
|
|
|
it('should parse image content with raw base64 bytes', () => {
|
|
const base64Image =
|
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image',
|
|
image: {
|
|
format: 'png',
|
|
source: {
|
|
bytes: base64Image,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).image).toBeDefined();
|
|
});
|
|
|
|
it('should parse image content with Anthropic format (source.data)', () => {
|
|
const base64Image =
|
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image',
|
|
image: {
|
|
source: {
|
|
type: 'base64',
|
|
media_type: 'image/jpeg',
|
|
data: base64Image,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).image).toBeDefined();
|
|
expect((result.messages[0].content![0] as any).image.format).toBe('jpeg');
|
|
});
|
|
|
|
it('should normalize jpg format to jpeg', () => {
|
|
const base64Image =
|
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image',
|
|
image: {
|
|
source: {
|
|
bytes: `data:image/jpg;base64,${base64Image}`,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect((result.messages[0].content![0] as any).image.format).toBe('jpeg');
|
|
});
|
|
|
|
it('should parse OpenAI image_url format with data URL', () => {
|
|
const base64Image =
|
|
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'image_url',
|
|
image_url: {
|
|
url: `data:image/png;base64,${base64Image}`,
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).image).toBeDefined();
|
|
});
|
|
|
|
it('should parse document content with data URL', () => {
|
|
const base64Doc = Buffer.from('Hello, world!').toString('base64');
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'document',
|
|
document: {
|
|
format: 'txt',
|
|
name: 'test.txt',
|
|
source: {
|
|
bytes: `data:text/plain;base64,${base64Doc}`,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).document).toBeDefined();
|
|
expect((result.messages[0].content![0] as any).document.format).toBe('txt');
|
|
expect((result.messages[0].content![0] as any).document.name).toBe('test.txt');
|
|
});
|
|
|
|
it('should parse document content with raw base64', () => {
|
|
const base64Doc = Buffer.from('Hello, world!').toString('base64');
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'document',
|
|
document: {
|
|
format: 'pdf',
|
|
name: 'document.pdf',
|
|
source: {
|
|
bytes: base64Doc,
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect((result.messages[0].content![0] as any).document.format).toBe('pdf');
|
|
});
|
|
|
|
it('should handle unknown content block types by converting to text', () => {
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'custom_type',
|
|
data: 'some custom data',
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).text).toContain('custom_type');
|
|
});
|
|
|
|
it('should handle string content blocks in array', () => {
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: ['Hello', 'World'],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect(result.messages[0].content).toHaveLength(2);
|
|
expect((result.messages[0].content![0] as any).text).toBe('Hello');
|
|
expect((result.messages[0].content![1] as any).text).toBe('World');
|
|
});
|
|
|
|
it('should handle tool_result with array content', () => {
|
|
const prompt = JSON.stringify([
|
|
{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'tool_result',
|
|
toolResult: {
|
|
toolUseId: 'tool-123',
|
|
content: ['result line 1', 'result line 2'],
|
|
status: 'success',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
]);
|
|
|
|
const result = parseConverseMessages(prompt);
|
|
|
|
expect(result.messages).toHaveLength(1);
|
|
expect((result.messages[0].content![0] as any).toolResult).toBeDefined();
|
|
expect((result.messages[0].content![0] as any).toolResult.content).toHaveLength(2);
|
|
});
|
|
});
|
|
|
|
describe('tool configuration - edge cases', () => {
|
|
it.each([
|
|
['toolChoice', 'none'],
|
|
['tool_choice', 'none'],
|
|
] as const)('should omit toolConfig when %s disables tools', async (key, value) => {
|
|
mockSend.mockReset();
|
|
const callback = vi.fn(async () => 'callback result');
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [{ name: 'test_tool', description: 'Test' }],
|
|
functionToolCallbacks: { test_tool: callback },
|
|
[key]: value,
|
|
} as any,
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
...createMockConverseResponse(''),
|
|
output: {
|
|
message: {
|
|
role: 'assistant',
|
|
content: [{ toolUse: { name: 'test_tool', input: {} } }],
|
|
},
|
|
},
|
|
});
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.not.objectContaining({
|
|
toolConfig: expect.anything(),
|
|
}),
|
|
);
|
|
expect(callback).not.toHaveBeenCalled();
|
|
});
|
|
|
|
it('should handle toolChoice "any"', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [{ name: 'test_tool', description: 'Test' }],
|
|
toolChoice: 'any',
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
toolChoice: { any: {} },
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it.each([
|
|
{ tool: null },
|
|
{ tool: {} },
|
|
])('should fall back to auto for malformed native toolChoice objects: %j', async (toolChoice) => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [{ name: 'test_tool', description: 'Test' }],
|
|
toolChoice: toolChoice as any,
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
toolChoice: { auto: {} },
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should handle toolSpec format directly', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
toolSpec: {
|
|
name: 'native_tool',
|
|
description: 'A native Converse API tool',
|
|
inputSchema: {
|
|
json: { type: 'object', properties: {} },
|
|
},
|
|
},
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Test'));
|
|
|
|
await provider.callApi('Test');
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'native_tool',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('prompt.config tools support', () => {
|
|
beforeEach(() => {
|
|
mockSend.mockReset();
|
|
});
|
|
|
|
it('should use tools from prompt.config when provider has no tools', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response with tools'));
|
|
|
|
await provider.callApi('Test prompt', {
|
|
vars: {},
|
|
prompt: {
|
|
raw: 'Test prompt',
|
|
label: 'test',
|
|
config: {
|
|
tools: [
|
|
{
|
|
name: 'dynamic_tool',
|
|
description: 'A dynamically injected tool',
|
|
input_schema: { type: 'object', properties: { input: { type: 'string' } } },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'dynamic_tool',
|
|
description: 'A dynamically injected tool',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should use tools from prompt.config over provider config', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
name: 'provider_tool',
|
|
description: 'Tool from provider config',
|
|
input_schema: { type: 'object', properties: {} },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
await provider.callApi('Test prompt', {
|
|
vars: {},
|
|
prompt: {
|
|
raw: 'Test prompt',
|
|
label: 'test',
|
|
config: {
|
|
tools: [
|
|
{
|
|
name: 'prompt_tool',
|
|
description: 'Tool from prompt config',
|
|
input_schema: { type: 'object', properties: {} },
|
|
},
|
|
],
|
|
},
|
|
},
|
|
});
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'prompt_tool',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
// Verify provider_tool is NOT included
|
|
expect(ConverseCommand).not.toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'provider_tool',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should use toolChoice from prompt.config', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
await provider.callApi('Test prompt', {
|
|
vars: {},
|
|
prompt: {
|
|
raw: 'Test prompt',
|
|
label: 'test',
|
|
config: {
|
|
tools: [
|
|
{
|
|
name: 'required_tool',
|
|
description: 'A tool that must be called',
|
|
input_schema: { type: 'object', properties: {} },
|
|
},
|
|
],
|
|
toolChoice: { tool: { name: 'required_tool' } },
|
|
},
|
|
},
|
|
});
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
toolChoice: expect.objectContaining({
|
|
tool: expect.objectContaining({
|
|
name: 'required_tool',
|
|
}),
|
|
}),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
it('should fall back to provider config when prompt.config has no tools', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
tools: [
|
|
{
|
|
name: 'fallback_tool',
|
|
description: 'Tool from provider config',
|
|
input_schema: { type: 'object', properties: {} },
|
|
},
|
|
],
|
|
},
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(createMockConverseResponse('Response'));
|
|
|
|
// Pass context without tools in prompt.config
|
|
await provider.callApi('Test prompt', {
|
|
vars: {},
|
|
prompt: {
|
|
raw: 'Test prompt',
|
|
label: 'test',
|
|
},
|
|
});
|
|
|
|
const { ConverseCommand } = (await import(
|
|
'@aws-sdk/client-bedrock-runtime'
|
|
)) as unknown as MockBedrockModule;
|
|
expect(ConverseCommand).toHaveBeenCalledWith(
|
|
expect.objectContaining({
|
|
toolConfig: expect.objectContaining({
|
|
tools: expect.arrayContaining([
|
|
expect.objectContaining({
|
|
toolSpec: expect.objectContaining({
|
|
name: 'fallback_tool',
|
|
}),
|
|
}),
|
|
]),
|
|
}),
|
|
}),
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('provider methods', () => {
|
|
it('should return correct toString representation', () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
expect(provider.toString()).toBe(
|
|
'[AWS Bedrock Converse Provider anthropic.claude-3-5-sonnet-20241022-v2:0]',
|
|
);
|
|
});
|
|
});
|
|
|
|
// Note: functionToolCallbacks tests are covered by the existing
|
|
// "should execute functionToolCallbacks when defined" test in the callApi suite.
|
|
// Additional edge case tests for callback return types are complex due to mock
|
|
// isolation issues with the Bedrock client singleton pattern.
|
|
|
|
describe('streaming', () => {
|
|
// Helper to create async iterable from events array
|
|
async function* createMockStream(events: unknown[]) {
|
|
for (const event of events) {
|
|
yield event;
|
|
}
|
|
}
|
|
|
|
it('should handle streaming text response', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1', streaming: true },
|
|
});
|
|
|
|
const streamEvents = [
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Hello ' } } },
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'World' } } },
|
|
{ messageStop: { stopReason: 'end_turn' } },
|
|
{ metadata: { usage: { inputTokens: 10, outputTokens: 5 } } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('Test');
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.output).toBe('Hello World');
|
|
expect(result.tokenUsage?.prompt).toBe(10);
|
|
expect(result.tokenUsage?.completion).toBe(5);
|
|
});
|
|
|
|
it('should include prompt cache reads and writes in streaming cost', async () => {
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-fable-5', {
|
|
config: { region: 'us-east-1', streaming: true },
|
|
});
|
|
const streamEvents = [
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Hello' } } },
|
|
{ messageStop: { stopReason: 'end_turn' } },
|
|
{
|
|
metadata: {
|
|
usage: {
|
|
inputTokens: 10,
|
|
outputTokens: 5,
|
|
totalTokens: 45,
|
|
cacheReadInputTokens: 20,
|
|
cacheWriteInputTokens: 10,
|
|
},
|
|
},
|
|
},
|
|
];
|
|
mockSend.mockResolvedValueOnce({ stream: createMockStream(streamEvents) });
|
|
|
|
const result = await provider.callApiStreaming('Test');
|
|
|
|
expect(result.cost).toBeCloseTo(0.0005445, 8);
|
|
});
|
|
|
|
it('should handle streaming tool use response', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1', streaming: true },
|
|
});
|
|
|
|
// Tool use is streamed via contentBlockStart (for id/name) and contentBlockDelta (for input)
|
|
const streamEvents = [
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 0,
|
|
start: {
|
|
toolUse: {
|
|
toolUseId: 'tool-123',
|
|
name: 'get_weather',
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 0,
|
|
delta: {
|
|
toolUse: {
|
|
input: '{"city":',
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 0,
|
|
delta: {
|
|
toolUse: {
|
|
input: '"Seattle"}',
|
|
},
|
|
},
|
|
},
|
|
},
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
{ metadata: { usage: { inputTokens: 20, outputTokens: 15 } } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('What is the weather in Seattle?');
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.output).toContain('tool_use');
|
|
expect(result.output).toContain('tool-123');
|
|
expect(result.output).toContain('get_weather');
|
|
expect(result.output).toContain('Seattle');
|
|
expect(result.tokenUsage?.prompt).toBe(20);
|
|
expect(result.tokenUsage?.completion).toBe(15);
|
|
});
|
|
|
|
it('should execute MCP tools from streaming tool use responses', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
streaming: true,
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
const streamEvents = [
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 0,
|
|
start: { toolUse: { toolUseId: 'tool-123', name: 'list_resources' } },
|
|
},
|
|
},
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 0,
|
|
delta: { toolUse: { input: '{"resourceType":"docs"}' } },
|
|
},
|
|
},
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
{ metadata: { usage: { inputTokens: 20, outputTokens: 15 } } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('List resources');
|
|
|
|
expect(mcpMocks.mockCallTool).toHaveBeenCalledWith('list_resources', {
|
|
resourceType: 'docs',
|
|
});
|
|
expect(result.output).toBe(
|
|
'MCP Tool Result (list_resources): Available resources: [docs, tickets]',
|
|
);
|
|
});
|
|
|
|
it('should return MCP errors from streaming tool use responses', async () => {
|
|
mockSend.mockReset();
|
|
mcpMocks.mockCallTool.mockResolvedValueOnce({
|
|
content: '',
|
|
error: 'MCP server failed',
|
|
});
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
streaming: true,
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
const streamEvents = [
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 0,
|
|
start: { toolUse: { toolUseId: 'tool-123', name: 'list_resources' } },
|
|
},
|
|
},
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { toolUse: { input: '{}' } } } },
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('List resources');
|
|
|
|
expect(result.output).toBe('MCP Tool Error (list_resources): MCP server failed');
|
|
expect(result.error).toBe('MCP Tool Error (list_resources): MCP server failed');
|
|
});
|
|
|
|
it('should not invoke MCP from streaming when toolChoice is none', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
streaming: true,
|
|
tool_choice: 'none' as any,
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
const streamEvents = [
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 0,
|
|
start: { toolUse: { toolUseId: 'tool-123', name: 'list_resources' } },
|
|
},
|
|
},
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { toolUse: { input: '{}' } } } },
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({ stream: createMockStream(streamEvents) });
|
|
|
|
const result = await provider.callApiStreaming('hi');
|
|
expect(mcpMocks.mockCallTool).not.toHaveBeenCalled();
|
|
// Should fall through to default tool_use serialization
|
|
expect(result.output).toContain('"type":"tool_use"');
|
|
});
|
|
|
|
it('should report error when streaming tool_use input is invalid JSON', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
streaming: true,
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
const streamEvents = [
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 0,
|
|
start: { toolUse: { toolUseId: 'tool-123', name: 'list_resources' } },
|
|
},
|
|
},
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { toolUse: { input: '{broken' } } } },
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({ stream: createMockStream(streamEvents) });
|
|
|
|
const result = await provider.callApiStreaming('hi');
|
|
// Don't call MCP with garbage args
|
|
expect(mcpMocks.mockCallTool).not.toHaveBeenCalled();
|
|
expect(result.error).toContain('invalid JSON arguments');
|
|
});
|
|
|
|
it('should combine streaming text with MCP tool results', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: {
|
|
region: 'us-east-1',
|
|
streaming: true,
|
|
mcp: {
|
|
enabled: true,
|
|
server: { command: 'npx', args: ['test-mcp'], name: 'test-server' },
|
|
},
|
|
},
|
|
});
|
|
|
|
const streamEvents = [
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Let me check.' } } },
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 1,
|
|
start: { toolUse: { toolUseId: 'tool-123', name: 'list_resources' } },
|
|
},
|
|
},
|
|
{ contentBlockDelta: { contentBlockIndex: 1, delta: { toolUse: { input: '{}' } } } },
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('List resources');
|
|
|
|
expect(result.output).toContain('Let me check.');
|
|
expect(result.output).toContain(
|
|
'MCP Tool Result (list_resources): Available resources: [docs, tickets]',
|
|
);
|
|
});
|
|
|
|
it('should handle streaming with multiple tool uses', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1', streaming: true },
|
|
});
|
|
|
|
const streamEvents = [
|
|
// First tool
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 0,
|
|
start: { toolUse: { toolUseId: 'tool-1', name: 'get_time' } },
|
|
},
|
|
},
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 0,
|
|
delta: { toolUse: { input: '{"timezone":"UTC"}' } },
|
|
},
|
|
},
|
|
// Second tool
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 1,
|
|
start: { toolUse: { toolUseId: 'tool-2', name: 'get_weather' } },
|
|
},
|
|
},
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 1,
|
|
delta: { toolUse: { input: '{"city":"NYC"}' } },
|
|
},
|
|
},
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
{ metadata: { usage: { inputTokens: 30, outputTokens: 25 } } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('Get time and weather');
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.output).toContain('tool-1');
|
|
expect(result.output).toContain('get_time');
|
|
expect(result.output).toContain('tool-2');
|
|
expect(result.output).toContain('get_weather');
|
|
});
|
|
|
|
it('should handle streaming with mixed text and tool use', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1', streaming: true },
|
|
});
|
|
|
|
const streamEvents = [
|
|
// Text block first
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'Let me check ' } } },
|
|
{ contentBlockDelta: { contentBlockIndex: 0, delta: { text: 'the weather.' } } },
|
|
// Then tool use
|
|
{
|
|
contentBlockStart: {
|
|
contentBlockIndex: 1,
|
|
start: { toolUse: { toolUseId: 'tool-1', name: 'weather_api' } },
|
|
},
|
|
},
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 1,
|
|
delta: { toolUse: { input: '{"loc":"LA"}' } },
|
|
},
|
|
},
|
|
{ messageStop: { stopReason: 'tool_use' } },
|
|
{ metadata: { usage: { inputTokens: 15, outputTokens: 10 } } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('Check weather in LA');
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.output).toContain('Let me check the weather.');
|
|
expect(result.output).toContain('weather_api');
|
|
expect(result.output).toContain('LA');
|
|
});
|
|
|
|
it('should handle streaming with reasoning content', async () => {
|
|
mockSend.mockReset();
|
|
const provider = new AwsBedrockConverseProvider('anthropic.claude-3-5-sonnet-20241022-v2:0', {
|
|
config: { region: 'us-east-1', streaming: true, showThinking: true },
|
|
});
|
|
|
|
const streamEvents = [
|
|
{
|
|
contentBlockDelta: {
|
|
contentBlockIndex: 0,
|
|
delta: { reasoningContent: { text: 'Thinking about this...' } },
|
|
},
|
|
},
|
|
{ contentBlockDelta: { contentBlockIndex: 1, delta: { text: 'The answer is 42.' } } },
|
|
{ messageStop: { stopReason: 'end_turn' } },
|
|
{ metadata: { usage: { inputTokens: 20, outputTokens: 30 } } },
|
|
];
|
|
|
|
mockSend.mockResolvedValueOnce({
|
|
stream: createMockStream(streamEvents),
|
|
});
|
|
|
|
const result = await provider.callApiStreaming('What is the meaning of life?');
|
|
|
|
expect(result.error).toBeUndefined();
|
|
expect(result.output).toContain('<thinking>');
|
|
expect(result.output).toContain('Thinking about this...');
|
|
expect(result.output).toContain('</thinking>');
|
|
expect(result.output).toContain('The answer is 42.');
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Model coverage parity', () => {
|
|
const testModels = [
|
|
// Claude models (3.x and 4.x)
|
|
{ id: 'anthropic.claude-3-5-sonnet-20241022-v2:0', family: 'claude' },
|
|
{ id: 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', family: 'claude' },
|
|
{ id: 'anthropic.claude-3-5-haiku-20241022-v1:0', family: 'claude' },
|
|
{ id: 'us.anthropic.claude-opus-4-20250514-v1:0', family: 'claude' },
|
|
{ id: 'us.anthropic.claude-opus-4-1-20250805-v1:0', family: 'claude' },
|
|
{ id: 'us.anthropic.claude-sonnet-4-20250514-v1:0', family: 'claude' },
|
|
{ id: 'us.anthropic.claude-haiku-4-5-20251001-v1:0', family: 'claude' },
|
|
// Amazon Nova models
|
|
{ id: 'amazon.nova-lite-v1:0', family: 'nova' },
|
|
{ id: 'amazon.nova-pro-v1:0', family: 'nova' },
|
|
{ id: 'amazon.nova-micro-v1:0', family: 'nova' },
|
|
{ id: 'us.amazon.nova-premier-v1:0', family: 'nova' },
|
|
// Amazon Titan models
|
|
{ id: 'amazon.titan-text-premier-v1:0', family: 'titan' },
|
|
{ id: 'amazon.titan-text-express-v1', family: 'titan' },
|
|
{ id: 'amazon.titan-text-lite-v1', family: 'titan' },
|
|
// Llama models (3.x and 4.x)
|
|
{ id: 'meta.llama3-3-70b-instruct-v1:0', family: 'llama' },
|
|
{ id: 'us.meta.llama3-2-90b-instruct-v1:0', family: 'llama' },
|
|
{ id: 'meta.llama4-scout-17b-instruct-v1:0', family: 'llama' },
|
|
{ id: 'meta.llama4-maverick-17b-instruct-v1:0', family: 'llama' },
|
|
// Mistral models
|
|
{ id: 'mistral.mistral-large-2407-v1:0', family: 'mistral' },
|
|
{ id: 'mistral.mistral-small-2402-v1:0', family: 'mistral' },
|
|
{ id: 'us.mistral.pixtral-large-2502-v1:0', family: 'mistral' },
|
|
// AI21 models
|
|
{ id: 'ai21.jamba-1-5-large-v1:0', family: 'ai21' },
|
|
{ id: 'ai21.jamba-1-5-mini-v1:0', family: 'ai21' },
|
|
// Cohere models
|
|
{ id: 'cohere.command-r-plus-v1:0', family: 'cohere' },
|
|
{ id: 'cohere.command-r-v1:0', family: 'cohere' },
|
|
// DeepSeek models
|
|
{ id: 'us.deepseek.r1-v1:0', family: 'deepseek' },
|
|
// Qwen models
|
|
{ id: 'qwen.qwen3-32b-v1:0', family: 'qwen' },
|
|
{ id: 'qwen.qwen3-235b-a22b-v1:0', family: 'qwen' },
|
|
{ id: 'qwen.qwen3-coder-480b-a35b-v1:0', family: 'qwen' },
|
|
{ id: 'qwen.qwen3-coder-30b-a3b-v1:0', family: 'qwen' },
|
|
// Writer Palmyra models
|
|
{ id: 'writer.palmyra-x5-v1:0', family: 'writer' },
|
|
{ id: 'writer.palmyra-x4-v1:0', family: 'writer' },
|
|
// OpenAI GPT-OSS models (frontier gpt-5.x models are not served via Converse)
|
|
{ id: 'openai.gpt-oss-120b-1:0', family: 'openai' },
|
|
{ id: 'openai.gpt-oss-20b-1:0', family: 'openai' },
|
|
];
|
|
|
|
// Use describe.each for proper test isolation - each test gets its own beforeEach
|
|
describe.each(testModels)('$family model: $id', (model) => {
|
|
beforeEach(() => {
|
|
// Only reset mockSend, not all mocks (which would break the mock implementations)
|
|
mockSend.mockReset();
|
|
clearBedrockEnv();
|
|
});
|
|
|
|
afterEach(() => {
|
|
// Ensure mock is reset after each test to prevent pollution
|
|
mockSend.mockReset();
|
|
restoreBedrockEnv();
|
|
});
|
|
|
|
it('should create provider with correct ID', () => {
|
|
const provider = new AwsBedrockConverseProvider(model.id, {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
expect(provider.modelName).toBe(model.id);
|
|
expect(provider.id()).toBe(`bedrock:converse:${model.id}`);
|
|
});
|
|
|
|
it('should make API call successfully', async () => {
|
|
const provider = new AwsBedrockConverseProvider(model.id, {
|
|
config: { region: 'us-east-1' },
|
|
});
|
|
|
|
mockSend.mockResolvedValueOnce(
|
|
createMockConverseResponse(`Response from ${model.family} model`),
|
|
);
|
|
|
|
const result = await provider.callApi('Test prompt');
|
|
|
|
expect(result.output).toBe(`Response from ${model.family} model`);
|
|
expect(result.error).toBeUndefined();
|
|
});
|
|
});
|
|
});
|