wehub-resource-sync
0d3cb498a3
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
924 lines
29 KiB
TypeScript
924 lines
29 KiB
TypeScript
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
|
import { fetchWithCache } from '../../src/cache';
|
|
import { AzureChatCompletionProvider } from '../../src/providers/azure/chat';
|
|
import { AzureCompletionProvider } from '../../src/providers/azure/completion';
|
|
import { AzureGenericProvider } from '../../src/providers/azure/generic';
|
|
import { maybeEmitAzureOpenAiWarning } from '../../src/providers/azure/warnings';
|
|
import { HuggingfaceTextGenerationProvider } from '../../src/providers/huggingface';
|
|
import { OpenAiCompletionProvider } from '../../src/providers/openai/completion';
|
|
import { mockProcessEnv } from '../util/utils';
|
|
|
|
import type { TestCase, TestSuite } from '../../src/types/index';
|
|
|
|
vi.mock('../../src/cache', async (importOriginal) => {
|
|
return {
|
|
...(await importOriginal()),
|
|
fetchWithCache: vi.fn(),
|
|
};
|
|
});
|
|
|
|
describe('Azure Provider Tests', () => {
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
vi.spyOn(AzureGenericProvider.prototype as any, 'getAuthHeaders').mockResolvedValue({});
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.clearAllMocks();
|
|
});
|
|
|
|
describe('maybeEmitAzureOpenAiWarning', () => {
|
|
it('should not emit warning when no Azure providers are used', () => {
|
|
const testSuite: TestSuite = {
|
|
providers: [new OpenAiCompletionProvider('foo')],
|
|
defaultTest: {},
|
|
prompts: [],
|
|
};
|
|
const tests: TestCase[] = [
|
|
{
|
|
assert: [{ type: 'llm-rubric', value: 'foo bar' }],
|
|
},
|
|
];
|
|
const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
|
|
expect(result).toBe(false);
|
|
});
|
|
|
|
it('should not emit warning when Azure provider is used alone, but no model graded eval', () => {
|
|
const testSuite: TestSuite = {
|
|
providers: [new AzureCompletionProvider('foo', { config: { apiHost: 'test.azure.com' } })],
|
|
defaultTest: {},
|
|
prompts: [],
|
|
};
|
|
const tests: TestCase[] = [
|
|
{
|
|
assert: [{ type: 'equals' }],
|
|
},
|
|
];
|
|
const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
|
|
expect(result).toBe(false);
|
|
});
|
|
|
|
it('should emit warning when Azure provider is used alone, but with model graded eval', () => {
|
|
const testSuite: TestSuite = {
|
|
providers: [new AzureCompletionProvider('foo', { config: { apiHost: 'test.azure.com' } })],
|
|
defaultTest: {},
|
|
prompts: [],
|
|
};
|
|
const tests: TestCase[] = [
|
|
{
|
|
assert: [{ type: 'llm-rubric', value: 'foo bar' }],
|
|
},
|
|
];
|
|
const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
|
|
expect(result).toBe(true);
|
|
});
|
|
|
|
it('should emit warning when Azure provider used with non-OpenAI provider', () => {
|
|
const testSuite: TestSuite = {
|
|
providers: [
|
|
new AzureCompletionProvider('foo', { config: { apiHost: 'test.azure.com' } }),
|
|
new HuggingfaceTextGenerationProvider('bar'),
|
|
],
|
|
defaultTest: {},
|
|
prompts: [],
|
|
};
|
|
const tests: TestCase[] = [
|
|
{
|
|
assert: [{ type: 'llm-rubric', value: 'foo bar' }],
|
|
},
|
|
];
|
|
const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
|
|
expect(result).toBe(true);
|
|
});
|
|
|
|
it('should not emit warning when Azure providers are used with a default provider set', () => {
|
|
const testSuite: TestSuite = {
|
|
providers: [new AzureCompletionProvider('foo', { config: { apiHost: 'test.azure.com' } })],
|
|
defaultTest: { options: { provider: 'azureopenai:....' } },
|
|
prompts: [],
|
|
};
|
|
const tests: TestCase[] = [
|
|
{
|
|
assert: [{ type: 'llm-rubric', value: 'foo bar' }],
|
|
},
|
|
];
|
|
const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
|
|
expect(result).toBe(false);
|
|
});
|
|
|
|
it('should not emit warning when both Azure and OpenAI providers are used', () => {
|
|
const testSuite: TestSuite = {
|
|
providers: [
|
|
new AzureCompletionProvider('foo', { config: { apiHost: 'test.azure.com' } }),
|
|
new OpenAiCompletionProvider('bar'),
|
|
],
|
|
defaultTest: {},
|
|
prompts: [],
|
|
};
|
|
const tests: TestCase[] = [
|
|
{
|
|
assert: [{ type: 'llm-rubric', value: 'foo bar' }],
|
|
},
|
|
];
|
|
const result = maybeEmitAzureOpenAiWarning(testSuite, tests);
|
|
expect(result).toBe(false);
|
|
});
|
|
});
|
|
|
|
describe('AzureOpenAiGenericProvider', () => {
|
|
describe('getApiBaseUrl', () => {
|
|
let restoreEnv: () => void;
|
|
|
|
beforeEach(() => {
|
|
restoreEnv = mockProcessEnv({ AZURE_OPENAI_API_HOST: undefined });
|
|
});
|
|
|
|
afterEach(() => {
|
|
restoreEnv();
|
|
});
|
|
|
|
it('should return apiBaseUrl if set', () => {
|
|
const provider = new AzureGenericProvider('test-deployment', {
|
|
config: { apiBaseUrl: 'https://custom.azure.com' },
|
|
});
|
|
expect(provider.getApiBaseUrl()).toBe('https://custom.azure.com');
|
|
});
|
|
|
|
it('should return apiBaseUrl without trailing slash if set', () => {
|
|
const provider = new AzureGenericProvider('test-deployment', {
|
|
config: { apiBaseUrl: 'https://custom.azure.com/' },
|
|
});
|
|
expect(provider.getApiBaseUrl()).toBe('https://custom.azure.com');
|
|
});
|
|
|
|
it('should construct URL from apiHost without protocol', () => {
|
|
const provider = new AzureGenericProvider('test-deployment', {
|
|
config: { apiHost: 'api.azure.com' },
|
|
});
|
|
expect(provider.getApiBaseUrl()).toBe('https://api.azure.com');
|
|
});
|
|
|
|
it('should remove protocol from apiHost if present', () => {
|
|
const provider = new AzureGenericProvider('test-deployment', {
|
|
config: { apiHost: 'https://api.azure.com' },
|
|
});
|
|
expect(provider.getApiBaseUrl()).toBe('https://api.azure.com');
|
|
});
|
|
|
|
it('should remove trailing slash from apiHost if present', () => {
|
|
const provider = new AzureGenericProvider('test-deployment', {
|
|
config: { apiHost: 'api.azure.com/' },
|
|
});
|
|
expect(provider.getApiBaseUrl()).toBe('https://api.azure.com');
|
|
});
|
|
|
|
it('should return undefined if neither apiBaseUrl nor apiHost is set', () => {
|
|
const provider = new AzureGenericProvider('test-deployment', {});
|
|
expect(provider.getApiBaseUrl()).toBeUndefined();
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('AzureOpenAiChatCompletionProvider', () => {
|
|
describe('config merging', () => {
|
|
let provider: AzureChatCompletionProvider;
|
|
|
|
beforeEach(() => {
|
|
provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
apiHost: 'test.azure.com',
|
|
apiKey: 'test-key',
|
|
functions: [{ name: 'provider_func', parameters: {} }],
|
|
max_tokens: 100,
|
|
temperature: 0.5,
|
|
},
|
|
});
|
|
});
|
|
|
|
it('should use provider config when no prompt config exists', async () => {
|
|
const context = {
|
|
prompt: { label: 'test prompt', raw: 'test prompt' },
|
|
vars: {},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body).toMatchObject({
|
|
functions: [{ name: 'provider_func', parameters: {} }],
|
|
max_tokens: 100,
|
|
temperature: 0.5,
|
|
});
|
|
});
|
|
|
|
it('should merge prompt config with provider config', async () => {
|
|
const context = {
|
|
prompt: {
|
|
config: {
|
|
functions: [{ name: 'prompt_func', parameters: {} }],
|
|
temperature: 0.7,
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body).toMatchObject({
|
|
functions: [{ name: 'prompt_func', parameters: {} }],
|
|
max_tokens: 100,
|
|
temperature: 0.7,
|
|
});
|
|
});
|
|
|
|
it('should handle undefined prompt config', async () => {
|
|
const context = {
|
|
prompt: { label: 'test prompt', raw: 'test prompt' },
|
|
vars: {},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body).toMatchObject({
|
|
functions: [{ name: 'provider_func', parameters: {} }],
|
|
max_tokens: 100,
|
|
temperature: 0.5,
|
|
});
|
|
});
|
|
|
|
it('should handle empty prompt config', async () => {
|
|
const context = {
|
|
prompt: { config: {}, label: 'test prompt', raw: 'test prompt' },
|
|
vars: {},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body).toMatchObject({
|
|
functions: [{ name: 'provider_func', parameters: {} }],
|
|
max_tokens: 100,
|
|
temperature: 0.5,
|
|
});
|
|
});
|
|
|
|
it('should handle complex nested config merging', async () => {
|
|
const context = {
|
|
prompt: {
|
|
config: {
|
|
response_format: { type: 'json_object' },
|
|
tool_choice: { function: { name: 'test' }, type: 'function' },
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body).toMatchObject({
|
|
functions: [{ name: 'provider_func', parameters: {} }],
|
|
max_tokens: 100,
|
|
response_format: { type: 'json_object' },
|
|
temperature: 0.5,
|
|
tool_choice: { function: { name: 'test' }, type: 'function' },
|
|
});
|
|
});
|
|
|
|
it('should handle json_schema response format', async () => {
|
|
const context = {
|
|
prompt: {
|
|
config: {
|
|
response_format: {
|
|
type: 'json_schema',
|
|
json_schema: {
|
|
name: 'test_schema',
|
|
strict: true,
|
|
schema: {
|
|
type: 'object',
|
|
properties: {
|
|
test: { type: 'string' },
|
|
},
|
|
required: ['test'],
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body.response_format).toMatchObject({
|
|
type: 'json_schema',
|
|
json_schema: {
|
|
name: 'test_schema',
|
|
strict: true,
|
|
schema: {
|
|
type: 'object',
|
|
properties: {
|
|
test: { type: 'string' },
|
|
},
|
|
required: ['test'],
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
});
|
|
});
|
|
|
|
it('should render variables in response format', async () => {
|
|
const context = {
|
|
prompt: {
|
|
config: {
|
|
response_format: {
|
|
type: 'json_schema',
|
|
json_schema: {
|
|
name: '{{schemaName}}',
|
|
strict: true,
|
|
schema: {
|
|
type: 'object',
|
|
properties: {
|
|
test: { type: 'string' },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {
|
|
schemaName: 'dynamic_schema',
|
|
},
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body.response_format.json_schema.name).toBe('dynamic_schema');
|
|
});
|
|
});
|
|
|
|
describe('response handling', () => {
|
|
let provider: AzureChatCompletionProvider;
|
|
|
|
beforeEach(() => {
|
|
provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
apiHost: 'test.azure.com',
|
|
apiKey: 'test-key',
|
|
},
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.resetAllMocks();
|
|
});
|
|
|
|
it('should parse JSON response with json_schema format when finish_reason is not content_filter', async () => {
|
|
const mockResponse = {
|
|
id: 'mock-id',
|
|
object: 'chat.completion',
|
|
created: Date.now(),
|
|
model: 'gpt-4',
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
message: {
|
|
role: 'assistant',
|
|
content: JSON.stringify({ test: 'value' }),
|
|
},
|
|
finish_reason: 'stop',
|
|
},
|
|
],
|
|
usage: {
|
|
prompt_tokens: 10,
|
|
completion_tokens: 20,
|
|
total_tokens: 30,
|
|
},
|
|
};
|
|
|
|
provider.config.response_format = {
|
|
type: 'json_schema',
|
|
json_schema: {
|
|
name: 'test_response',
|
|
strict: true,
|
|
schema: {
|
|
type: 'object',
|
|
properties: {
|
|
test: { type: 'string' },
|
|
},
|
|
required: ['test'],
|
|
additionalProperties: false,
|
|
},
|
|
},
|
|
};
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: mockResponse,
|
|
cached: false,
|
|
status: 200,
|
|
statusText: 'OK',
|
|
});
|
|
|
|
const result = await provider.callApi('test prompt');
|
|
expect(result.output).toEqual({ test: 'value' });
|
|
});
|
|
|
|
it('should handle API errors', async () => {
|
|
vi.mocked(fetchWithCache).mockRejectedValueOnce(new Error('API Error'));
|
|
|
|
const result = await provider.callApi('test prompt');
|
|
expect(result.error).toBe('API call error: API Error');
|
|
});
|
|
|
|
it('should handle invalid JSON response', async () => {
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: 'invalid json',
|
|
cached: false,
|
|
status: 200,
|
|
statusText: 'OK',
|
|
});
|
|
|
|
const result = await provider.callApi('test prompt');
|
|
expect(result.error).toContain('API returned invalid JSON response');
|
|
});
|
|
|
|
it('should handle tool calls in response', async () => {
|
|
const mockResponse = {
|
|
id: 'mock-id',
|
|
object: 'chat.completion',
|
|
created: Date.now(),
|
|
model: 'gpt-4',
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
message: {
|
|
role: 'assistant',
|
|
tool_calls: [
|
|
{
|
|
type: 'function',
|
|
function: {
|
|
name: 'test',
|
|
arguments: '{}',
|
|
},
|
|
},
|
|
],
|
|
},
|
|
finish_reason: 'stop',
|
|
},
|
|
],
|
|
usage: {
|
|
prompt_tokens: 10,
|
|
completion_tokens: 20,
|
|
total_tokens: 30,
|
|
},
|
|
};
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: mockResponse,
|
|
cached: false,
|
|
status: 200,
|
|
statusText: 'OK',
|
|
});
|
|
|
|
const result = await provider.callApi('test prompt');
|
|
expect(result.output).toEqual([
|
|
{
|
|
type: 'function',
|
|
function: {
|
|
name: 'test',
|
|
arguments: '{}',
|
|
},
|
|
},
|
|
]);
|
|
});
|
|
|
|
it('should handle content filter error response', async () => {
|
|
const mockResponse = {
|
|
error: {
|
|
message:
|
|
"The response was filtered due to the prompt triggering Azure OpenAI's content management policy.",
|
|
code: 'content_filter',
|
|
status: 400,
|
|
innererror: {
|
|
code: 'ResponsibleAIPolicyViolation',
|
|
content_filter_result: {
|
|
hate: { filtered: true, severity: 'medium' },
|
|
jailbreak: { filtered: false, detected: false },
|
|
self_harm: { filtered: false, severity: 'safe' },
|
|
sexual: { filtered: false, severity: 'safe' },
|
|
violence: { filtered: false, severity: 'low' },
|
|
},
|
|
},
|
|
},
|
|
};
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: mockResponse,
|
|
cached: false,
|
|
status: 400,
|
|
statusText: 'Bad Request',
|
|
});
|
|
|
|
const result = await provider.callApi('test prompt');
|
|
expect(result.output).toBe(mockResponse.error.message);
|
|
expect(result.guardrails).toEqual({
|
|
flagged: true,
|
|
flaggedInput: true,
|
|
flaggedOutput: false,
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('structured outputs', () => {
|
|
let provider: AzureChatCompletionProvider;
|
|
|
|
beforeEach(() => {
|
|
vi.clearAllMocks();
|
|
provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
apiHost: 'test.azure.com',
|
|
apiKey: 'test-key',
|
|
},
|
|
});
|
|
});
|
|
|
|
afterEach(() => {
|
|
vi.clearAllMocks();
|
|
vi.restoreAllMocks();
|
|
});
|
|
|
|
it('should parse JSON response when prompt config specifies json_object format', async () => {
|
|
const mockResponse = {
|
|
id: 'mock-id',
|
|
object: 'chat.completion',
|
|
created: Date.now(),
|
|
model: 'gpt-4',
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
message: {
|
|
role: 'assistant',
|
|
content: '{"result": 42, "explanation": "test"}',
|
|
},
|
|
finish_reason: 'stop',
|
|
},
|
|
],
|
|
usage: {
|
|
prompt_tokens: 10,
|
|
completion_tokens: 20,
|
|
total_tokens: 30,
|
|
},
|
|
};
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: mockResponse,
|
|
cached: false,
|
|
status: 200,
|
|
statusText: 'OK',
|
|
});
|
|
|
|
const result = await provider.callApi('test prompt', {
|
|
prompt: {
|
|
config: {
|
|
response_format: { type: 'json_object' },
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {},
|
|
});
|
|
|
|
expect(result.output).toEqual({
|
|
result: 42,
|
|
explanation: 'test',
|
|
});
|
|
});
|
|
|
|
it('should handle invalid JSON when response format is specified', async () => {
|
|
const mockResponse = {
|
|
id: 'mock-id',
|
|
object: 'chat.completion',
|
|
created: Date.now(),
|
|
model: 'gpt-4',
|
|
choices: [
|
|
{
|
|
index: 0,
|
|
message: {
|
|
role: 'assistant',
|
|
content: 'Invalid JSON response',
|
|
},
|
|
finish_reason: 'stop',
|
|
},
|
|
],
|
|
usage: {
|
|
prompt_tokens: 10,
|
|
completion_tokens: 20,
|
|
total_tokens: 30,
|
|
},
|
|
};
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: mockResponse,
|
|
cached: false,
|
|
status: 200,
|
|
statusText: 'OK',
|
|
});
|
|
|
|
const result = await provider.callApi('test prompt', {
|
|
prompt: {
|
|
config: {
|
|
response_format: { type: 'json_object' },
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {},
|
|
});
|
|
|
|
// Should still return the original string if JSON parsing fails
|
|
expect(result.output).toBe('Invalid JSON response');
|
|
});
|
|
|
|
it('should use correct API URL based on datasources config from prompt', async () => {
|
|
const mockResponse = {
|
|
id: 'mock-id',
|
|
choices: [
|
|
{
|
|
message: {
|
|
role: 'assistant',
|
|
content: 'test response',
|
|
},
|
|
},
|
|
],
|
|
usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
|
|
};
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: mockResponse,
|
|
cached: false,
|
|
status: 200,
|
|
statusText: 'OK',
|
|
});
|
|
|
|
await provider.callApi('test prompt', {
|
|
prompt: {
|
|
config: {
|
|
dataSources: [{ type: 'test' }],
|
|
apiVersion: '2024-custom',
|
|
},
|
|
label: 'test prompt',
|
|
raw: 'test prompt',
|
|
},
|
|
vars: {},
|
|
});
|
|
|
|
// Verify the URL includes extensions and uses the custom API version
|
|
expect(vi.mocked(fetchWithCache).mock.calls[0][0]).toContain(
|
|
'/extensions/chat/completions?api-version=2024-custom',
|
|
);
|
|
});
|
|
});
|
|
|
|
describe('reasoning models', () => {
|
|
it('should detect reasoning models with o1 flag', () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
o1: true,
|
|
},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should detect reasoning models with isReasoningModel flag', () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
isReasoningModel: true,
|
|
},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should detect reasoning models with either flag set', () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
o1: false,
|
|
isReasoningModel: true,
|
|
},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect o1 models by deployment name', () => {
|
|
const provider = new AzureChatCompletionProvider('o1-preview', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect o3 models by deployment name', () => {
|
|
const provider = new AzureChatCompletionProvider('o3-mini', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect o4 models by deployment name', () => {
|
|
const provider = new AzureChatCompletionProvider('o4-preview', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it.each([
|
|
'gpt-5.6-sol',
|
|
'gpt-5.6-terra',
|
|
'gpt-5.6-luna',
|
|
])('should auto-detect %s by deployment name', (model) => {
|
|
const provider = new AzureChatCompletionProvider(model, {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should not detect non-reasoning models', () => {
|
|
const provider = new AzureChatCompletionProvider('gpt-4o', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(false);
|
|
});
|
|
|
|
it('should auto-detect reasoning models with mixed case', () => {
|
|
const provider = new AzureChatCompletionProvider('GPT-5.4', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect prefixed o1 deployment names', () => {
|
|
const provider = new AzureChatCompletionProvider('prod-o1-preview', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect prefixed gpt-5 deployment names', () => {
|
|
const provider = new AzureChatCompletionProvider('staging-gpt-5.4', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should not detect non-reasoning models with similar names', () => {
|
|
const provider = new AzureChatCompletionProvider('gpt-4-turbo', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(false);
|
|
});
|
|
|
|
// Third-party reasoning model detection tests
|
|
it('should auto-detect DeepSeek-R1 reasoning models', () => {
|
|
const provider = new AzureChatCompletionProvider('DeepSeek-R1', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect deepseek-r1 with different separators', () => {
|
|
const providerHyphen = new AzureChatCompletionProvider('deepseek-r1-distill', {
|
|
config: {},
|
|
});
|
|
const providerUnderscore = new AzureChatCompletionProvider('deepseek_r1', {
|
|
config: {},
|
|
});
|
|
expect((providerHyphen as any).isReasoningModel()).toBe(true);
|
|
expect((providerUnderscore as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect Phi-4-reasoning models', () => {
|
|
const provider = new AzureChatCompletionProvider('phi-4-reasoning', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect Phi-4-mini-reasoning models', () => {
|
|
const provider = new AzureChatCompletionProvider('Phi-4-mini-reasoning', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect Grok reasoning models', () => {
|
|
const provider = new AzureChatCompletionProvider('grok-3-reasoning', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should auto-detect grok-mini-reasoning models', () => {
|
|
const provider = new AzureChatCompletionProvider('grok-3-mini-reasoning', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(true);
|
|
});
|
|
|
|
it('should not detect regular Grok models as reasoning', () => {
|
|
const provider = new AzureChatCompletionProvider('grok-3', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(false);
|
|
});
|
|
|
|
it('should not detect regular DeepSeek-V3 as reasoning', () => {
|
|
const provider = new AzureChatCompletionProvider('DeepSeek-V3', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(false);
|
|
});
|
|
|
|
it('should not detect regular Phi-4 as reasoning', () => {
|
|
const provider = new AzureChatCompletionProvider('Phi-4', {
|
|
config: {},
|
|
});
|
|
expect((provider as any).isReasoningModel()).toBe(false);
|
|
});
|
|
|
|
it('should use max_completion_tokens for reasoning models', async () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
isReasoningModel: true,
|
|
max_completion_tokens: 2000,
|
|
max_tokens: 1000,
|
|
},
|
|
});
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt');
|
|
expect(body).toHaveProperty('max_completion_tokens', 2000);
|
|
expect(body).not.toHaveProperty('max_tokens');
|
|
});
|
|
|
|
it('should use reasoning_effort for reasoning models', async () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
isReasoningModel: true,
|
|
reasoning_effort: 'high',
|
|
},
|
|
});
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt');
|
|
expect(body).toHaveProperty('reasoning_effort', 'high');
|
|
});
|
|
|
|
it('should not include temperature for reasoning models', async () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
isReasoningModel: true,
|
|
temperature: 0.7,
|
|
},
|
|
});
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt');
|
|
expect(body).not.toHaveProperty('temperature');
|
|
});
|
|
|
|
it('should support variable rendering in reasoning_effort', async () => {
|
|
const provider = new AzureChatCompletionProvider('test-deployment', {
|
|
config: {
|
|
isReasoningModel: true,
|
|
reasoning_effort: '{{effort}}' as any,
|
|
apiHost: 'test.azure.com',
|
|
},
|
|
});
|
|
const context = {
|
|
prompt: { label: 'test prompt', raw: 'test prompt' },
|
|
vars: { effort: 'high' as const },
|
|
};
|
|
const { body } = await (provider as any).getOpenAiBody('test prompt', context);
|
|
expect(body).toHaveProperty('reasoning_effort', 'high');
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('AzureCompletionProvider', () => {
|
|
it('should handle basic completion with caching', async () => {
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: {
|
|
choices: [{ text: 'hello' }],
|
|
usage: { total_tokens: 10, prompt_tokens: 5, completion_tokens: 5 },
|
|
},
|
|
cached: false,
|
|
} as any);
|
|
|
|
vi.mocked(fetchWithCache).mockResolvedValueOnce({
|
|
data: {
|
|
choices: [{ text: 'hello' }],
|
|
usage: { total_tokens: 10 },
|
|
},
|
|
cached: true,
|
|
} as any);
|
|
|
|
const provider = new AzureCompletionProvider('test', {
|
|
config: { apiHost: 'test.azure.com' },
|
|
});
|
|
(provider as any).authHeaders = {};
|
|
|
|
const result1 = await provider.callApi('test prompt');
|
|
const result2 = await provider.callApi('test prompt');
|
|
|
|
expect(result1.output).toBe('hello');
|
|
expect(result2.output).toBe('hello');
|
|
expect(result1.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5 });
|
|
expect(result2.tokenUsage).toEqual({ cached: 10, total: 10 });
|
|
});
|
|
});
|
|
});
|