chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,110 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { getEnvString } from '../../../src/envars';
|
||||
import { getDefaultProviders } from '../../../src/providers/defaults';
|
||||
import { hasGoogleDefaultCredentials } from '../../../src/providers/google/util';
|
||||
import { hasCodexDefaultCredentials } from '../../../src/providers/openai/codexDefaults';
|
||||
|
||||
vi.mock('../../../src/envars');
|
||||
vi.mock('../../../src/logger', () => ({
|
||||
__esModule: true,
|
||||
default: {
|
||||
debug: vi.fn(),
|
||||
info: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
},
|
||||
}));
|
||||
vi.mock('../../../src/providers/google/util', async (importOriginal) => {
|
||||
return {
|
||||
...(await importOriginal()),
|
||||
hasGoogleDefaultCredentials: vi.fn().mockResolvedValue(false),
|
||||
};
|
||||
});
|
||||
vi.mock('../../../src/providers/openai/codexDefaults', async (importOriginal) => {
|
||||
return {
|
||||
...(await importOriginal<typeof import('../../../src/providers/openai/codexDefaults')>()),
|
||||
hasCodexDefaultCredentials: vi.fn().mockReturnValue(false),
|
||||
};
|
||||
});
|
||||
|
||||
const mockedGetEnvString = vi.mocked(getEnvString);
|
||||
const mockedHasCodexDefaultCredentials = vi.mocked(hasCodexDefaultCredentials);
|
||||
const mockedHasGoogleDefaultCredentials = vi.mocked(hasGoogleDefaultCredentials);
|
||||
|
||||
describe('GitHub Models Default Providers', () => {
|
||||
beforeEach(() => {
|
||||
mockedGetEnvString.mockReset();
|
||||
mockedHasCodexDefaultCredentials.mockReset();
|
||||
mockedHasCodexDefaultCredentials.mockReturnValue(false);
|
||||
mockedHasGoogleDefaultCredentials.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it('should use GitHub token for github: model when only GITHUB_TOKEN is available', async () => {
|
||||
// Mock environment where only GITHUB_TOKEN is set
|
||||
mockedGetEnvString.mockImplementation(function (key: string, defaultValue = '') {
|
||||
if (key === 'GITHUB_TOKEN') {
|
||||
return 'test-github-token';
|
||||
}
|
||||
return defaultValue;
|
||||
});
|
||||
|
||||
const providers = await getDefaultProviders();
|
||||
|
||||
// Should use GitHub Models for grading and suggestions
|
||||
expect(providers.gradingProvider.id()).toBe('openai/gpt-5');
|
||||
expect(providers.gradingJsonProvider.id()).toBe('openai/gpt-5');
|
||||
expect(providers.suggestionsProvider.id()).toBe('openai/gpt-5');
|
||||
|
||||
// Should fall back to OpenAI for embeddings and moderation (not supported by GitHub)
|
||||
expect(providers.embeddingProvider.id()).toBe('openai:text-embedding-3-large');
|
||||
expect(providers.moderationProvider.id()).toBe('openai:omni-moderation-latest');
|
||||
});
|
||||
|
||||
it('should prefer OpenAI over GitHub when both tokens are available', async () => {
|
||||
mockedGetEnvString.mockImplementation(function (key: string, defaultValue = '') {
|
||||
if (key === 'OPENAI_API_KEY') {
|
||||
return 'test-openai-key';
|
||||
}
|
||||
if (key === 'GITHUB_TOKEN') {
|
||||
return 'test-github-token';
|
||||
}
|
||||
return defaultValue;
|
||||
});
|
||||
|
||||
const providers = await getDefaultProviders();
|
||||
|
||||
// Should use OpenAI, not GitHub
|
||||
expect(providers.gradingProvider.id()).toBe('openai:gpt-5.5-2026-04-23');
|
||||
});
|
||||
|
||||
it('should prefer Anthropic over GitHub when Anthropic is available', async () => {
|
||||
mockedGetEnvString.mockImplementation(function (key: string, defaultValue = '') {
|
||||
if (key === 'ANTHROPIC_API_KEY') {
|
||||
return 'test-anthropic-key';
|
||||
}
|
||||
if (key === 'GITHUB_TOKEN') {
|
||||
return 'test-github-token';
|
||||
}
|
||||
return defaultValue;
|
||||
});
|
||||
|
||||
const providers = await getDefaultProviders();
|
||||
|
||||
// Should use Anthropic
|
||||
expect(providers.gradingProvider.id()).toContain('claude');
|
||||
});
|
||||
|
||||
it('should use GitHub with env overrides', async () => {
|
||||
mockedGetEnvString.mockImplementation(function (_key: string, defaultValue = '') {
|
||||
return defaultValue;
|
||||
});
|
||||
|
||||
const providers = await getDefaultProviders({
|
||||
GITHUB_TOKEN: 'override-github-token',
|
||||
});
|
||||
|
||||
// Should use GitHub Models
|
||||
expect(providers.gradingProvider.id()).toBe('openai/gpt-5');
|
||||
expect(providers.suggestionsProvider.id()).toBe('openai/gpt-5');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,135 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createGitHubProvider } from '../../../src/providers/github/index';
|
||||
import { OpenAiChatCompletionProvider } from '../../../src/providers/openai/chat';
|
||||
import { createMockProvider } from '../../factories/provider';
|
||||
|
||||
const mockOpenAiChatCompletionProvider = vi.hoisted(() =>
|
||||
vi.fn(function (this: unknown) {
|
||||
return {};
|
||||
}),
|
||||
);
|
||||
|
||||
vi.mock('../../../src/providers/openai/chat', async (importOriginal) => {
|
||||
return {
|
||||
...(await importOriginal()),
|
||||
OpenAiChatCompletionProvider: mockOpenAiChatCompletionProvider,
|
||||
};
|
||||
});
|
||||
|
||||
describe('createGitHubProvider', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mockOpenAiChatCompletionProvider.mockReset();
|
||||
});
|
||||
|
||||
it('should create provider with default model when no model specified', () => {
|
||||
const mockProvider = createMockProvider({ id: 'github:openai/gpt-5' });
|
||||
mockOpenAiChatCompletionProvider.mockImplementation(function () {
|
||||
return mockProvider as any;
|
||||
});
|
||||
|
||||
const result = createGitHubProvider('github:', {}, {} as any);
|
||||
|
||||
expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('openai/gpt-5', {
|
||||
config: {
|
||||
apiBaseUrl: 'https://models.github.ai/inference',
|
||||
apiKeyEnvar: 'GITHUB_TOKEN',
|
||||
},
|
||||
});
|
||||
expect(result).toBe(mockProvider);
|
||||
});
|
||||
|
||||
it('should create provider with specified model', () => {
|
||||
const mockProvider = createMockProvider({ id: 'github:openai/gpt-4.1-mini' });
|
||||
mockOpenAiChatCompletionProvider.mockImplementation(function () {
|
||||
return mockProvider as any;
|
||||
});
|
||||
|
||||
const result = createGitHubProvider('github:openai/gpt-4.1-mini', {}, {} as any);
|
||||
|
||||
expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('openai/gpt-4.1-mini', {
|
||||
config: {
|
||||
apiBaseUrl: 'https://models.github.ai/inference',
|
||||
apiKeyEnvar: 'GITHUB_TOKEN',
|
||||
},
|
||||
});
|
||||
expect(result).toBe(mockProvider);
|
||||
});
|
||||
|
||||
it('should handle models with colons in their names', () => {
|
||||
const mockProvider = createMockProvider({ id: 'github:anthropic/claude-3.5:sonnet' });
|
||||
mockOpenAiChatCompletionProvider.mockImplementation(function () {
|
||||
return mockProvider as any;
|
||||
});
|
||||
|
||||
const result = createGitHubProvider('github:anthropic/claude-3.5:sonnet', {}, {} as any);
|
||||
|
||||
expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('anthropic/claude-3.5:sonnet', {
|
||||
config: {
|
||||
apiBaseUrl: 'https://models.github.ai/inference',
|
||||
apiKeyEnvar: 'GITHUB_TOKEN',
|
||||
},
|
||||
});
|
||||
expect(result).toBe(mockProvider);
|
||||
});
|
||||
|
||||
it('should merge existing config with GitHub-specific config', () => {
|
||||
const mockProvider = createMockProvider({ id: 'github:openai/gpt-4.1' });
|
||||
mockOpenAiChatCompletionProvider.mockImplementation(function () {
|
||||
return mockProvider as any;
|
||||
});
|
||||
|
||||
const existingConfig = {
|
||||
temperature: 0.7,
|
||||
max_tokens: 2048,
|
||||
apiKey: 'custom-key',
|
||||
};
|
||||
|
||||
const result = createGitHubProvider(
|
||||
'github:openai/gpt-4.1',
|
||||
{ config: existingConfig },
|
||||
{} as any,
|
||||
);
|
||||
|
||||
expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('openai/gpt-4.1', {
|
||||
config: {
|
||||
temperature: 0.7,
|
||||
max_tokens: 2048,
|
||||
apiKey: 'custom-key',
|
||||
apiBaseUrl: 'https://models.github.ai/inference',
|
||||
apiKeyEnvar: 'GITHUB_TOKEN',
|
||||
},
|
||||
});
|
||||
expect(result).toBe(mockProvider);
|
||||
});
|
||||
|
||||
it('should pass through all provider options', () => {
|
||||
const mockProvider = createMockProvider({ id: 'github:openai/gpt-4.1' });
|
||||
mockOpenAiChatCompletionProvider.mockImplementation(function () {
|
||||
return mockProvider as any;
|
||||
});
|
||||
|
||||
const providerOptions = {
|
||||
id: 'custom-id',
|
||||
label: 'Custom Label',
|
||||
config: { temperature: 0.5 },
|
||||
transform: 'output.trim()',
|
||||
delay: 1000,
|
||||
};
|
||||
|
||||
const result = createGitHubProvider('github:openai/gpt-4.1', providerOptions, {} as any);
|
||||
|
||||
expect(OpenAiChatCompletionProvider).toHaveBeenCalledWith('openai/gpt-4.1', {
|
||||
id: 'custom-id',
|
||||
label: 'Custom Label',
|
||||
transform: 'output.trim()',
|
||||
delay: 1000,
|
||||
config: {
|
||||
temperature: 0.5,
|
||||
apiBaseUrl: 'https://models.github.ai/inference',
|
||||
apiKeyEnvar: 'GITHUB_TOKEN',
|
||||
},
|
||||
});
|
||||
expect(result).toBe(mockProvider);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { loadApiProvider } from '../../../src/providers/index';
|
||||
|
||||
import type { ApiProvider } from '../../../src/types/providers';
|
||||
|
||||
describe('GitHub Models Provider Integration', () => {
|
||||
it('should load GitHub provider with default model', async () => {
|
||||
const provider = (await loadApiProvider('github:', {
|
||||
basePath: process.cwd(),
|
||||
})) as ApiProvider;
|
||||
|
||||
expect(provider).toBeDefined();
|
||||
expect(provider.id()).toBe('openai/gpt-5'); // Default model
|
||||
});
|
||||
|
||||
it('should load GitHub provider with specific model', async () => {
|
||||
const provider = (await loadApiProvider('github:openai/gpt-4o-mini', {
|
||||
basePath: process.cwd(),
|
||||
})) as ApiProvider;
|
||||
|
||||
expect(provider).toBeDefined();
|
||||
expect(provider.id()).toBe('openai/gpt-4o-mini');
|
||||
});
|
||||
|
||||
it('should load GitHub provider with azureml models', async () => {
|
||||
const provider = (await loadApiProvider('github:azureml/Phi-4', {
|
||||
basePath: process.cwd(),
|
||||
})) as ApiProvider;
|
||||
|
||||
expect(provider).toBeDefined();
|
||||
expect(provider.id()).toBe('azureml/Phi-4');
|
||||
});
|
||||
|
||||
it('should set correct configuration', async () => {
|
||||
const provider = (await loadApiProvider('github:openai/gpt-4o', {
|
||||
basePath: process.cwd(),
|
||||
options: {
|
||||
config: {
|
||||
temperature: 0.7,
|
||||
max_tokens: 1000,
|
||||
},
|
||||
},
|
||||
})) as ApiProvider;
|
||||
|
||||
expect(provider).toBeDefined();
|
||||
|
||||
// Check that it's using OpenAI provider under the hood
|
||||
const providerClass = provider.constructor.name;
|
||||
expect(providerClass).toBe('OpenAiChatCompletionProvider');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user