chore: import upstream snapshot with attribution
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
CI / Shell Format Check (push) Has been cancelled
CI / Check Ruby (3.4) (push) Has been cancelled
CI / CI Config (push) Has been cancelled
CI / Test on Node ${{ matrix.node }} and ${{ matrix.os }}${{ matrix.shard && format(' (shard {0}/3)', matrix.shard) || '' }} (push) Has been cancelled
CI / Build on Node ${{ matrix.node }} (push) Has been cancelled
CI / Style Check (push) Has been cancelled
CI / Generate Assets (push) Has been cancelled
CI / Check Python (3.14) (push) Has been cancelled
CI / Check Python (3.9) (push) Has been cancelled
CI / Build Docs (push) Has been cancelled
CI / Code Scan Action (push) Has been cancelled
CI / Site tests (push) Has been cancelled
CI / webui tests (push) Has been cancelled
CI / Run Integration Tests (push) Has been cancelled
CI / Run Smoke Tests (push) Has been cancelled
CI / Go Tests (push) Has been cancelled
CI / Share Test (push) Has been cancelled
CI / Redteam (Production API) (push) Has been cancelled
CI / Redteam (Staging API) (push) Has been cancelled
CI / GitHub Actions Lint (push) Has been cancelled
CI / Check Ruby (3.0) (push) Has been cancelled
release-please / release-please (push) Has been cancelled
release-please / build (push) Has been cancelled
release-please / publish-npm (push) Has been cancelled
release-please / publish-npm-backfill (push) Has been cancelled
release-please / docker (push) Has been cancelled
release-please / publish-code-scan-action (push) Has been cancelled
release-please / attest-code-scan-action (push) Has been cancelled
Deploy local.promptfoo.app / Deploy to Cloudflare Pages (push) Has been cancelled
Test and Publish Multi-arch Docker Image / test (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-amd64 platform:linux/amd64 runner:ubuntu-latest]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / build-docker-and-push-digests (map[digest-suffix:linux-arm64 platform:linux/arm64 runner:ubuntu-24.04-arm]) (push) Has been cancelled
Test and Publish Multi-arch Docker Image / merge-docker-digests (push) Has been cancelled
Test and Publish Multi-arch Docker Image / Attest Multi-arch Image (push) Has been cancelled
Validate Renovate Config / Validate Renovate Configuration (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,102 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
// Use vi.hoisted() + vi.mock() instead of vi.resetModules() + vi.doMock() + dynamic import.
|
||||
// The old pattern re-imported the entire assertions module (~90 imports) for each test,
|
||||
// which caused timeouts on Windows due to slow module resolution.
|
||||
const mockHandleMeteorAssertion = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock('../../src/assertions/meteor', () => ({
|
||||
handleMeteorAssertion: mockHandleMeteorAssertion,
|
||||
}));
|
||||
|
||||
import { runAssertion } from '../../src/assertions';
|
||||
|
||||
describe('METEOR assertion', () => {
|
||||
beforeEach(() => {
|
||||
mockHandleMeteorAssertion.mockReset();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('should use the handleMeteorAssertion when natural is available', async () => {
|
||||
mockHandleMeteorAssertion.mockResolvedValue({
|
||||
pass: true,
|
||||
score: 0.85,
|
||||
reason: 'METEOR test passed',
|
||||
assertion: { type: 'meteor' },
|
||||
});
|
||||
|
||||
const result = await runAssertion({
|
||||
prompt: 'Test prompt',
|
||||
provider: {} as any,
|
||||
assertion: {
|
||||
type: 'meteor',
|
||||
value: 'Expected output',
|
||||
threshold: 0.7,
|
||||
},
|
||||
test: {} as any,
|
||||
providerResponse: { output: 'Actual output' },
|
||||
});
|
||||
|
||||
// Verify the mock was called and the result is as expected
|
||||
expect(mockHandleMeteorAssertion).toHaveBeenCalledWith(expect.anything());
|
||||
expect(result.pass).toBe(true);
|
||||
expect(result.score).toBe(0.85);
|
||||
expect(result.reason).toBe('METEOR test passed');
|
||||
});
|
||||
|
||||
it('should handle errors when natural package is missing', async () => {
|
||||
// Mock handleMeteorAssertion to throw when called (simulates missing 'natural' module)
|
||||
mockHandleMeteorAssertion.mockImplementation(() => {
|
||||
throw new Error("Cannot find module 'natural'");
|
||||
});
|
||||
|
||||
const result = await runAssertion({
|
||||
prompt: 'Test prompt',
|
||||
provider: {} as any,
|
||||
assertion: {
|
||||
type: 'meteor',
|
||||
value: 'Expected output',
|
||||
threshold: 0.7,
|
||||
},
|
||||
test: {} as any,
|
||||
providerResponse: { output: 'Actual output' },
|
||||
});
|
||||
|
||||
// Verify the error is handled correctly and returns a friendly message
|
||||
expect(result.pass).toBe(false);
|
||||
expect(result.score).toBe(0);
|
||||
expect(result.reason).toBe(
|
||||
'METEOR assertion requires the natural package. Please install it using: npm install natural@^8.1.0',
|
||||
);
|
||||
expect(result.assertion).toEqual({
|
||||
type: 'meteor',
|
||||
value: 'Expected output',
|
||||
threshold: 0.7,
|
||||
});
|
||||
});
|
||||
|
||||
it('should rethrow other errors that are not related to missing module', async () => {
|
||||
// Mock handleMeteorAssertion to throw a non-module-related error
|
||||
mockHandleMeteorAssertion.mockImplementation(() => {
|
||||
throw new Error('Some other error');
|
||||
});
|
||||
|
||||
// The error should be rethrown since it's not a "Cannot find module" error
|
||||
await expect(
|
||||
runAssertion({
|
||||
prompt: 'Test prompt',
|
||||
provider: {} as any,
|
||||
assertion: {
|
||||
type: 'meteor',
|
||||
value: 'Expected output',
|
||||
threshold: 0.7,
|
||||
},
|
||||
test: {} as any,
|
||||
providerResponse: { output: 'Actual output' },
|
||||
}),
|
||||
).rejects.toThrow('Some other error');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user