import * as fs from 'fs';
import * as path from 'path';

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runAssertion } from '../../src/assertions/index';
import { OpenAiChatCompletionProvider } from '../../src/providers/openai/chat';
import { DefaultEmbeddingProvider } from '../../src/providers/openai/defaults';
import { fetchWithRetries } from '../../src/util/fetch/index';
import { createMockProvider } from '../factories/provider';
import { TestGrader } from '../util/utils';

import type { ApiProvider, Assertion, AtomicTestCase, GradingResult } from '../../src/types/index';

vi.mock('../../src/redteam/remoteGeneration', () => ({
  shouldGenerateRemote: vi.fn().mockReturnValue(false),
}));

// Causes a SIGSEGV in github actions.
vi.mock('libsql');

vi.mock('proxy-agent', () => ({
  ProxyAgent: vi.fn().mockImplementation(() => ({})),
}));

vi.mock('node:module', () => {
  const mockRequire: NodeJS.Require = {
    resolve: vi.fn() as unknown as NodeJS.RequireResolve,
  } as unknown as NodeJS.Require;
  return {
    createRequire: vi.fn().mockReturnValue(mockRequire),
  };
});

vi.mock('../../src/util/fetch/index.ts', async () => {
  const actual = await vi.importActual<typeof import('../../src/util/fetch/index')>(
    '../../src/util/fetch/index.ts',
  );
  return {
    ...actual,
    fetchWithRetries: vi.fn(actual.fetchWithRetries),
  };
});

vi.mock('glob', () => ({
  globSync: vi.fn(),
}));

vi.mock('fs', () => {
  const fsMock = {
    readFileSync: vi.fn(),
    existsSync: vi.fn(),
    writeFileSync: vi.fn(),
    mkdirSync: vi.fn(),
    promises: {
      readFile: vi.fn(),
    },
  };
  return {
    ...fsMock,
    default: fsMock,
  };
});

vi.mock('../../src/esm', () => ({
  getDirectory: () => '/test/dir',
  importModule: vi.fn((_filePath: string, functionName?: string) => {
    return Promise.resolve(functionName ? {} : undefined);
  }),
}));
vi.mock('../../src/database', () => ({
  getDb: vi.fn(),
}));
vi.mock('path', async () => {
  const actual = await vi.importActual<typeof import('path')>('path');
  const mocked = {
    ...actual,
    resolve: vi.fn(),
    extname: vi.fn(),
  };
  return {
    ...mocked,
    default: mocked,
  };
});

vi.mock('../../src/cliState', () => ({
  default: {
    basePath: '/base/path',
  },
  basePath: '/base/path',
}));
vi.mock('../../src/matchers/rag', async () => {
  const actual =
    await vi.importActual<typeof import('../../src/matchers/rag')>('../../src/matchers/rag');
  return {
    ...actual,
    matchesContextRelevance: vi
      .fn()
      .mockResolvedValue({ pass: true, score: 1, reason: 'Mocked reason' }),
    matchesContextFaithfulness: vi
      .fn()
      .mockResolvedValue({ pass: true, score: 1, reason: 'Mocked reason' }),
  };
});

const Grader = new TestGrader();

describe('runAssertion', () => {
  afterEach(() => {
    vi.clearAllMocks();
  });

  const equalityAssertion: Assertion = {
    type: 'equals',
    value: 'Expected output',
  };

  const equalityAssertionWithObject: Assertion = {
    type: 'equals',
    value: { key: 'value' },
  };

  const isJsonAssertion: Assertion = {
    type: 'is-json',
  };

  const isJsonAssertionWithSchema: Assertion = {
    type: 'is-json',
    value: {
      required: ['latitude', 'longitude'],
      type: 'object',
      properties: {
        latitude: {
          type: 'number',
          minimum: -90,
          maximum: 90,
        },
        longitude: {
          type: 'number',
          minimum: -180,
          maximum: 180,
        },
      },
    },
  };

  const isJsonAssertionWithSchemaYamlString: Assertion = {
    type: 'is-json',
    value: `
          required: ["latitude", "longitude"]
          type: object
          properties:
            latitude:
              type: number
              minimum: -90
              maximum: 90
            longitude:
              type: number
              minimum: -180
              maximum: 180
`,
  };

  const isSqlAssertion: Assertion = {
    type: 'is-sql',
  };

  const notIsSqlAssertion: Assertion = {
    type: 'not-is-sql',
  };

  const isSqlAssertionWithDatabase: Assertion = {
    type: 'is-sql',
    value: {
      databaseType: 'MySQL',
    },
  };

  const isSqlAssertionWithDatabaseAndWhiteTableList: Assertion = {
    type: 'is-sql',
    value: {
      databaseType: 'MySQL',
      allowedTables: ['(select|update|insert|delete)::null::departments'],
    },
  };

  const isSqlAssertionWithDatabaseAndWhiteColumnList: Assertion = {
    type: 'is-sql',
    value: {
      databaseType: 'MySQL',
      allowedColumns: ['select::null::name', 'update::null::id'],
    },
  };

  const isSqlAssertionWithDatabaseAndBothList: Assertion = {
    type: 'is-sql',
    value: {
      databaseType: 'MySQL',
      allowedTables: ['(select|update|insert|delete)::null::departments'],
      allowedColumns: ['select::null::name', 'update::null::id'],
    },
  };

  const containsJsonAssertion: Assertion = {
    type: 'contains-json',
  };

  const containsJsonAssertionWithSchema: Assertion = {
    type: 'contains-json',
    value: {
      required: ['latitude', 'longitude'],
      type: 'object',
      properties: {
        latitude: {
          type: 'number',
          minimum: -90,
          maximum: 90,
        },
        longitude: {
          type: 'number',
          minimum: -180,
          maximum: 180,
        },
      },
    },
  };

  const javascriptStringAssertion: Assertion = {
    type: 'javascript',
    value: 'output === "Expected output"',
  };

  const javascriptStringAssertionWithNumber: Assertion = {
    type: 'javascript',
    value: 'output.length * 10',
  };

  const javascriptBooleanAssertionWithConfig: Assertion = {
    type: 'javascript',
    value: 'output.length <= context.config.maximumOutputSize',
    config: {
      maximumOutputSize: 20,
    },
  };

  const javascriptStringAssertionWithNumberAndThreshold: Assertion = {
    type: 'javascript',
    value: 'output.length * 10',
    threshold: 0.5,
  };

  it('should pass when the equality assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: equalityAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the equality assertion fails', async () => {
    const output = 'Actual output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: equalityAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output "Actual output" to equal "Expected output"',
    });
  });

  const notEqualsAssertion: Assertion = {
    type: 'not-equals',
    value: 'Unexpected output',
  };

  it('should pass when the not-equals assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notEqualsAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the not-equals assertion fails', async () => {
    const output = 'Unexpected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notEqualsAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output "Unexpected output" to not equal "Unexpected output"',
    });
  });

  it('should handle output as an object', async () => {
    const output = { key: 'value' };
    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: equalityAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output "{"key":"value"}" to equal "Expected output"',
    });
  });

  it('should pass when the equality assertion with object passes', async () => {
    const output = { key: 'value' };

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: equalityAssertionWithObject,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the equality assertion with object fails', async () => {
    const output = { key: 'not value' };

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: equalityAssertionWithObject,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output "{"key":"not value"}" to equal "{"key":"value"}"',
    });
  });

  it('should pass when the equality assertion with object passes with external json', async () => {
    const assertion: Assertion = {
      type: 'equals',
      value: 'file:///output.json',
    };

    vi.mocked(path.resolve).mockReturnValue('/base/path/output.json');
    vi.mocked(path.extname).mockReturnValue('.json');
    vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ key: 'value' }));

    const output = '{"key":"value"}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/output.json', 'utf8');
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the equality assertion with object fails with external object', async () => {
    const assertion: Assertion = {
      type: 'equals',
      value: 'file:///output.json',
    };

    vi.mocked(path.resolve).mockReturnValue('/base/path/output.json');
    vi.mocked(path.extname).mockReturnValue('.json');
    vi.mocked(fs.readFileSync).mockReturnValue(JSON.stringify({ key: 'value' }));

    const output = '{"key":"not value"}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/output.json', 'utf8');
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output "{"key":"not value"}" to equal "{"key":"value"}"',
    });
  });

  it('should pass when the is-json assertion passes', async () => {
    const output = '{"key":"value"}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isJsonAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-json assertion fails', async () => {
    const output = 'Not valid JSON';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isJsonAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to be valid JSON',
    });
  });

  it('should pass when the is-json assertion passes with schema', async () => {
    const output = '{"latitude": 80.123, "longitude": -1}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isJsonAssertionWithSchema,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-json assertion fails with schema', async () => {
    const output = '{"latitude": "high", "longitude": [-1]}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isJsonAssertionWithSchema,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
    });
  });

  it('should pass when the is-json assertion passes with schema YAML string', async () => {
    const output = '{"latitude": 80.123, "longitude": -1}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isJsonAssertionWithSchemaYamlString,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-json assertion fails with schema YAML string', async () => {
    const output = '{"latitude": "high", "longitude": [-1]}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isJsonAssertionWithSchemaYamlString,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
    });
  });

  it('should fail when the not-is-json assertion finds matching schema', async () => {
    const output = '{"latitude": 80.123, "longitude": -1}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'not-is-json',
        value: isJsonAssertionWithSchema.value,
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      score: 0,
      reason: 'Output is JSON that conforms to the provided schema',
    });
  });

  it('should pass when the not-is-json assertion finds non-matching schema', async () => {
    const output = '{"latitude": "high", "longitude": [-1]}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'not-is-json',
        value: isJsonAssertionWithSchema.value,
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: 1,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the not-is-json assertion with schema receives non-JSON input', async () => {
    const output = 'this is not json at all';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'not-is-json',
        value: isJsonAssertionWithSchema.value,
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: 1,
      reason: 'Assertion passed',
    });
  });

  it('should validate JSON with formats using ajv-formats', async () => {
    const output = '{"date": "2021-08-29"}';
    const schemaWithFormat = {
      type: 'object',
      properties: {
        date: {
          type: 'string',
          format: 'date',
        },
      },
      required: ['date'],
    };

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: { type: 'is-json', value: schemaWithFormat },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });

    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should validate JSON with formats using ajv-formats - failure', async () => {
    const output = '{"date": "not a date"}';
    const schemaWithFormat = {
      type: 'object',
      properties: {
        date: {
          type: 'string',
          format: 'date',
        },
      },
      required: ['date'],
    };

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: { type: 'is-json', value: schemaWithFormat },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });

    expect(result).toMatchObject({
      pass: false,
      reason:
        'JSON does not conform to the provided schema. Errors: data/date must match format "date"',
    });
  });

  it('should pass when the is-json assertion passes with external schema', async () => {
    const assertion: Assertion = {
      type: 'is-json',
      value: 'file:///schema.json',
    };

    vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
    vi.mocked(path.extname).mockReturnValue('.json');
    vi.mocked(fs.readFileSync).mockReturnValue(
      JSON.stringify({
        required: ['latitude', 'longitude'],
        type: 'object',
        properties: {
          latitude: {
            type: 'number',
            minimum: -90,
            maximum: 90,
          },
          longitude: {
            type: 'number',
            minimum: -180,
            maximum: 180,
          },
        },
      }),
    );

    const output = '{"latitude": 80.123, "longitude": -1}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-json assertion fails with external schema', async () => {
    const assertion: Assertion = {
      type: 'is-json',
      value: 'file:///schema.json',
    };

    vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
    vi.mocked(path.extname).mockReturnValue('.json');
    vi.mocked(fs.readFileSync).mockReturnValue(
      JSON.stringify({
        required: ['latitude', 'longitude'],
        type: 'object',
        properties: {
          latitude: {
            type: 'number',
            minimum: -90,
            maximum: 90,
          },
          longitude: {
            type: 'number',
            minimum: -180,
            maximum: 180,
          },
        },
      }),
    );

    const output = '{"latitude": "high", "longitude": [-1]}';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
    expect(result).toMatchObject({
      pass: false,
      reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
    });
  });

  it('should pass when the is-sql assertion passes', async () => {
    const output = 'SELECT id, name FROM users';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-sql assertion fails', async () => {
    const output = 'SELECT * FROM orders ORDERY BY order_date';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'SQL statement does not conform to the provided MySQL database syntax.',
    });
  });

  it('should pass when the not-is-sql assertion passes', async () => {
    const output = 'SELECT * FROM orders ORDERY BY order_date';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: notIsSqlAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the not-is-sql assertion fails', async () => {
    const output = 'SELECT id, name FROM users';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: notIsSqlAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'The output SQL statement is valid',
    });
  });

  it('should pass when the is-sql assertion passes given MySQL Database syntax', async () => {
    const output = 'SELECT id, name FROM users';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabase,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-sql assertion fails given MySQL Database syntax', async () => {
    const output = `SELECT first_name, last_name FROM employees WHERE first_name ILIKE 'john%'`;

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabase,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'SQL statement does not conform to the provided MySQL database syntax.',
    });
  });

  it('should pass when the is-sql assertion passes given MySQL Database syntax and allowedTables', async () => {
    const output = 'SELECT * FROM departments WHERE department_id = 1';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-sql assertion fails given MySQL Database syntax and allowedTables', async () => {
    const output = 'UPDATE employees SET department_id = 2 WHERE employee_id = 1';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndWhiteTableList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: `SQL references unauthorized table(s). Found: [update::null::employees]. Allowed: [(select|update|insert|delete)::null::departments].`,
    });
  });

  it('should pass when the is-sql assertion passes given MySQL Database syntax and allowedColumns', async () => {
    const output = 'SELECT name FROM t';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-sql assertion fails given MySQL Database syntax and allowedColumns', async () => {
    const output = 'SELECT age FROM a WHERE id = 1';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndWhiteColumnList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: `SQL references unauthorized column(s). Found: [select::null::age, select::null::id]. Allowed: [select::null::name, update::null::id].`,
    });
  });

  it('should pass when the is-sql assertion passes given MySQL Database syntax, allowedTables, and allowedColumns', async () => {
    const output = 'SELECT name FROM departments';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndBothList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the is-sql assertion fails given MySQL Database syntax, allowedTables, and allowedColumns', async () => {
    const output = `INSERT INTO departments (name) VALUES ('HR')`;

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndBothList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: `SQL references unauthorized column(s). Found: [insert::departments::name]. Allowed: [select::null::name, update::null::id].`,
    });
  });

  it('should fail when the is-sql assertion fails due to missing table authority for MySQL Database syntax', async () => {
    const output = 'UPDATE a SET id = 1';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndBothList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: `SQL references unauthorized table(s). Found: [update::null::a]. Allowed: [(select|update|insert|delete)::null::departments].`,
    });
  });

  it('should fail when the is-sql assertion fails due to missing authorities for DELETE statement in MySQL Database syntax', async () => {
    const output = `DELETE FROM employees;`;

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: isSqlAssertionWithDatabaseAndBothList,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: `SQL references unauthorized table(s). Found: [delete::null::employees]. Allowed: [(select|update|insert|delete)::null::departments]. SQL references unauthorized column(s). Found: [delete::employees::(.*)]. Allowed: [select::null::name, update::null::id].`,
    });
  });

  it('should pass when the contains-sql assertion passes', async () => {
    const output = 'wassup\n```\nSELECT id, name FROM users\n```\nyolo';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'contains-sql',
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the contains-sql assertion sees `sql` in code block', async () => {
    const output = 'wassup\n```sql\nSELECT id, name FROM users\n```\nyolo';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'contains-sql',
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the contains-sql assertion sees sql without code block', async () => {
    const output = 'SELECT id, name FROM users';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'contains-sql',
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the contains-sql does not contain code block', async () => {
    const output = 'nothin';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'contains-sql',
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
    });
  });

  it('should fail when the contains-sql does not contain sql in code block', async () => {
    const output = '```python\nprint("Hello, World!")\n```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'contains-sql',
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
    });
  });

  it('should pass when the contains-json assertion passes', async () => {
    const output =
      'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} \n\n blah blah';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: containsJsonAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the contains-json assertion passes with multiple json values', async () => {
    const output =
      'this is some other stuff \n\n {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}} another {"key": "value", "key2": {"key3": "value2", "key4": ["value3", "value4"]}}\n\n blah blah';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: containsJsonAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the contains-json assertion fails', async () => {
    const output = 'Not valid JSON';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: containsJsonAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to contain valid JSON',
    });
  });

  it('should pass when the contains-json assertion passes with schema', async () => {
    const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: containsJsonAssertionWithSchema,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the contains-json assertion passes with schema with YAML string', async () => {
    const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: containsJsonAssertionWithSchema,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the contains-json assertion passes with external schema', async () => {
    const assertion: Assertion = {
      type: 'contains-json',
      value: 'file:///schema.json',
    };

    vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
    vi.mocked(path.extname).mockReturnValue('.json');
    vi.mocked(fs.readFileSync).mockReturnValue(
      JSON.stringify({
        required: ['latitude', 'longitude'],
        type: 'object',
        properties: {
          latitude: {
            type: 'number',
            minimum: -90,
            maximum: 90,
          },
          longitude: {
            type: 'number',
            minimum: -180,
            maximum: 180,
          },
        },
      }),
    );

    const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail contains-json assertion with invalid data against external schema', async () => {
    const assertion: Assertion = {
      type: 'contains-json',
      value: 'file:///schema.json',
    };

    vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
    vi.mocked(path.extname).mockReturnValue('.json');
    vi.mocked(fs.readFileSync).mockReturnValue(
      JSON.stringify({
        required: ['latitude', 'longitude'],
        type: 'object',
        properties: {
          latitude: {
            type: 'number',
            minimum: -90,
            maximum: 90,
          },
          longitude: {
            type: 'number',
            minimum: -180,
            maximum: 180,
          },
        },
      }),
    );

    const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
    expect(result).toMatchObject({
      pass: false,
      reason: 'JSON does not conform to the provided schema. Errors: data/latitude must be number',
    });
  });

  it('should fail contains-json assertion with predefined schema and invalid data', async () => {
    const output = 'here is the answer\n\n```{"latitude": "medium", "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: containsJsonAssertionWithSchema,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toEqual(
      expect.objectContaining({
        pass: false,
        reason:
          'JSON does not conform to the provided schema. Errors: data/latitude must be number',
      }),
    );
  });

  it('should fail when the not-contains-json assertion finds matching schema', async () => {
    const output = 'here is the answer\n\n```{"latitude": 80.123, "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'not-contains-json',
        value: containsJsonAssertionWithSchema.value,
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      score: 0,
      reason: 'Output contains JSON conforming to the provided schema',
    });
  });

  it('should pass when the not-contains-json assertion finds no matching schema', async () => {
    const output = 'here is the answer\n\n```{"latitude": "not a number", "longitude": -1}```';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'not-contains-json',
        value: containsJsonAssertionWithSchema.value,
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: 1,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the not-contains-json assertion with schema finds no JSON at all', async () => {
    const output = 'here is the answer with no JSON whatsoever';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: {
        type: 'not-contains-json',
        value: containsJsonAssertionWithSchema.value,
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: 1,
      reason: 'Assertion passed',
    });
  });

  it('should pass when the javascript assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptStringAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should pass a score through when the javascript returns a number', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptStringAssertionWithNumber,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: output.length * 10,
      reason: 'Assertion passed',
    });
  });

  it('should pass when javascript returns an output string that is smaller than the maximum size threshold', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptBooleanAssertionWithConfig,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: 1.0,
      reason: 'Assertion passed',
    });
  });

  it('should disregard invalid inputs for assert index', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptBooleanAssertionWithConfig,
      test: {
        assert: [
          {
            type: 'javascript',
            value: 'output.length <= context.config.maximumOutputSize',
            config: {
              maximumOutputSize: 1,
            },
          } as Assertion,
        ],
      } as AtomicTestCase,
      providerResponse: { output },
      assertIndex: 45,
    });
    expect(result).toMatchObject({
      pass: true,
      score: 1.0,
      reason: 'Assertion passed',
    });
  });

  it('should fail when javascript returns an output string that is larger than the maximum size threshold', async () => {
    const output = 'Expected output with some extra characters';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptBooleanAssertionWithConfig,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      score: 0,
      reason: expect.stringContaining('Custom function returned false'),
    });
  });

  it('should pass when javascript returns a number above threshold', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptStringAssertionWithNumberAndThreshold,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      score: output.length * 10,
      reason: 'Assertion passed',
    });
  });

  it('should fail when javascript returns a number below threshold', async () => {
    const output = '';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptStringAssertionWithNumberAndThreshold,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      score: output.length * 10,
      reason: expect.stringContaining('Custom function returned false'),
    });
  });

  it('should set score when javascript returns false', async () => {
    const output = 'Test output';

    const assertion: Assertion = {
      type: 'javascript',
      value: 'output.length < 1',
    };

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      score: 0,
      reason: expect.stringContaining('Custom function returned false'),
    });
  });

  it('should fail when the javascript assertion fails', async () => {
    const output = 'Different output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: javascriptStringAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Custom function returned false\noutput === "Expected output"',
    });
  });

  it('should pass when assertion passes - with vars', async () => {
    const output = 'Expected output';

    const assertion: Assertion = {
      type: 'equals',
      value: '{{ foo }}',
    };
    const result: GradingResult = await runAssertion({
      prompt: 'variable value',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion,
      test: { vars: { foo: 'Expected output' } } as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  const notContainsAssertion: Assertion = {
    type: 'not-contains',
    value: 'Unexpected output',
  };

  it('should pass when the not-contains assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notContainsAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the not-contains assertion fails', async () => {
    const output = 'Unexpected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notContainsAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to not contain "Unexpected output"',
    });
  });

  // Test for icontains assertion
  const containsLowerAssertion: Assertion = {
    type: 'icontains',
    value: 'expected output',
  };

  it('should pass when the icontains assertion passes', async () => {
    const output = 'EXPECTED OUTPUT';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsLowerAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the icontains assertion fails', async () => {
    const output = 'Different output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsLowerAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to contain "expected output"',
    });
  });

  // Test for not-icontains assertion
  const notContainsLowerAssertion: Assertion = {
    type: 'not-icontains',
    value: 'unexpected output',
  };

  it('should pass when the not-icontains assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notContainsLowerAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the not-icontains assertion fails', async () => {
    const output = 'UNEXPECTED OUTPUT';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notContainsLowerAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to not contain "unexpected output"',
    });
  });

  // Test for contains-any assertion
  const containsAnyAssertion: Assertion = {
    type: 'contains-any',
    value: ['option1', 'option2', 'option3'],
  };

  it('should pass when the contains-any assertion passes', async () => {
    const output = 'This output contains option1';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsAnyAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the contains-any assertion fails', async () => {
    const output = 'This output does not contain any option';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsAnyAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to contain one of "option1, option2, option3"',
    });
  });

  it('should pass when the icontains-any assertion passes', async () => {
    const output = 'This output contains OPTION1';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: {
        type: 'icontains-any',
        value: ['option1', 'option2', 'option3'],
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the icontains-any assertion fails', async () => {
    const output = 'This output does not contain any option';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: {
        type: 'icontains-any',
        value: ['option1', 'option2', 'option3'],
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to contain one of "option1, option2, option3"',
    });
  });

  // Test for contains-all assertion
  const containsAllAssertion: Assertion = {
    type: 'contains-all',
    value: ['option1', 'option2', 'option3'],
  };

  it('should pass when the contains-all assertion passes', async () => {
    const output = 'This output contains option1, option2, and option3';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsAllAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the contains-all assertion fails', async () => {
    const output = 'This output contains only option1 and option2';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsAllAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to contain all of [option1, option2, option3]. Missing: [option3]',
    });
  });

  it('should pass when the icontains-all assertion passes', async () => {
    const output = 'This output contains OPTION1, option2, and opTiOn3';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: {
        type: 'icontains-all',
        value: ['option1', 'option2', 'option3'],
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the icontains-all assertion fails', async () => {
    const output = 'This output contains OPTION1, option2, and opTiOn3';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: {
        type: 'icontains-all',
        value: ['option1', 'option2', 'option3', 'option4'],
      },
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason:
        'Expected output to contain all of [option1, option2, option3, option4]. Missing: [option4]',
    });
  });

  // Test for regex assertion
  const containsRegexAssertion: Assertion = {
    type: 'regex',
    value: '\\d{3}-\\d{2}-\\d{4}',
  };

  it('should pass when the regex assertion passes', async () => {
    const output = 'This output contains 123-45-6789';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsRegexAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the regex assertion fails', async () => {
    const output = 'This output does not contain the pattern';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: containsRegexAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to match regex "\\d{3}-\\d{2}-\\d{4}"',
    });
  });

  // Test for not-regex assertion
  const notContainsRegexAssertion: Assertion = {
    type: 'not-regex',
    value: '\\d{3}-\\d{2}-\\d{4}',
  };

  it('should pass when the not-regex assertion passes', async () => {
    const output = 'This output does not contain the pattern';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notContainsRegexAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the not-regex assertion fails', async () => {
    const output = 'This output contains 123-45-6789';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notContainsRegexAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to not match regex "\\d{3}-\\d{2}-\\d{4}"',
    });
  });

  // Tests for webhook assertion
  const webhookAssertion: Assertion = {
    type: 'webhook',
    value: 'https://example.com/webhook',
  };

  it('should pass when the webhook assertion passes', async () => {
    const output = 'Expected output';

    vi.mocked(fetchWithRetries).mockImplementation(() =>
      Promise.resolve(
        new Response(JSON.stringify({ pass: true }), {
          status: 200,
          headers: { 'Content-Type': 'application/json' },
        }),
      ),
    );

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: webhookAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the webhook assertion fails', async () => {
    const output = 'Different output';
    vi.mocked(fetchWithRetries).mockImplementation(() =>
      Promise.resolve(
        new Response(JSON.stringify({ pass: false }), {
          status: 200,
          headers: { 'Content-Type': 'application/json' },
        }),
      ),
    );

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: webhookAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Webhook returned false',
    });
  });

  it('should fail when the webhook returns an error', async () => {
    const output = 'Expected output';

    vi.mocked(fetchWithRetries).mockImplementation(() =>
      Promise.resolve(
        new Response('', {
          status: 500,
          headers: { 'Content-Type': 'application/json' },
        }),
      ),
    );

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: webhookAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Webhook error: Webhook response status: 500',
    });
  });

  // Test for rouge-n assertion
  const rougeNAssertion: Assertion = {
    type: 'rouge-n',
    value: 'This is the expected output.',
    threshold: 0.75,
  };
  const notRougeNAssertion: Assertion = {
    type: 'not-rouge-n',
    value: 'This is the expected output.',
    threshold: 0.75,
  };

  it('should pass when the rouge-n assertion passes', async () => {
    const output = 'This is the expected output.';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: rougeNAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'ROUGE-N score 1.00 is greater than or equal to threshold 0.75',
    });
  });

  it('should fail when the rouge-n assertion fails', async () => {
    const output = 'some different output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: rougeNAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'ROUGE-N score 0.22 is less than threshold 0.75',
    });
  });

  it('should pass when the not-rouge-n assertion score is below threshold', async () => {
    const output = 'some different output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notRougeNAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'ROUGE-N score 0.22 is less than threshold 0.75',
    });
    expect(result.score).toBeCloseTo(0.78, 2);
  });

  it('should fail when the not-rouge-n assertion score is above threshold', async () => {
    const output = 'This is the expected output.';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: notRougeNAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'ROUGE-N score 1.00 is greater than or equal to threshold 0.75',
    });
    expect(result.score).toBe(0);
  });

  // Test for starts-with assertion
  const startsWithAssertion: Assertion = {
    type: 'starts-with',
    value: 'Expected',
  };

  it('should pass when the starts-with assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: startsWithAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the starts-with assertion fails', async () => {
    const output = 'Different output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion: startsWithAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Expected output to start with "Expected"',
    });
  });

  it('should use the provider from the assertion if it exists', async () => {
    // Assertion grader passes
    const output = 'Expected output';
    const assertion: Assertion = {
      type: 'llm-rubric',
      value: 'Expected output',
      provider: Grader,
    };

    // Test grader fails
    const BogusGrader = createMockProvider({
      id: 'BogusGrader',
      callApi: vi.fn<ApiProvider['callApi']>().mockRejectedValue(new Error('Should not be called')),
    });
    const test: AtomicTestCase = {
      assert: [assertion],
      options: {
        provider: BogusGrader,
      },
    };

    // Expect test to pass because assertion grader takes priority
    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      assertion,
      test,
      providerResponse: { output },
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Test grading output',
    });
  });

  // Test for levenshtein assertion
  const levenshteinAssertion: Assertion = {
    type: 'levenshtein',
    value: 'Expected output',
    threshold: 5,
  };

  it('should pass when the levenshtein assertion passes', async () => {
    const output = 'Expected output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: levenshteinAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: true,
      reason: 'Assertion passed',
    });
  });

  it('should fail when the levenshtein assertion fails', async () => {
    const output = 'Different output';

    const result: GradingResult = await runAssertion({
      prompt: 'Some prompt',
      provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      assertion: levenshteinAssertion,
      test: {} as AtomicTestCase,
      providerResponse: { output },
    });
    expect(result).toMatchObject({
      pass: false,
      reason: 'Levenshtein distance 8 is greater than threshold 5',
    });
  });

  describe('latency assertion', () => {
    it('should pass when the latency assertion passes', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'latency',
          threshold: 100,
        },
        latencyMs: 50,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should fail when the latency assertion fails', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'latency',
          threshold: 100,
        },
        latencyMs: 1000,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Latency 1000ms is greater than threshold 100ms',
      });
    });

    it('should throw an error when grading result is missing latencyMs', async () => {
      const output = 'Expected output';

      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
          assertion: {
            type: 'latency',
            threshold: 100,
          },
          test: {} as AtomicTestCase,
          providerResponse: { output },
        }),
      ).rejects.toThrow(
        'Latency assertion does not support cached results. Rerun the eval with --no-cache',
      );
    });

    it('should pass when the latency is 0ms', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'latency',
          threshold: 100,
        },
        latencyMs: 0,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should throw an error when threshold is not provided', async () => {
      const output = 'Expected output';

      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
          assertion: {
            type: 'latency',
          },
          latencyMs: 50,
          test: {} as AtomicTestCase,
          providerResponse: { output },
        }),
      ).rejects.toThrow('Latency assertion must have a threshold in milliseconds');
    });

    it('should handle latency equal to threshold', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'latency',
          threshold: 100,
        },
        latencyMs: 100,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should NOT throw error when threshold is 0', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'latency',
          threshold: 0,
        },
        latencyMs: 0,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should fail when latency exceeds threshold=0', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'latency',
          threshold: 0,
        },
        latencyMs: 1,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Latency 1ms is greater than threshold 0ms',
      });
    });

    it('should fail (not-latency) when latency is within threshold', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-latency',
          threshold: 100,
        },
        latencyMs: 50,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Latency 50ms is less than or equal to threshold 100ms',
      });
    });

    it('should pass (not-latency) when latency exceeds threshold', async () => {
      const output = 'Expected output';

      const provider = new OpenAiChatCompletionProvider('gpt-4o-mini');
      const providerResponse = { output };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-latency',
          threshold: 100,
        },
        latencyMs: 1000,
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });
  });

  describe('perplexity assertion', () => {
    it('should pass when the perplexity assertion passes', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3]; // Dummy logProbs for testing
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity',
          threshold: 2,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should fail when the perplexity assertion fails', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3]; // Dummy logProbs for testing
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity',
          threshold: 0.2,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Perplexity 1.28 is greater than threshold 0.2',
      });
    });

    it('should PASS when no threshold is specified (default behavior)', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity',
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });

      expect(result.pass).toBe(true); // No threshold = always pass
    });

    it('should respect threshold=0 as a valid threshold', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity',
          threshold: 0,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });

      // Perplexity will be > 0, so with threshold=0, should fail
      expect(result.pass).toBe(false);
    });

    it('should fail (not-perplexity) when perplexity is within threshold', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-perplexity',
          threshold: 2,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Perplexity 1.28 is less than or equal to threshold 2',
      });
    });

    it('should pass (not-perplexity) when perplexity exceeds threshold', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-perplexity',
          threshold: 0.2,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });
  });

  describe('perplexity-score assertion', () => {
    it('should pass when the perplexity-score assertion passes', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity-score',
          threshold: 0.25,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
      // Forward variant keeps the raw normalized perplexity as its score.
      expect(result.score).toBeCloseTo(0.4378, 3);
    });

    it('should fail when the perplexity-score assertion fails', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity-score',
          threshold: 0.5,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Perplexity score 0.44 is less than threshold 0.5',
      });
    });

    it('should PASS when no threshold is specified for perplexity-score', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity-score',
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });

      expect(result.pass).toBe(true); // No threshold = always pass
    });

    it('should respect threshold=0 as valid for perplexity-score', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'perplexity-score',
          threshold: 0,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });

      // Perplexity score will be > 0, so should pass with threshold=0
      expect(result.pass).toBe(true);
    });

    it('should fail (not-perplexity-score) when score is at or above threshold', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-perplexity-score',
          threshold: 0.25,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Perplexity score 0.44 is greater than or equal to threshold 0.25',
      });
      // Score is inverted for the not- variant so it stays correlated with pass.
      expect(result.score).toBeCloseTo(0.5622, 3);
    });

    it('should pass (not-perplexity-score) when score is below threshold', async () => {
      const logProbs = [-0.2, -0.4, -0.1, -0.3];
      const provider = {
        callApi: vi.fn().mockResolvedValue({ logProbs }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', logProbs };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-perplexity-score',
          threshold: 0.5,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
      // Passing not- assertion contributes a high (inverted) score, not the raw 0.44.
      expect(result.score).toBeCloseTo(0.5622, 3);
    });
  });

  describe('cost assertion', () => {
    it('should pass when the cost is below the threshold', async () => {
      const cost = 0.0005;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'cost',
          threshold: 0.001,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should fail when the cost exceeds the threshold', async () => {
      const cost = 0.002;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'cost',
          threshold: 0.001,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Cost 0.0020 is greater than threshold 0.001',
      });
    });

    it('should NOT throw error when threshold is 0', async () => {
      const cost = 0;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'cost',
          threshold: 0,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should fail when cost exceeds threshold=0', async () => {
      const cost = 0.01;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'cost',
          threshold: 0,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Cost 0.010 is greater than threshold 0',
      });
    });

    it('should fail (not-cost) when the cost is within threshold', async () => {
      const cost = 0.0005;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-cost',
          threshold: 0.001,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'Cost 0.00050 is less than or equal to threshold 0.001',
      });
    });

    it('should pass (not-cost) when the cost exceeds threshold', async () => {
      const cost = 0.002;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider,
        assertion: {
          type: 'not-cost',
          threshold: 0.001,
        },
        test: {} as AtomicTestCase,
        providerResponse,
      });
      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should throw when no threshold is provided', async () => {
      const cost = 0.0005;
      const provider = {
        callApi: vi.fn().mockResolvedValue({ cost }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output', cost };
      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider,
          assertion: {
            type: 'cost',
          },
          test: {} as AtomicTestCase,
          providerResponse,
        }),
      ).rejects.toThrow('Cost assertion must have a threshold');
    });

    it('should throw when the provider does not return cost', async () => {
      const provider = {
        callApi: vi.fn().mockResolvedValue({ output: 'Some output' }),
      } as unknown as ApiProvider;
      const providerResponse = { output: 'Some output' };
      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider,
          assertion: {
            type: 'cost',
            threshold: 0.001,
          },
          test: {} as AtomicTestCase,
          providerResponse,
        }),
      ).rejects.toThrow('Cost assertion does not support providers that do not return cost');
    });
  });

  describe('Similarity assertion', () => {
    beforeEach(() => {
      vi.spyOn(DefaultEmbeddingProvider, 'callEmbeddingApi').mockImplementation((text) => {
        if (text === 'Test output' || text.startsWith('Similar output')) {
          return Promise.resolve({
            embedding: [1, 0, 0],
            tokenUsage: { total: 5, prompt: 2, completion: 3 },
          });
        } else if (text.startsWith('Different output')) {
          return Promise.resolve({
            embedding: [0, 1, 0],
            tokenUsage: { total: 5, prompt: 2, completion: 3 },
          });
        }
        return Promise.reject(new Error('Unexpected input'));
      });
    });

    afterEach(() => {
      vi.restoreAllMocks();
    });

    it('should pass for a similar assertion with a string value', async () => {
      const output = 'Test output';

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion: {
          type: 'similar',
          value: 'Similar output',
        },
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Similarity 1.00 is greater than or equal to threshold 0.75',
      });
    });

    it('should fail for a similar assertion with a string value', async () => {
      const output = 'Test output';

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion: {
          type: 'similar',
          value: 'Different output',
        },
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toMatchObject({
        pass: false,
        reason: 'Similarity 0.00 is less than threshold 0.75',
      });
    });

    it('should pass for a similar assertion with an array of string values', async () => {
      const output = 'Test output';

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion: {
          type: 'similar',
          value: ['Similar output 1', 'Different output 1'],
        },
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Similarity 1.00 is greater than or equal to threshold 0.75',
      });
    });

    it('should fail for a similar assertion with an array of string values', async () => {
      const output = 'Test output';

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion: {
          type: 'similar',
          value: ['Different output 1', 'Different output 2'],
        },
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });
      expect(result).toMatchObject({
        pass: false,
        reason: 'None of the provided values met the similarity threshold',
      });
    });
  });

  describe('is-xml', () => {
    const provider = {
      callApi: vi.fn().mockResolvedValue({ cost: 0.001 }),
    } as unknown as ApiProvider;

    it('should pass when the output is valid XML', async () => {
      const output = '<root><child>Content</child></root>';
      const assertion: Assertion = { type: 'is-xml' };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should fail when the output is not valid XML', async () => {
      const output = '<root><child>Content</child></root';
      const assertion: Assertion = { type: 'is-xml' };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toMatchObject({
        pass: false,
        score: 0,
        reason: expect.stringMatching(/XML parsing failed/),
        assertion,
      });
    });

    it('should pass when required elements are present', async () => {
      const output =
        '<analysis><classification>T-shirt</classification><color>Red</color></analysis>';
      const assertion: Assertion = {
        type: 'is-xml',
        value: 'analysis.classification,analysis.color',
      };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should fail when required elements are missing', async () => {
      const output = '<analysis><classification>T-shirt</classification></analysis>';
      const assertion: Assertion = {
        type: 'is-xml',
        value: 'analysis.classification,analysis.color',
      };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: false,
        score: 0,
        reason: 'XML is missing required elements: analysis.color',
        assertion,
      });
    });

    it('should pass when nested elements are present', async () => {
      const output =
        '<root><parent><child><grandchild>Content</grandchild></child></parent></root>';
      const assertion: Assertion = {
        type: 'is-xml',
        value: 'root.parent.child.grandchild',
      };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should handle inverse assertion correctly', async () => {
      const output = 'This is not XML';
      const assertion: Assertion = { type: 'not-is-xml' };

      const result = await runAssertion({
        prompt: 'Generate non-XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should pass when required elements are specified as an array', async () => {
      const output = '<root><element1>Content1</element1><element2>Content2</element2></root>';
      const assertion: Assertion = {
        type: 'is-xml',
        value: ['root.element1', 'root.element2'],
      };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should pass when required elements are specified as an object', async () => {
      const output = '<root><element1>Content1</element1><element2>Content2</element2></root>';
      const assertion: Assertion = {
        type: 'contains-xml',
        value: {
          requiredElements: ['root.element1', 'root.element2'],
        },
      };

      const result = await runAssertion({
        prompt: 'Generate XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should throw an error when xml assertion value is invalid', async () => {
      const output = '<root><element1>Content1</element1><element2>Content2</element2></root>';
      const assertion: Assertion = {
        type: 'is-xml',
        value: { invalidKey: ['root.element1', 'root.element2'] },
      };

      await expect(
        runAssertion({
          prompt: 'Generate XML',
          provider,
          assertion,
          test: {} as AtomicTestCase,
          providerResponse: { output },
        }),
      ).rejects.toThrow('xml assertion must contain a string, array value, or no value');
    });

    it('should handle multiple XML blocks in contains-xml assertion', async () => {
      const output = 'Some text <xml1>content1</xml1> more text <xml2>content2</xml2>';
      const assertion: Assertion = {
        type: 'contains-xml',
        value: ['xml1', 'xml2'],
      };

      const result = await runAssertion({
        prompt: 'Generate text with multiple XML blocks',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });
  });

  describe('contains-xml', () => {
    const provider = {
      callApi: vi.fn().mockResolvedValue({ cost: 0.001 }),
    } as unknown as ApiProvider;
    it('should pass when the output contains valid XML', async () => {
      const output = 'Some text before <root><child>Content</child></root> and after';
      const assertion: Assertion = { type: 'contains-xml' };

      const result = await runAssertion({
        prompt: 'Generate text with XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should fail when the output does not contain valid XML', async () => {
      const output = 'This is just plain text without any XML';
      const assertion: Assertion = { type: 'contains-xml' };

      const result = await runAssertion({
        prompt: 'Generate text without XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: false,
        score: 0,
        reason: 'No XML content found in the output',
        assertion,
      });
    });

    it('should pass when required elements are present in the XML', async () => {
      const output =
        'Before <analysis><classification>T-shirt</classification><color>Red</color></analysis> After';
      const assertion: Assertion = {
        type: 'contains-xml',
        value: 'analysis.classification,analysis.color',
      };

      const result = await runAssertion({
        prompt: 'Generate text with specific XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should fail when required elements are missing in the XML', async () => {
      const output = 'Before <analysis><classification>T-shirt</classification></analysis> After';
      const assertion: Assertion = {
        type: 'contains-xml',
        value: 'analysis.classification,analysis.color',
      };

      const result = await runAssertion({
        prompt: 'Generate text with specific XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: false,
        score: 0,
        reason: 'No valid XML content found matching the requirements',
        assertion,
      });
    });

    it('should pass when nested elements are present in the XML', async () => {
      const output =
        'Start <root><parent><child><grandchild>Content</grandchild></child></parent></root> End';
      const assertion: Assertion = {
        type: 'contains-xml',
        value: 'root.parent.child.grandchild',
      };

      const result = await runAssertion({
        prompt: 'Generate text with nested XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should handle inverse assertion correctly', async () => {
      const output = 'This is just plain text without any XML';
      const assertion: Assertion = { type: 'not-contains-xml' };

      const result = await runAssertion({
        prompt: 'Generate text without XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Assertion passed',
        assertion,
      });
    });

    it('should fail inverse assertion when XML is present', async () => {
      const output = 'Some text with <xml>content</xml> in it';
      const assertion: Assertion = { type: 'not-contains-xml' };

      const result = await runAssertion({
        prompt: 'Generate text without XML',
        provider,
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toEqual({
        pass: false,
        score: 0,
        reason: 'XML is valid and contains all required elements',
        assertion,
      });
    });
  });

  describe('context-relevance assertion', () => {
    it('should pass when all required vars are present', async () => {
      const assertion: Assertion = {
        type: 'context-relevance',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          query: 'What is the capital of France?',
          context: 'Paris is the capital of France.',
        },
      };

      const result = await runAssertion({
        assertion,
        test,
        providerResponse: { output: 'Some output' },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Mocked reason',
        assertion,
        metadata: {
          context: 'Paris is the capital of France.',
        },
      });
    });

    it('should use resolved vars for context-based grading', async () => {
      const assertion: Assertion = {
        type: 'context-relevance',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          query: 'What is the capital of France?',
          context: 'file://docs/france.md',
        },
      };

      const result = await runAssertion({
        assertion,
        test,
        vars: {
          query: 'What is the capital of France?',
          context: 'Paris is the capital of France.',
        },
        providerResponse: { output: 'Some output' },
      });

      expect(result).toMatchObject({
        pass: true,
        metadata: {
          context: 'Paris is the capital of France.',
        },
      });
    });

    it('should throw an error when vars object is missing', async () => {
      const assertion: Assertion = {
        type: 'context-relevance',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {};

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: 'Some output' },
        }),
      ).rejects.toThrow('context-relevance assertion requires a test with variables');
    });

    it('should throw an error when query var is missing', async () => {
      const assertion: Assertion = {
        type: 'context-relevance',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          context: 'Paris is the capital of France.',
        },
      };

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: 'Some output' },
        }),
      ).rejects.toThrow(
        'context-relevance assertion requires a "query" variable with the user question',
      );
    });

    it('should throw an error when context var is missing', async () => {
      const assertion: Assertion = {
        type: 'context-relevance',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          query: 'What is the capital of France?',
        },
      };

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: 'Some output' },
        }),
      ).rejects.toThrow(
        'Context is required for context-based assertions. Provide either a "context" variable (string or array of strings) in your test case or use "contextTransform" to extract context from the provider response.',
      );
    });
  });

  describe('context-faithfulness assertion', () => {
    it('should pass when all required vars are present', async () => {
      const assertion: Assertion = {
        type: 'context-faithfulness',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          query: 'What is the capital of France?',
          context: 'Paris is the capital of France.',
        },
      };

      const result = await runAssertion({
        assertion,
        test,
        providerResponse: { output: 'The capital of France is Paris.' },
      });

      expect(result).toEqual({
        pass: true,
        score: 1,
        reason: 'Mocked reason',
        assertion,
        metadata: {
          context: 'Paris is the capital of France.',
        },
      });
    });

    it('should throw an error when vars object is missing', async () => {
      const assertion: Assertion = {
        type: 'context-faithfulness',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {};

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: 'Some output' },
        }),
      ).rejects.toThrow('context-faithfulness assertion requires a test with variables');
    });

    it('should throw an error when query var is missing', async () => {
      const assertion: Assertion = {
        type: 'context-faithfulness',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          context: 'Paris is the capital of France.',
        },
      };

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: 'Some output' },
        }),
      ).rejects.toThrow(
        'context-faithfulness assertion requires a "query" variable with the user question',
      );
    });

    it('should throw an error when context var is missing', async () => {
      const assertion: Assertion = {
        type: 'context-faithfulness',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          query: 'What is the capital of France?',
        },
      };

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: 'Some output' },
        }),
      ).rejects.toThrow(
        'Context is required for context-based assertions. Provide either a "context" variable (string or array of strings) in your test case or use "contextTransform" to extract context from the provider response.',
      );
    });

    it('should throw an error when output is not a string', async () => {
      const assertion: Assertion = {
        type: 'context-faithfulness',
        threshold: 0.7,
      };
      const test: AtomicTestCase = {
        vars: {
          query: 'What is the capital of France?',
          context: 'Paris is the capital of France.',
        },
      };

      await expect(
        runAssertion({
          assertion,
          test,
          providerResponse: { output: { some: 'object' } },
        }),
      ).rejects.toThrow('context-faithfulness assertion requires string output from the provider');
    });
  });

  describe('assertion transform with metadata', () => {
    it('should pass metadata to assertion transform when available', async () => {
      const output = 'Test output';
      const metadata = { key: 'value', nested: { data: 123 } };

      const assertion: Assertion = {
        type: 'equals',
        value: 'Metadata: {"key":"value","nested":{"data":123}}',
        transform: 'context.metadata ? `Metadata: ${JSON.stringify(context.metadata)}` : output',
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output, metadata },
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should handle transform when metadata is undefined', async () => {
      const output = 'Test output';

      const assertion: Assertion = {
        type: 'equals',
        value: 'No metadata',
        transform: 'context.metadata ? "Has metadata" : "No metadata"',
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should handle transform when providerResponse has no metadata', async () => {
      const output = 'Test output';

      const assertion: Assertion = {
        type: 'equals',
        value: 'No metadata',
        transform: 'context.metadata ? "Has metadata" : "No metadata"',
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output }, // No metadata property
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should handle empty metadata object', async () => {
      const output = 'Test output';
      const metadata = {};

      const assertion: Assertion = {
        type: 'equals',
        value: 'Empty metadata',
        transform:
          '(context.metadata && Object.keys(context.metadata).length === 0) ? "Empty metadata" : output',
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output, metadata },
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });

    it('should preserve existing context properties when adding metadata', async () => {
      const output = 'Test output';
      const metadata = { responseTime: 100 };
      const testVars = { userInput: 'test input' };

      const assertion: Assertion = {
        type: 'equals',
        value: 'All context properties present',
        transform:
          '(context.vars && context.prompt && context.metadata) ? "All context properties present" : "Missing context properties"',
      };

      const result: GradingResult = await runAssertion({
        prompt: 'test prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: { vars: testVars } as AtomicTestCase,
        providerResponse: { output, metadata },
      });

      expect(result).toMatchObject({
        pass: true,
        reason: 'Assertion passed',
      });
    });
  });

  describe('inline function transforms (Node.js package)', () => {
    it('should support a function as assertion transform', async () => {
      const output = 'hello world';
      const assertion: Assertion = {
        type: 'equals',
        value: 'HELLO WORLD',
        transform: (output) => String(output).toUpperCase(),
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result.pass).toBe(true);
      expect(result.reason).toBe('Assertion passed');
    });

    it('should pass context to inline function transform', async () => {
      const output = 'raw output';
      const assertion: Assertion = {
        type: 'equals',
        value: 'search, calculate',
        transform: (_output, context: any) => {
          const tools = context.metadata?.toolCalls ?? [];
          return tools.map((t: any) => t.name).join(', ');
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: {
          output,
          metadata: { toolCalls: [{ name: 'search' }, { name: 'calculate' }] },
        },
      });

      expect(result.pass).toBe(true);
    });

    it('should support async function as assertion transform', async () => {
      const output = 'hello';
      const assertion: Assertion = {
        type: 'equals',
        value: 'hello async',
        transform: async (output) => output + ' async',
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output },
      });

      expect(result.pass).toBe(true);
    });

    it('should surface synchronous errors thrown by an inline function transform', async () => {
      const assertion: Assertion = {
        type: 'equals',
        value: 'anything',
        transform: () => {
          throw new Error('inline transform boom');
        },
      };

      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
          assertion,
          test: {} as AtomicTestCase,
          providerResponse: { output: 'hello world' },
        }),
      ).rejects.toThrow('inline transform boom');
    });

    it('should surface async rejections from an inline function transform', async () => {
      const assertion: Assertion = {
        type: 'equals',
        value: 'anything',
        transform: async () => {
          throw new Error('async transform boom');
        },
      };

      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
          assertion,
          test: {} as AtomicTestCase,
          providerResponse: { output: 'hello world' },
        }),
      ).rejects.toThrow('async transform boom');
    });
  });

  describe('file references', () => {
    it('should handle file reference in string value', async () => {
      const assertion: Assertion = {
        type: 'equals',
        value: 'file://expected_output.txt',
      };

      const expectedContent = 'Expected output';
      vi.mocked(fs.readFileSync).mockReturnValue(expectedContent);
      vi.mocked(path.resolve).mockReturnValue('/base/path/expected_output.txt');
      vi.mocked(path.extname).mockReturnValue('.txt');

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output: 'Expected output' },
      });

      expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/expected_output.txt', 'utf8');
      expect(result.pass).toBe(true);
    });

    it('should handle file references in array values', async () => {
      const assertion: Assertion = {
        type: 'contains-any',
        value: ['The expected output', 'string output', 'file://my_expected_output.txt'],
      };

      const fileContent = 'file content';
      vi.mocked(fs.readFileSync).mockReturnValue(fileContent);
      vi.mocked(path.resolve).mockReturnValue('/base/path/my_expected_output.txt');
      vi.mocked(path.extname).mockReturnValue('.txt');

      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
          assertion,
          test: {} as AtomicTestCase,
          providerResponse: { output: 'file content' },
        }),
      ).resolves.toEqual(
        expect.objectContaining({
          pass: true,
        }),
      );
      expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/my_expected_output.txt', 'utf8');

      await expect(
        runAssertion({
          prompt: 'Some prompt',
          provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
          assertion,
          test: {} as AtomicTestCase,
          providerResponse: { output: 'string output' },
        }),
      ).resolves.toEqual(
        expect.objectContaining({
          pass: true,
        }),
      );
    });

    it('should handle file reference in object value', async () => {
      const assertion: Assertion = {
        type: 'is-json',
        value: 'file://schema.json',
      };

      const schemaContent = JSON.stringify({
        type: 'object',
        properties: {
          key: { type: 'string' },
        },
      });
      vi.mocked(fs.readFileSync).mockReturnValue(schemaContent);
      vi.mocked(path.resolve).mockReturnValue('/base/path/schema.json');
      vi.mocked(path.extname).mockReturnValue('.json');

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
        assertion,
        test: {} as AtomicTestCase,
        providerResponse: { output: '{"key": "value"}' },
      });

      expect(fs.readFileSync).toHaveBeenCalledWith('/base/path/schema.json', 'utf8');
      expect(result.pass).toBe(true);
    });
  });

  describe('Rendered assertion values', () => {
    it('should store rendered assertion value in metadata when variables are substituted', async () => {
      const assertion: Assertion = {
        type: 'contains',
        value: 'User said: {{ user_input }}',
      };

      const test: AtomicTestCase = {
        vars: {
          user_input: 'hello world',
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'User said: hello world' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      expect(result.metadata?.renderedAssertionValue).toBe('User said: hello world');
      expect(result.assertion?.value).toBe('User said: {{ user_input }}');
    });

    it('should store rendered assertion value with loops', async () => {
      const assertion: Assertion = {
        type: 'contains',
        value:
          '{% for item in items %}{{ item.name }}{% if not loop.last %}, {% endif %}{% endfor %}',
      };

      const test: AtomicTestCase = {
        vars: {
          items: [{ name: 'apple' }, { name: 'banana' }, { name: 'cherry' }],
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'apple, banana, cherry' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      expect(result.metadata?.renderedAssertionValue).toBe('apple, banana, cherry');
    });

    it('should not store rendered value when template equals original', async () => {
      const assertion: Assertion = {
        type: 'contains',
        value: 'Static text without variables',
      };

      const test: AtomicTestCase = {
        vars: {
          user_input: 'hello',
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'Static text without variables' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      expect(result.metadata?.renderedAssertionValue).toBeUndefined();
    });

    it('should store rendered value for complex template matching GitHub issue example', async () => {
      const assertion: Assertion = {
        type: 'contains',
        value: `Did {{ agent_name }} select the expected tools across the conversation?

{% for turn in turns %}
{% if turn.expected_tool %}Turn {{ loop.index }}: Expected tool "{{ turn.expected_tool }}"
{% endif %}
{% endfor %}`,
      };

      const test: AtomicTestCase = {
        vars: {
          agent_name: 'Virtual Assistant',
          turns: [
            { expected_tool: 'preview_create' },
            { expected_tool: 'preview_create' },
            { expected_tool: 'execute_create' },
          ],
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'some output' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      expect(result.metadata?.renderedAssertionValue).toContain('Virtual Assistant');
      expect(result.metadata?.renderedAssertionValue).toContain(
        'Turn 1: Expected tool "preview_create"',
      );
      expect(result.metadata?.renderedAssertionValue).toContain(
        'Turn 2: Expected tool "preview_create"',
      );
      expect(result.metadata?.renderedAssertionValue).toContain(
        'Turn 3: Expected tool "execute_create"',
      );
    });

    it('should store rendered value for javascript assertions', async () => {
      const assertion: Assertion = {
        type: 'javascript',
        value: "output.includes('{{ expected_value }}')",
      };

      const test: AtomicTestCase = {
        vars: {
          expected_value: 'hello',
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'hello world' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      expect(result.metadata?.renderedAssertionValue).toBe("output.includes('hello')");
    });

    it('should store rendered value even when assertion fails', async () => {
      const assertion: Assertion = {
        type: 'contains',
        value: 'Expected: {{ value }}',
      };

      const test: AtomicTestCase = {
        vars: {
          value: 'foo',
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'Different output - no match' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      // Assertion should fail (output doesn't contain "Expected: foo")
      expect(result.pass).toBe(false);
      // But rendered value should still be captured in metadata
      expect(result.metadata?.renderedAssertionValue).toBe('Expected: foo');
      // Original template should remain in assertion
      expect(result.assertion?.value).toBe('Expected: {{ value }}');
    });

    it('should preserve existing metadata when adding renderedAssertionValue', async () => {
      const assertion: Assertion = {
        type: 'contains',
        value: 'User: {{ name }}',
      };

      const test: AtomicTestCase = {
        vars: {
          name: 'Alice',
        },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'User: Alice' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      // Should have rendered value
      expect(result.metadata?.renderedAssertionValue).toBe('User: Alice');
      // Original template in assertion
      expect(result.assertion?.value).toBe('User: {{ name }}');
      // Metadata object should exist and be extensible
      expect(result.metadata).toBeTruthy();
    });

    // Regression for https://github.com/promptfoo/promptfoo/issues/7861: the
    // reporter claimed defaultTest llm-rubric rubrics sent raw {{var}} text to
    // the grader. In reality the assertion pipeline renders the value before
    // grading — so the grading LLM DOES receive the substituted string. Prove
    // this by letting the assertion run all the way through a real grader
    // provider and inspecting the prompt it received.
    it('renders llm-rubric value with vars before calling the grader', async () => {
      const capturedPrompt = vi.fn<ApiProvider['callApi']>(async () => ({
        output: JSON.stringify({ pass: true, score: 1, reason: 'graded' }),
      }));
      const capturingGrader = createMockProvider({
        id: 'capturing-grader',
        callApi: capturedPrompt,
      });

      const assertion: Assertion = {
        type: 'llm-rubric',
        value: 'Does the output correctly reference the input: {{myVar}}?',
        provider: capturingGrader,
      };

      const test: AtomicTestCase = {
        vars: { myVar: 'hello world' },
      };

      const result: GradingResult = await runAssertion({
        prompt: 'Some prompt',
        assertion,
        test,
        providerResponse: { output: 'static model output' },
        provider: new OpenAiChatCompletionProvider('gpt-4o-mini'),
      });

      const gradingPrompt = capturedPrompt.mock.calls[0]?.[0] as string;
      expect(gradingPrompt).toContain('hello world');
      expect(gradingPrompt).not.toContain('{{myVar}}');

      expect(result.metadata?.renderedAssertionValue).toBe(
        'Does the output correctly reference the input: hello world?',
      );
      expect(result.assertion?.value).toBe(
        'Does the output correctly reference the input: {{myVar}}?',
      );
    });
  });
});