promptfoo--promptfoo/test/providers/openai/agents.test.ts

import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';

const mockRun = vi.hoisted(() => vi.fn());
const mockGetOrCreateTrace = vi.hoisted(() => vi.fn(async (fn: () => Promise<unknown>) => fn()));
const mockRetryPolicies = vi.hoisted(() => {
  const neverPolicy = vi.fn();
  const providerSuggestedPolicy = vi.fn();
  const networkErrorPolicy = vi.fn();
  const retryAfterPolicy = vi.fn();
  const httpStatusPolicy = vi.fn();
  const anyPolicy = vi.fn();
  const allPolicy = vi.fn();

  return {
    never: vi.fn(() => neverPolicy),
    providerSuggested: vi.fn(() => providerSuggestedPolicy),
    networkError: vi.fn(() => networkErrorPolicy),
    retryAfter: vi.fn(() => retryAfterPolicy),
    httpStatus: vi.fn(() => httpStatusPolicy),
    any: vi.fn(() => anyPolicy),
    all: vi.fn(() => allPolicy),
  };
});

vi.mock('@openai/agents', async (importOriginal) => {
  const actual = await importOriginal<typeof import('@openai/agents')>();

  class MockAgent {
    name: string;
    instructions: unknown;
    prompt?: unknown;
    model?: string;
    modelSettings: Record<string, unknown>;
    handoffDescription: string;
    handoffOutputTypeWarningEnabled?: boolean;
    outputType?: unknown;
    tools: any[];
    handoffs: any[];
    inputGuardrails: any[];
    outputGuardrails: any[];
    mcpServers: any[];
    toolUseBehavior: unknown;
    resetToolChoice: boolean;

    constructor(config: Record<string, any>) {
      this.name = config.name;
      this.instructions = config.instructions;
      this.prompt = config.prompt;
      this.model = config.model;
      this.modelSettings = config.modelSettings ?? {};
      this.handoffDescription = config.handoffDescription ?? '';
      this.handoffOutputTypeWarningEnabled = config.handoffOutputTypeWarningEnabled;
      this.outputType = config.outputType;
      this.tools = config.tools ?? [];
      this.handoffs = config.handoffs ?? [];
      this.inputGuardrails = config.inputGuardrails ?? [];
      this.outputGuardrails = config.outputGuardrails ?? [];
      this.mcpServers = config.mcpServers ?? [];
      this.toolUseBehavior = config.toolUseBehavior ?? 'run_llm_again';
      this.resetToolChoice = config.resetToolChoice ?? true;
    }

    clone(config: Record<string, any>) {
      return new MockAgent({
        ...this,
        ...config,
      });
    }
  }

  return {
    ...actual,
    Agent: MockAgent,
    BatchTraceProcessor: class BatchTraceProcessor {
      exporter: unknown;
      options: unknown;

      constructor(exporter: unknown, options: unknown) {
        this.exporter = exporter;
        this.options = options;
      }
    },
    addTraceProcessor: vi.fn(),
    getOrCreateTrace: mockGetOrCreateTrace,
    handoff: vi.fn((agent: MockAgent, config?: Record<string, any>) => ({
      agent,
      agentName: agent.name,
      getHandoffAsFunctionTool: vi.fn(),
      onInvokeHandoff: vi.fn(),
      toolDescription: config?.toolDescriptionOverride,
      isEnabled: vi.fn(async () => true),
    })),
    retryPolicies: mockRetryPolicies,
    run: mockRun,
    setTraceProcessors: vi.fn(),
    startTraceExportLoop: vi.fn(),
    tool: vi.fn((options: Record<string, any>) => ({
      type: 'function',
      name: options.name ?? 'anonymous_tool',
      description: options.description,
      parameters: options.parameters,
      strict: options.strict ?? true,
      deferLoading: options.deferLoading,
      invoke: vi.fn(async () => {
        if (typeof options.execute === 'function') {
          return options.execute({});
        }
        return options.execute;
      }),
      needsApproval: vi.fn(async () => false),
      isEnabled: vi.fn(async () => true),
    })),
  };
});

vi.mock('../../../src/esm', async (importOriginal) => ({
  ...(await importOriginal()),
  importModule: vi.fn(),
}));

vi.mock('../../../src/logger', () => ({
  __esModule: true,
  default: {
    debug: vi.fn(),
    info: vi.fn(),
    warn: vi.fn(),
    error: vi.fn(),
  },
}));

import {
  Agent,
  addTraceProcessor,
  handoff,
  MemorySession,
  OpenAIConversationsSession,
  OpenAIResponsesCompactionSession,
  setTraceProcessors,
  tool,
} from '@openai/agents';
import { Manifest, SandboxAgent } from '@openai/agents/sandbox';
import cliState from '../../../src/cliState';
import { importModule } from '../../../src/esm';
import { OpenAiAgentsProvider } from '../../../src/providers/openai/agents';
import {
  loadAgentDefinition,
  loadSandboxConfig,
  loadSessionDefinition,
} from '../../../src/providers/openai/agents-loader';

function resetOpenAiAgentsMocks() {
  mockRun.mockReset().mockResolvedValue({
    finalOutput: 'Agent answer',
    usage: {
      totalTokens: 12,
      promptTokens: 7,
      completionTokens: 5,
    },
    newItems: [],
  });
  mockGetOrCreateTrace.mockReset().mockImplementation(async (fn: () => Promise<unknown>) => fn());
  mockRetryPolicies.never.mockReset().mockImplementation(() => vi.fn());
  mockRetryPolicies.providerSuggested.mockReset().mockImplementation(() => vi.fn());
  mockRetryPolicies.networkError.mockReset().mockImplementation(() => vi.fn());
  mockRetryPolicies.retryAfter.mockReset().mockImplementation(() => vi.fn());
  mockRetryPolicies.httpStatus.mockReset().mockImplementation(() => vi.fn());
  mockRetryPolicies.any.mockReset().mockImplementation(() => vi.fn());
  mockRetryPolicies.all.mockReset().mockImplementation(() => vi.fn());
}

describe('OpenAiAgentsProvider', () => {
  const mockImportModule = vi.mocked(importModule);
  const baseInputGuardrail = {
    name: 'base-input',
    execute: vi.fn(async () => ({ tripwireTriggered: false, outputInfo: {} })),
  };
  const baseOutputGuardrail = {
    name: 'base-output',
    execute: vi.fn(async () => ({ tripwireTriggered: false, outputInfo: {} })),
  };

  beforeEach(() => {
    vi.clearAllMocks();
    resetOpenAiAgentsMocks();
    cliState.basePath = undefined;
  });

  afterEach(() => {
    cliState.basePath = undefined;
  });

  it('creates a real SDK agent from inline definitions', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
          tools: [
            {
              name: 'lookup_order',
              description: 'Look up an order',
              parameters: {
                type: 'object',
                properties: {},
                required: [],
                additionalProperties: false,
              },
              execute: async () => ({ status: 'shipped' }),
            },
          ],
          handoffs: [
            {
              agent: {
                name: 'Billing Agent',
                instructions: 'Handle billing requests.',
              },
              description: 'Transfer billing issues',
            },
          ],
          inputGuardrails: [
            {
              name: 'inline-input',
              execute: async () => ({ tripwireTriggered: false, outputInfo: {} }),
            },
          ],
          outputGuardrails: [
            {
              name: 'inline-output',
              execute: async () => ({ tripwireTriggered: false, outputInfo: {} }),
            },
          ],
        },
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun).toHaveBeenCalledTimes(1);
    const agent = mockRun.mock.calls[0][0];

    expect(agent).toBeInstanceOf(Agent);
    expect(agent.name).toBe('Inline Support Agent');
    expect(agent.tools).toHaveLength(1);
    expect(agent.tools[0].name).toBe('lookup_order');
    expect(handoff).toHaveBeenCalledWith(
      expect.objectContaining({ name: 'Billing Agent' }),
      expect.objectContaining({ toolDescriptionOverride: 'Transfer billing issues' }),
    );
    expect(agent.handoffs).toHaveLength(1);
    expect(agent.inputGuardrails.map((guardrail: { name: string }) => guardrail.name)).toEqual([
      'inline-input',
    ]);
    expect(agent.outputGuardrails.map((guardrail: { name: string }) => guardrail.name)).toEqual([
      'inline-output',
    ]);
  });

  it('merges file-based tools, handoffs, and guardrails into the loaded agent', async () => {
    const baseTool = vi.mocked(tool)({
      name: 'base_tool',
      description: 'Base tool',
      parameters: {
        type: 'object',
        properties: {},
        required: [],
        additionalProperties: false,
      },
      execute: async () => ({ ok: true }),
    });
    const baseHandoffAgent = new Agent({
      name: 'Existing Specialist',
      instructions: 'Already configured specialist.',
    });
    const baseAgent = new Agent({
      name: 'Support Agent',
      instructions: 'Help the user.',
      tools: [baseTool],
      handoffs: [baseHandoffAgent],
      inputGuardrails: [baseInputGuardrail],
      outputGuardrails: [baseOutputGuardrail],
    });

    mockImportModule.mockImplementation(async (modulePath: string) => {
      switch (modulePath) {
        case '/tmp/agent.ts':
          return { default: baseAgent };
        case '/tmp/tools.ts':
          return {
            default: [
              {
                name: 'lookup_order',
                description: 'Look up an order',
                parameters: {
                  type: 'object',
                  properties: {},
                  required: [],
                  additionalProperties: false,
                },
                execute: async () => ({ status: 'shipped' }),
              },
            ],
          };
        case '/tmp/handoffs.ts':
          return {
            default: [
              {
                agent: {
                  name: 'Billing Agent',
                  instructions: 'Handle billing requests.',
                },
                description: 'Transfer billing issues',
              },
            ],
          };
        case '/tmp/input-guardrails.ts':
          return {
            default: [
              {
                name: 'file-input',
                execute: async () => ({ tripwireTriggered: false, outputInfo: {} }),
              },
            ],
          };
        case '/tmp/output-guardrails.ts':
          return {
            default: [
              {
                name: 'file-output',
                execute: async () => ({ tripwireTriggered: false, outputInfo: {} }),
              },
            ],
          };
        default:
          throw new Error(`Unexpected module path: ${modulePath}`);
      }
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: 'file:///tmp/agent.ts',
        tools: 'file:///tmp/tools.ts',
        handoffs: 'file:///tmp/handoffs.ts',
        inputGuardrails: 'file:///tmp/input-guardrails.ts',
        outputGuardrails: 'file:///tmp/output-guardrails.ts',
        model: 'gpt-5-mini',
        modelSettings: {
          retry: {
            maxRetries: 2,
            backoff: {
              initialDelayMs: 50,
              multiplier: 2,
            },
            policy: {
              any: ['providerSuggested', { httpStatus: [429] }],
            },
          },
        },
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun).toHaveBeenCalledTimes(1);

    const agent = mockRun.mock.calls[0][0];
    const runOptions = mockRun.mock.calls[0][2];

    expect(agent.tools.map((loadedTool: { name: string }) => loadedTool.name)).toEqual([
      'base_tool',
      'lookup_order',
    ]);
    expect(agent.handoffs).toHaveLength(2);
    expect(agent.inputGuardrails.map((guardrail: { name: string }) => guardrail.name)).toEqual([
      'base-input',
      'file-input',
    ]);
    expect(agent.outputGuardrails.map((guardrail: { name: string }) => guardrail.name)).toEqual([
      'base-output',
      'file-output',
    ]);
    expect(runOptions.model).toBe('gpt-5-mini');
    expect(runOptions.modelSettings.retry.maxRetries).toBe(2);
    expect(typeof runOptions.modelSettings.retry.policy).toBe('function');
    expect(mockRetryPolicies.providerSuggested).toHaveBeenCalledTimes(1);
    expect(mockRetryPolicies.httpStatus).toHaveBeenCalledWith([429]);
    expect(mockRetryPolicies.any).toHaveBeenCalledTimes(1);
  });

  it('replaces function tool execution with configured mocks when executeTools is mock', async () => {
    const baseTool = vi.mocked(tool)({
      name: 'lookup_order',
      description: 'Look up an order',
      parameters: {
        type: 'object',
        properties: {},
        required: [],
        additionalProperties: false,
      },
      execute: async () => ({ status: 'real' }),
    });
    const baseAgent = new Agent({
      name: 'Support Agent',
      instructions: 'Help the user.',
      tools: [baseTool],
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: baseAgent,
        executeTools: 'mock',
        toolMocks: {
          lookup_order: { status: 'mocked' },
        },
      },
    });

    await provider.callApi('Where is my order?');

    const agent = mockRun.mock.calls[0][0];
    const toolResult = await agent.tools[0].invoke({}, '{}');

    expect(toolResult).toEqual({ status: 'mocked' });
  });

  it('preserves explicit zero-valued token usage fields', async () => {
    mockRun.mockResolvedValue({
      finalOutput: 'Agent answer',
      usage: {
        totalTokens: 0,
        promptTokens: 0,
        completionTokens: 0,
      },
      newItems: [],
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    const result = await provider.callApi('Where is my order?');

    expect(result.tokenUsage).toEqual({
      total: 0,
      prompt: 0,
      completion: 0,
    });
    expect(result.cost).toBeUndefined();
  });

  it('preserves extended SDK usage details from modern Agents SDK runs', async () => {
    mockRun.mockResolvedValue({
      finalOutput: 'Agent answer',
      runContext: {
        usage: {
          requests: 2,
          inputTokens: 11,
          outputTokens: 7,
          totalTokens: 18,
          inputTokensDetails: [{ cached_tokens: 3 }],
          outputTokensDetails: [
            {
              reasoning_tokens: 2,
              accepted_prediction_tokens: 1,
              rejected_prediction_tokens: 4,
            },
          ],
        },
      },
      newItems: [],
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    const result = await provider.callApi('Where is my order?');

    expect(result.tokenUsage).toEqual({
      total: 18,
      prompt: 11,
      completion: 7,
      cached: 3,
      numRequests: 2,
      completionDetails: {
        reasoning: 2,
        acceptedPrediction: 1,
        rejectedPrediction: 4,
        cacheReadInputTokens: 3,
      },
    });
  });

  it('preserves an explicit maxTurns value of 0 in run options', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        maxTurns: 0,
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun).toHaveBeenCalledWith(
      expect.any(Agent),
      'Where is my order?',
      expect.objectContaining({ maxTurns: 0 }),
    );
  });

  it('preserves an explicit maxTurns value of null in run options', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        maxTurns: null,
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun).toHaveBeenCalledWith(
      expect.any(Agent),
      'Where is my order?',
      expect.objectContaining({ maxTurns: null }),
    );
  });

  it('passes SDK run options plus top-level session and sandbox configs to run()', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        session: {
          type: 'memory',
          sessionId: 'session-1',
        },
        sandbox: {
          type: 'unix-local',
          clientOptions: {
            workspaceBaseDir: '/tmp/promptfoo-agents-sdk',
          },
        },
        runOptions: {
          previousResponseId: 'resp_123',
          conversationId: 'conv_123',
          reasoningItemIdPolicy: 'omit',
        },
      },
    });

    await provider.callApi('Where is my order?');

    const runOptions = mockRun.mock.calls[0][2];
    expect(runOptions.previousResponseId).toBe('resp_123');
    expect(runOptions.conversationId).toBe('conv_123');
    expect(runOptions.reasoningItemIdPolicy).toBe('omit');
    expect(runOptions.session).toBeInstanceOf(MemorySession);
    expect(await runOptions.session.getSessionId()).toBe('session-1');
    expect(runOptions.sandbox.client.backendId).toBe('unix_local');
  });

  it('drops reserved streaming mode from configured run options', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        runOptions: {
          stream: true,
        } as any,
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun.mock.calls[0][2].stream).toBeUndefined();
  });

  it('does not treat the provider label as a model override', async () => {
    const provider = new OpenAiAgentsProvider('support-agent', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun.mock.calls[0][2].model).toBeUndefined();
  });

  it('passes Promptfoo vars through the SDK local run context', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    await provider.callApi(
      'What customer tier am I on?',
      {
        prompt: {
          raw: 'What customer tier am I on?',
          label: 'prompt',
        } as any,
        vars: {
          customer_tier: 'gold',
        },
      },
      undefined,
    );

    expect(mockRun.mock.calls[0][2].context).toEqual({
      customer_tier: 'gold',
    });
  });

  it('re-runs file-exported session factories for each provider call', async () => {
    const createSession = vi.fn(async () => new MemorySession());
    mockImportModule.mockResolvedValue({
      default: createSession,
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        session: 'file://./sessions/support-session.ts',
      },
    });

    await provider.callApi('first');
    await provider.callApi('second');

    expect(createSession).toHaveBeenCalledTimes(2);
    expect(mockRun.mock.calls[0][2].session).not.toBe(mockRun.mock.calls[1][2].session);
  });

  it('serializes calls that intentionally reuse one shared configured session', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        session: {
          type: 'memory',
          sessionId: 'shared-session',
        },
      },
    });

    let releaseFirstRun: () => void = () => {};
    const firstRunStarted = new Promise<void>((resolve) => {
      mockRun.mockImplementationOnce(async () => {
        resolve();
        await new Promise<void>((release) => {
          releaseFirstRun = release;
        });
        return { finalOutput: 'first', newItems: [], usage: undefined };
      });
    });
    mockRun.mockImplementationOnce(async () => ({
      finalOutput: 'second',
      newItems: [],
      usage: undefined,
    }));

    const firstCall = provider.callApi('first');
    await firstRunStarted;
    const secondCall = provider.callApi('second');

    await Promise.resolve();
    expect(mockRun).toHaveBeenCalledTimes(1);

    releaseFirstRun();
    await Promise.all([firstCall, secondCall]);

    expect(mockRun).toHaveBeenCalledTimes(2);
    expect(mockRun.mock.calls[0][2].session).toBe(mockRun.mock.calls[1][2].session);
  });

  it('serializes calls when a session factory reuses one Session instance', async () => {
    const sharedSession = new MemorySession({ sessionId: 'shared-session' });
    const createSession = vi.fn(async () => sharedSession);
    mockImportModule.mockResolvedValue({
      default: createSession,
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        session: 'file://./sessions/support-session.ts',
      },
    });

    let releaseFirstRun: () => void = () => {};
    const firstRunStarted = new Promise<void>((resolve) => {
      mockRun.mockImplementationOnce(async () => {
        resolve();
        await new Promise<void>((release) => {
          releaseFirstRun = release;
        });
        return { finalOutput: 'first', newItems: [], usage: undefined };
      });
    });
    mockRun.mockImplementationOnce(async () => ({
      finalOutput: 'second',
      newItems: [],
      usage: undefined,
    }));

    const firstCall = provider.callApi('first');
    await firstRunStarted;
    const secondCall = provider.callApi('second');

    await Promise.resolve();
    expect(mockRun).toHaveBeenCalledTimes(1);

    releaseFirstRun();
    await Promise.all([firstCall, secondCall]);

    expect(createSession).toHaveBeenCalledTimes(2);
    expect(mockRun).toHaveBeenCalledTimes(2);
    expect(mockRun.mock.calls[0][2].session).toBe(sharedSession);
    expect(mockRun.mock.calls[1][2].session).toBe(sharedSession);
  });

  it('reuses one configured session when the first calls start concurrently', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        session: {
          type: 'memory',
          sessionId: 'shared-session',
        },
      },
    });

    await Promise.all([provider.callApi('first'), provider.callApi('second')]);

    expect(mockRun).toHaveBeenCalledTimes(2);
    expect(mockRun.mock.calls[0][2].session).toBe(mockRun.mock.calls[1][2].session);
  });

  it('loads file-exported executable run options', async () => {
    const sessionInputCallback = vi.fn();
    mockImportModule.mockResolvedValue({
      default: sessionInputCallback,
    });

    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        runOptions: {
          sessionInputCallback: 'file://./callbacks/session-input.ts',
        },
      },
    });

    await provider.callApi('Where is my order?');

    expect(mockRun.mock.calls[0][2].sessionInputCallback).toBe(sessionInputCallback);
  });

  it('joins SDK tracing to the evaluator trace when traceparent is provided', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
      },
    });

    await provider.callApi(
      'Where is my order?',
      {
        evaluationId: 'eval-1',
        prompt: {
          raw: 'Where is my order?',
          label: 'prompt',
        } as any,
        testCaseId: 'case-1',
        traceparent: '00-0123456789abcdef0123456789abcdef-0123456789abcdef-01',
        vars: {},
      },
      undefined,
    );

    expect(mockGetOrCreateTrace).toHaveBeenCalledWith(expect.any(Function), {
      traceId: 'trace_0123456789abcdef0123456789abcdef',
      metadata: {
        'evaluation.id': 'eval-1',
        'test.case.id': 'case-1',
        'promptfoo.parent_span_id': '0123456789abcdef',
      },
    });
  });

  it('adds the Promptfoo trace exporter without replacing existing processors', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Inline Support Agent',
          instructions: 'Help the user.',
        },
        tracing: true,
      },
    });

    await provider.callApi('Where is my order?');

    expect(addTraceProcessor).toHaveBeenCalledTimes(1);
    expect(setTraceProcessors).not.toHaveBeenCalled();
  });

  it('passes structured multimodal JSON prompts to the SDK as agent input items', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'Vision Agent',
          instructions: 'Describe the image.',
        },
      },
    });
    const prompt = JSON.stringify([
      {
        role: 'user',
        content: [
          { type: 'input_text', text: 'What is in this image?' },
          {
            type: 'input_image',
            image: 'data:image/png;base64,iVBORw0KGgo=',
          },
        ],
      },
    ]);

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(
      expect.any(Agent),
      [
        {
          role: 'user',
          content: [
            { type: 'input_text', text: 'What is in this image?' },
            {
              type: 'input_image',
              image: 'data:image/png;base64,iVBORw0KGgo=',
            },
          ],
        },
      ],
      expect.any(Object),
    );
  });

  it('keeps arbitrary JSON object prompts as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"foo":"bar"}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });

  it('keeps arbitrary JSON prompts with unknown type fields as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"type":"custom_payload","value":"bar"}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });

  it('passes a single JSON message item to the SDK as structured agent input', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"role":"user","content":"Describe this request."}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(
      expect.any(Agent),
      [{ role: 'user', content: 'Describe this request.' }],
      expect.any(Object),
    );
  });

  it('keeps malformed role-shaped JSON prompts as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"role":"user","foo":"bar"}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });

  it('keeps malformed content-part JSON prompts as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"role":"user","content":[{"type":"input_text"}]}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });

  it('keeps malformed assistant JSON prompts without status as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"role":"assistant","content":[{"type":"output_text","text":"hello"}]}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });

  it('keeps malformed tool-call JSON prompts as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '{"type":"function_call"}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });

  it('passes SDK tool outputs, compaction, and unknown items as structured input', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Continue from prior items.',
        },
      },
    });
    const prompt = JSON.stringify([
      {
        type: 'shell_call_output',
        callId: 'shell-call-1',
        output: [
          {
            stdout: 'done',
            stderr: '',
            outcome: {
              type: 'exit',
              exitCode: 0,
            },
          },
        ],
      },
      {
        type: 'apply_patch_call_output',
        callId: 'patch-call-1',
        status: 'completed',
      },
      {
        type: 'compaction',
        encrypted_content: 'opaque-payload',
      },
      {
        type: 'unknown',
      },
    ]);

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(
      expect.any(Agent),
      [
        {
          type: 'shell_call_output',
          callId: 'shell-call-1',
          output: [
            {
              stdout: 'done',
              stderr: '',
              outcome: {
                type: 'exit',
                exitCode: 0,
              },
            },
          ],
        },
        {
          type: 'apply_patch_call_output',
          callId: 'patch-call-1',
          status: 'completed',
        },
        {
          type: 'compaction',
          encrypted_content: 'opaque-payload',
        },
        {
          type: 'unknown',
        },
      ],
      expect.any(Object),
    );
  });

  it('passes valid messages with empty content arrays as structured input', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Continue from prior messages.',
        },
      },
    });
    const prompt = '{"role":"user","content":[]}';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(
      expect.any(Agent),
      [{ role: 'user', content: [] }],
      expect.any(Object),
    );
  });

  it('keeps empty JSON arrays as plain text', async () => {
    const provider = new OpenAiAgentsProvider('gpt-5-mini', {
      config: {
        agent: {
          name: 'JSON Agent',
          instructions: 'Echo JSON.',
        },
      },
    });
    const prompt = '[]';

    await provider.callApi(prompt);

    expect(mockRun).toHaveBeenCalledWith(expect.any(Agent), prompt, expect.any(Object));
  });
});

describe('loadAgentDefinition', () => {
  it('rejects nullish agent configs with the standard validation error', async () => {
    await expect(loadAgentDefinition(null)).rejects.toThrow(
      'Invalid agent configuration: expected Agent instance, file:// URL, or inline definition',
    );
    await expect(loadAgentDefinition(undefined)).rejects.toThrow(
      'Invalid agent configuration: expected Agent instance, file:// URL, or inline definition',
    );
  });

  it('rejects array agent configs with the standard validation error', async () => {
    await expect(loadAgentDefinition([] as any)).rejects.toThrow(
      'Invalid agent configuration: expected Agent instance, file:// URL, or inline definition',
    );
  });

  it('creates inline sandbox agents with normalized manifests', async () => {
    const agent = await loadAgentDefinition({
      type: 'sandbox',
      name: 'Inline Workspace Agent',
      instructions: 'Inspect the workspace.',
      defaultManifest: {
        entries: {},
      },
    });

    expect(agent).toBeInstanceOf(SandboxAgent);
    expect((agent as SandboxAgent).defaultManifest).toBeInstanceOf(Manifest);
  });

  it('preserves inline agent handoff output warning settings', async () => {
    const agent = await loadAgentDefinition({
      name: 'Inline Agent',
      instructions: 'Help the user.',
      handoffOutputTypeWarningEnabled: false,
    });

    expect((agent as any).handoffOutputTypeWarningEnabled).toBe(false);
  });
});

describe('session and sandbox loaders', () => {
  it('creates built-in OpenAI session definitions', async () => {
    await expect(
      loadSessionDefinition({
        type: 'openai-conversations',
        conversationId: 'conv_123',
      }),
    ).resolves.toBeInstanceOf(OpenAIConversationsSession);

    await expect(
      loadSessionDefinition({
        type: 'openai-responses-compaction',
        client: {} as any,
        model: 'gpt-5-mini',
        underlyingSession: {
          type: 'memory',
          sessionId: 'session-1',
        },
      }),
    ).resolves.toBeInstanceOf(OpenAIResponsesCompactionSession);
  });

  it('creates built-in local and docker sandbox configs', async () => {
    await expect(
      loadSandboxConfig({
        type: 'unix-local',
      }),
    ).resolves.toMatchObject({
      client: expect.objectContaining({ backendId: 'unix_local' }),
    });

    await expect(
      loadSandboxConfig({
        type: 'docker',
        manifest: {
          entries: {},
        },
      }),
    ).resolves.toMatchObject({
      client: expect.objectContaining({ backendId: 'docker' }),
      manifest: expect.any(Manifest),
    });
  });
});