chore: import upstream snapshot with attribution

test/util/config/default.test.ts

@@ -0,0 +1,177 @@
+import path from 'path';
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { configCache, loadDefaultConfig } from '../../../src/util/config/default';
+import { maybeReadConfig } from '../../../src/util/config/load';
+
+vi.mock('../../../src/util/config/load', () => ({
+  maybeReadConfig: vi.fn(),
+}));
+
+describe('loadDefaultConfig', () => {
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.spyOn(process, 'cwd').mockImplementation(() => '/test/path');
+    configCache.clear();
+  });
+
+  it('should return empty config when no config file is found', async () => {
+    vi.mocked(maybeReadConfig).mockResolvedValue(undefined);
+
+    const result = await loadDefaultConfig();
+    expect(result).toEqual({
+      defaultConfig: {},
+      defaultConfigPath: undefined,
+    });
+    expect(maybeReadConfig).toHaveBeenCalledTimes(9);
+    expect(maybeReadConfig).toHaveBeenNthCalledWith(
+      1,
+      path.normalize('/test/path/promptfooconfig.yaml'),
+    );
+  });
+
+  it('should return the first valid config file found', async () => {
+    const mockConfig = { prompts: ['Some prompt'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig)
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce(mockConfig);
+
+    const result = await loadDefaultConfig();
+    expect(result).toEqual({
+      defaultConfig: mockConfig,
+      defaultConfigPath: path.normalize('/test/path/promptfooconfig.json'),
+    });
+    expect(maybeReadConfig).toHaveBeenCalledTimes(3);
+  });
+
+  it('should stop checking extensions after finding a valid config', async () => {
+    const mockConfig = { prompts: ['Some prompt'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(undefined).mockResolvedValueOnce(mockConfig);
+
+    await loadDefaultConfig();
+
+    expect(maybeReadConfig).toHaveBeenCalledTimes(2);
+    expect(maybeReadConfig).toHaveBeenNthCalledWith(
+      1,
+      path.normalize('/test/path/promptfooconfig.yaml'),
+    );
+    expect(maybeReadConfig).toHaveBeenNthCalledWith(
+      2,
+      path.normalize('/test/path/promptfooconfig.yml'),
+    );
+  });
+
+  it('should use provided directory when specified', async () => {
+    const mockConfig = { prompts: ['Some prompt'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(mockConfig);
+
+    const customDir = '/custom/directory';
+    const result = await loadDefaultConfig(customDir);
+    expect(result).toEqual({
+      defaultConfig: mockConfig,
+      defaultConfigPath: path.join(customDir, 'promptfooconfig.yaml'),
+    });
+    expect(maybeReadConfig).toHaveBeenCalledWith(path.join(customDir, 'promptfooconfig.yaml'));
+  });
+
+  it('should use custom config name when provided', async () => {
+    const mockConfig = { prompts: ['Custom config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(mockConfig);
+
+    const result = await loadDefaultConfig(undefined, 'redteam');
+    expect(result).toEqual({
+      defaultConfig: mockConfig,
+      defaultConfigPath: path.normalize('/test/path/redteam.yaml'),
+    });
+    expect(maybeReadConfig).toHaveBeenCalledWith(path.normalize('/test/path/redteam.yaml'));
+  });
+
+  it('should use different caches for different config names', async () => {
+    const mockConfig1 = { prompts: ['Config 1'], providers: [], tests: [] };
+    const mockConfig2 = { prompts: ['Config 2'], providers: [], tests: [] };
+
+    vi.mocked(maybeReadConfig)
+      .mockResolvedValueOnce(mockConfig1)
+      .mockResolvedValueOnce(mockConfig2);
+
+    const result1 = await loadDefaultConfig(undefined, 'promptfooconfig');
+    const result2 = await loadDefaultConfig(undefined, 'redteam');
+
+    expect(result1).not.toEqual(result2);
+    expect(result1.defaultConfig).toEqual(mockConfig1);
+    expect(result2.defaultConfig).toEqual(mockConfig2);
+
+    const cachedResult1 = await loadDefaultConfig(undefined, 'promptfooconfig');
+    const cachedResult2 = await loadDefaultConfig(undefined, 'redteam');
+
+    expect(cachedResult1).toEqual(result1);
+    expect(cachedResult2).toEqual(result2);
+    expect(maybeReadConfig).toHaveBeenCalledTimes(2);
+  });
+
+  it('should use different caches for different directories', async () => {
+    const mockConfig1 = { prompts: ['Config 1'], providers: [], tests: [] };
+    const mockConfig2 = { prompts: ['Config 2'], providers: [], tests: [] };
+
+    vi.mocked(maybeReadConfig)
+      .mockResolvedValueOnce(mockConfig1)
+      .mockResolvedValueOnce(mockConfig2);
+
+    const dir1 = '/dir1';
+    const dir2 = '/dir2';
+
+    const result1 = await loadDefaultConfig(dir1);
+    const result2 = await loadDefaultConfig(dir2);
+
+    expect(result1).not.toEqual(result2);
+    expect(result1.defaultConfig).toEqual(mockConfig1);
+    expect(result2.defaultConfig).toEqual(mockConfig2);
+
+    const cachedResult1 = await loadDefaultConfig(dir1);
+    const cachedResult2 = await loadDefaultConfig(dir2);
+
+    expect(cachedResult1).toEqual(result1);
+    expect(cachedResult2).toEqual(result2);
+    expect(maybeReadConfig).toHaveBeenCalledTimes(2);
+  });
+
+  it('should use cache for subsequent calls with same parameters', async () => {
+    const mockConfig = { prompts: ['Cached config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValueOnce(mockConfig);
+
+    const result1 = await loadDefaultConfig();
+    const result2 = await loadDefaultConfig();
+
+    expect(result1).toEqual(result2);
+    expect(maybeReadConfig).toHaveBeenCalledTimes(1);
+  });
+
+  it('should handle errors when reading config files', async () => {
+    vi.mocked(maybeReadConfig).mockRejectedValue(new Error('Permission denied'));
+
+    await expect(loadDefaultConfig()).rejects.toThrow('Permission denied');
+  });
+
+  it('should handle various config names', async () => {
+    const mockConfig = { prompts: ['Test config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValue(mockConfig);
+
+    const configNames = ['test1', 'test2', 'test3'];
+    for (const name of configNames) {
+      const result = await loadDefaultConfig(undefined, name);
+      expect(result.defaultConfigPath).toContain(name);
+    }
+  });
+
+  it('should handle interaction between configName and directory', async () => {
+    const mockConfig = { prompts: ['Combined config'], providers: [], tests: [] };
+    vi.mocked(maybeReadConfig).mockResolvedValue(mockConfig);
+
+    const customDir = '/custom/dir';
+    const customName = 'customconfig';
+    const result = await loadDefaultConfig(customDir, customName);
+
+    expect(result.defaultConfigPath).toEqual(path.join(customDir, `${customName}.yaml`));
+  });
+});

test/util/config/main.test.ts

@@ -0,0 +1,244 @@
+import * as os from 'os';
+import * as path from 'path';
+
+import * as yaml from 'js-yaml';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import cliState from '../../../src/cliState';
+import logger from '../../../src/logger';
+import {
+  getConfigDirectoryPath,
+  refreshConfigDirectoryPathFromEnv,
+  setConfigDirectoryPath,
+} from '../../../src/util/config/manage';
+import { writePromptfooConfig } from '../../../src/util/config/writer';
+import { mockProcessEnv } from '../../util/utils';
+
+import type { UnifiedConfig } from '../../../src/types/index';
+
+// Create hoisted mock functions for fs
+const mockFs = vi.hoisted(() => ({
+  existsSync: vi.fn(),
+  mkdirSync: vi.fn(),
+  writeFileSync: vi.fn(),
+  readFileSync: vi.fn(),
+  readdirSync: vi.fn(),
+  statSync: vi.fn(),
+  unlinkSync: vi.fn(),
+  rmdirSync: vi.fn(),
+  copyFileSync: vi.fn(),
+  renameSync: vi.fn(),
+  accessSync: vi.fn(),
+  chmodSync: vi.fn(),
+  constants: {},
+}));
+
+// Mock both 'fs' and 'node:fs' to cover all import patterns
+vi.mock('fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+vi.mock('node:fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+vi.mock('os');
+// js-yaml v5 is native ESM with a sealed namespace, so vi.spyOn cannot patch it.
+// Wrap dump in a spy-able mock that keeps the real implementation.
+vi.mock('js-yaml', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('js-yaml')>();
+  return { ...actual, dump: vi.fn(actual.dump) };
+});
+vi.mock('../../../src/logger', () => ({
+  default: {
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+  },
+}));
+
+describe('config', () => {
+  const mockHomedir = '/mock/home';
+  const defaultConfigPath = path.join(mockHomedir, '.promptfoo');
+  let restoreEnv: () => void;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(os.homedir).mockReturnValue(mockHomedir);
+    mockFs.existsSync.mockReturnValue(false);
+    restoreEnv = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: undefined });
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  afterEach(() => {
+    restoreEnv();
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  describe('getConfigDirectoryPath', () => {
+    it('returns default path when no custom path is set', () => {
+      expect(getConfigDirectoryPath()).toBe(defaultConfigPath);
+    });
+
+    it('reads a config directory refreshed after early env-file loading', () => {
+      const restoreConfigDir = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: '/env-file/config' });
+      try {
+        refreshConfigDirectoryPathFromEnv();
+        expect(getConfigDirectoryPath()).toBe('/env-file/config');
+      } finally {
+        restoreConfigDir();
+        refreshConfigDirectoryPathFromEnv();
+      }
+    });
+
+    it('does not let eval config env overrides move the global config directory', () => {
+      const originalConfig = cliState.config;
+      cliState.config = { env: { PROMPTFOO_CONFIG_DIR: '/eval-config/path' } };
+      try {
+        expect(getConfigDirectoryPath()).toBe(defaultConfigPath);
+      } finally {
+        cliState.config = originalConfig;
+      }
+    });
+
+    it('does not move after later process environment changes', () => {
+      const restoreConfigDir = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: '/late/config' });
+      try {
+        expect(getConfigDirectoryPath()).toBe(defaultConfigPath);
+      } finally {
+        restoreConfigDir();
+      }
+    });
+
+    it('does not create directory when createIfNotExists is false', () => {
+      getConfigDirectoryPath(false);
+      expect(mockFs.mkdirSync).not.toHaveBeenCalled();
+    });
+
+    // Note: The directory creation test cannot verify mkdirSync was called because
+    // the source uses require('fs') inside the function which bypasses Vitest's ESM mocking.
+    // We verify the function completes successfully with the correct return value instead.
+    it('handles createIfNotExists flag and returns correct path', () => {
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(defaultConfigPath);
+    });
+
+    it('does not create directory when it already exists', () => {
+      mockFs.existsSync.mockReturnValue(true);
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(defaultConfigPath);
+    });
+  });
+
+  describe('setConfigDirectoryPath', () => {
+    it('updates the config directory path', () => {
+      const newPath = '/new/config/path';
+      setConfigDirectoryPath(newPath);
+      expect(getConfigDirectoryPath()).toBe(newPath);
+    });
+
+    it('overrides the environment variable', () => {
+      const envPath = '/env/path';
+      const newPath = '/new/path';
+      const restoreConfigDir = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: envPath });
+      try {
+        setConfigDirectoryPath(newPath);
+        expect(getConfigDirectoryPath()).toBe(newPath);
+      } finally {
+        restoreConfigDir();
+      }
+    });
+  });
+});
+
+describe('writePromptfooConfig', () => {
+  const mockOutputPath = '/mock/output/path.yaml';
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+  });
+
+  it('writes a basic config to the specified path', () => {
+    const mockConfig: Partial<UnifiedConfig> = { description: 'Test config' };
+    const mockYaml = 'description: Test config\n';
+    vi.mocked(yaml.dump).mockReturnValue(mockYaml);
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    expect(mockFs.writeFileSync).toHaveBeenCalledWith(
+      mockOutputPath,
+      `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n${mockYaml}`,
+    );
+  });
+
+  it('orders the keys of the config correctly', () => {
+    const mockConfig: Partial<UnifiedConfig> = {
+      tests: [{ assert: [{ type: 'equals', value: 'test assertion' }] }],
+      description: 'Test config',
+      prompts: ['prompt1'],
+      providers: ['provider1'],
+      defaultTest: { assert: [{ type: 'equals', value: 'default assertion' }] },
+    };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    const dumpCall = vi.mocked(yaml.dump).mock.calls[0][0];
+    const keys = Object.keys(dumpCall);
+    expect(keys).toEqual(['description', 'prompts', 'providers', 'defaultTest', 'tests']);
+  });
+
+  it('uses js-yaml to dump the config with skipInvalid option', () => {
+    const mockConfig: Partial<UnifiedConfig> = { description: 'Test config' };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    expect(yaml.dump).toHaveBeenCalledWith(expect.anything(), { skipInvalid: true });
+  });
+
+  it('handles empty config', () => {
+    vi.mocked(yaml.dump).mockReturnValueOnce('');
+    const mockConfig: Partial<UnifiedConfig> = {};
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+    expect(mockFs.writeFileSync).not.toHaveBeenCalled();
+    expect(logger.warn).toHaveBeenCalledWith('Warning: config is empty, skipping write');
+  });
+
+  it('preserves all fields of the UnifiedConfig', () => {
+    const mockConfig: Partial<UnifiedConfig> = {
+      description: 'Full config test',
+      prompts: ['prompt1', 'prompt2'],
+      providers: ['provider1', 'provider2'],
+      defaultTest: { assert: [{ type: 'equals', value: 'default assertion' }] },
+      tests: [
+        { assert: [{ type: 'equals', value: 'test assertion 1' }] },
+        { assert: [{ type: 'equals', value: 'test assertion 2' }] },
+      ],
+      outputPath: './output',
+    };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    const dumpCall = vi.mocked(yaml.dump).mock.calls[0][0];
+    expect(dumpCall).toEqual(expect.objectContaining(mockConfig));
+  });
+
+  it('handles config with undefined values', () => {
+    const mockConfig: Partial<UnifiedConfig> = {
+      description: 'Config with undefined',
+      prompts: undefined,
+      providers: ['provider1'],
+    };
+
+    writePromptfooConfig(mockConfig, mockOutputPath);
+
+    const dumpCall = vi.mocked(yaml.dump).mock.calls[0][0];
+    expect(dumpCall).toHaveProperty('description', 'Config with undefined');
+    expect(dumpCall).toHaveProperty('providers', ['provider1']);
+    expect(dumpCall).not.toHaveProperty('prompts');
+  });
+});

test/util/config/manage.test.ts

@@ -0,0 +1,173 @@
+import * as os from 'os';
+import * as path from 'path';
+
+import * as yaml from 'js-yaml';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+  getConfigDirectoryPath,
+  refreshConfigDirectoryPathFromEnv,
+  setConfigDirectoryPath,
+} from '../../../src/util/config/manage';
+import { writePromptfooConfig } from '../../../src/util/config/writer';
+import { mockProcessEnv } from '../../util/utils';
+
+// Create hoisted mock functions for fs
+const mockFs = vi.hoisted(() => ({
+  existsSync: vi.fn(),
+  mkdirSync: vi.fn(),
+  writeFileSync: vi.fn(),
+  readFileSync: vi.fn(),
+  readdirSync: vi.fn(),
+  statSync: vi.fn(),
+  unlinkSync: vi.fn(),
+  rmdirSync: vi.fn(),
+  copyFileSync: vi.fn(),
+  renameSync: vi.fn(),
+  accessSync: vi.fn(),
+  chmodSync: vi.fn(),
+  constants: {},
+}));
+
+// Mock both 'fs' and 'node:fs' to cover all import patterns
+vi.mock('fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+vi.mock('node:fs', () => ({
+  ...mockFs,
+  default: mockFs,
+}));
+
+// js-yaml v5 is native ESM with a sealed namespace, so vi.spyOn cannot patch it.
+// Wrap dump in a spy-able mock that keeps the real implementation.
+vi.mock('js-yaml', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('js-yaml')>();
+  return { ...actual, dump: vi.fn(actual.dump) };
+});
+
+// Mock os module
+vi.mock('os');
+vi.mock('../../../src/logger', () => ({
+  default: {
+    warn: vi.fn(),
+  },
+}));
+
+describe('config management', () => {
+  let restoreEnv: () => void;
+
+  beforeEach(() => {
+    vi.resetAllMocks();
+    vi.mocked(os.homedir).mockReturnValue('/home/user');
+    restoreEnv = mockProcessEnv({ PROMPTFOO_CONFIG_DIR: undefined });
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  afterEach(() => {
+    restoreEnv();
+    refreshConfigDirectoryPathFromEnv();
+    setConfigDirectoryPath(undefined);
+  });
+
+  describe('getConfigDirectoryPath', () => {
+    it('should return default config path when no custom path set', () => {
+      setConfigDirectoryPath(undefined);
+      const configPath = getConfigDirectoryPath();
+      expect(configPath).toBe(path.join('/home/user', '.promptfoo'));
+    });
+
+    // Note: The directory creation test cannot verify mkdirSync was called because
+    // the source uses require('fs') inside the function which bypasses Vitest's ESM mocking.
+    // We verify the function completes successfully with the correct return value instead.
+    it('should handle createIfNotExists flag and return correct path', () => {
+      mockFs.existsSync.mockReturnValue(false);
+      setConfigDirectoryPath(undefined);
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(path.join('/home/user', '.promptfoo'));
+    });
+
+    it('should not create directory if it already exists', () => {
+      mockFs.existsSync.mockReturnValue(true);
+      const result = getConfigDirectoryPath(true);
+      expect(result).toBe(path.join('/home/user', '.promptfoo'));
+    });
+  });
+
+  describe('setConfigDirectoryPath', () => {
+    it('should set custom config directory path', () => {
+      const customPath = '/custom/path';
+      setConfigDirectoryPath(customPath);
+      const configPath = getConfigDirectoryPath();
+      expect(configPath).toBe(customPath);
+    });
+
+    it('should handle undefined path', () => {
+      setConfigDirectoryPath(undefined);
+      const configPath = getConfigDirectoryPath();
+      expect(configPath).toBe(path.join('/home/user', '.promptfoo'));
+    });
+  });
+
+  describe('writePromptfooConfig', () => {
+    const outputPath = 'config.yaml';
+
+    it('should write config with schema comment', () => {
+      const config = {
+        description: 'test config',
+        prompts: ['prompt1'],
+      };
+
+      writePromptfooConfig(config, outputPath);
+
+      expect(mockFs.writeFileSync).toHaveBeenCalledWith(
+        outputPath,
+        `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n${yaml.dump(config)}`,
+      );
+    });
+
+    it('should write config with header comments', () => {
+      const config = {
+        description: 'test config',
+      };
+      const headerComments = ['Comment 1', 'Comment 2'];
+
+      writePromptfooConfig(config, outputPath, headerComments);
+
+      const expectedContent =
+        `# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json\n` +
+        `# Comment 1\n# Comment 2\n${yaml.dump(config)}`;
+
+      expect(mockFs.writeFileSync).toHaveBeenCalledWith(outputPath, expectedContent);
+    });
+
+    it('should handle empty config', () => {
+      const config = {};
+      const result = writePromptfooConfig(config, outputPath);
+      expect(result).toEqual({});
+    });
+
+    it('should order config keys', () => {
+      const config = {
+        tests: ['test1'],
+        description: 'desc',
+        prompts: ['prompt1'],
+      };
+
+      const result = writePromptfooConfig(config, outputPath);
+
+      expect(Object.keys(result)[0]).toBe('description');
+      expect(Object.keys(result)[1]).toBe('prompts');
+      expect(Object.keys(result)[2]).toBe('tests');
+    });
+
+    it('should handle empty yaml content', () => {
+      vi.mocked(yaml.dump).mockReturnValueOnce('');
+      const config = { description: 'test' };
+      const result = writePromptfooConfig(config, outputPath);
+      expect(result).toEqual(config);
+      expect(mockFs.writeFileSync).not.toHaveBeenCalled();
+    });
+  });
+});

test/util/config/scenarios.test.ts

@@ -0,0 +1,239 @@
+import * as fs from 'fs';
+import * as fsPromises from 'fs/promises';
+
+import { globSync } from 'glob';
+import * as yaml from 'js-yaml';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { validateAssertions } from '../../../src/assertions/validateAssertions';
+import cliState from '../../../src/cliState';
+import { readPrompts, readProviderPromptMap } from '../../../src/prompts/index';
+import { loadApiProviders } from '../../../src/providers/index';
+import { type TestCase } from '../../../src/types/index';
+// Import after mocking
+import { resolveConfigs } from '../../../src/util/config/load';
+import { maybeLoadFromExternalFile } from '../../../src/util/file';
+import { readFilters } from '../../../src/util/index';
+import { readTests } from '../../../src/util/testCaseReader';
+import { createMockProvider } from '../../factories/provider';
+
+vi.mock('fs');
+vi.mock('fs/promises');
+vi.mock('glob', () => ({
+  globSync: vi.fn(),
+  hasMagic: vi.fn((pattern: string | string[]) => {
+    const p = Array.isArray(pattern) ? pattern.join('') : pattern;
+    return p.includes('*') || p.includes('?') || p.includes('[') || p.includes('{');
+  }),
+}));
+
+// Mock all the dependencies first
+vi.mock('../../../src/util/file', async () => {
+  const actual =
+    await vi.importActual<typeof import('../../../src/util/file')>('../../../src/util/file');
+  return {
+    ...actual,
+    maybeLoadFromExternalFile: vi.fn(),
+  };
+});
+vi.mock('../../../src/prompts');
+vi.mock('../../../src/providers');
+vi.mock('../../../src/util/testCaseReader');
+vi.mock('../../../src/util', async () => {
+  const actual = await vi.importActual<typeof import('../../../src/util')>('../../../src/util');
+  return {
+    ...actual,
+    readFilters: vi.fn(),
+  };
+});
+vi.mock('../../../src/assertions/validateAssertions');
+
+describe('Scenario loading with glob patterns', () => {
+  const originalBasePath = cliState.basePath;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    cliState.basePath = '/test/path';
+
+    // Setup default mocks
+    vi.mocked(readPrompts).mockResolvedValue([{ raw: 'Test prompt', label: 'Test prompt' }]);
+    vi.mocked(readProviderPromptMap).mockReturnValue({});
+    vi.mocked(loadApiProviders).mockResolvedValue([
+      createMockProvider({ id: 'openai:gpt-3.5-turbo' }),
+    ]);
+    vi.mocked(readTests).mockImplementation(async (tests) =>
+      Array.isArray(tests) ? (tests as TestCase[]) : [],
+    );
+    vi.mocked(readFilters).mockResolvedValue({});
+    vi.mocked(validateAssertions).mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    cliState.basePath = originalBasePath;
+  });
+
+  it('should flatten scenarios when loaded with glob patterns', async () => {
+    const scenario1 = {
+      description: 'Scenario 1',
+      config: [{ vars: { name: 'Alice' } }],
+      tests: [{ vars: { question: 'Test 1' } }],
+    };
+
+    const scenario2 = {
+      description: 'Scenario 2',
+      config: [{ vars: { name: 'Bob' } }],
+      tests: [{ vars: { question: 'Test 2' } }],
+    };
+
+    // Mock file system
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fsPromises.readFile).mockImplementation((filePath) => {
+      if (filePath === 'config.yaml') {
+        return Promise.resolve(
+          yaml.dump({
+            prompts: ['Test prompt'],
+            providers: ['openai:gpt-3.5-turbo'],
+            scenarios: ['file://scenarios/*.yaml'],
+          }),
+        );
+      }
+      return Promise.resolve('');
+    });
+
+    // Mock glob to return config file
+    vi.mocked(globSync).mockReturnValue(['config.yaml']);
+
+    // Mock maybeLoadFromExternalFile to return nested array (simulating glob expansion)
+    vi.mocked(maybeLoadFromExternalFile).mockImplementation((input) => {
+      if (Array.isArray(input) && input[0] === 'file://scenarios/*.yaml') {
+        // Return nested array as would happen with glob pattern
+        return [[scenario1, scenario2]];
+      }
+      return input;
+    });
+
+    const cmdObj = { config: ['config.yaml'] };
+    const defaultConfig = {};
+
+    const { testSuite } = await resolveConfigs(cmdObj, defaultConfig);
+
+    // Check if maybeLoadFromExternalFile was called with the expected argument
+    expect(maybeLoadFromExternalFile).toHaveBeenCalledWith(['file://scenarios/*.yaml']);
+
+    // Verify scenarios are flattened correctly
+    expect(testSuite.scenarios).toHaveLength(2);
+    expect(testSuite.scenarios![0]).toEqual(scenario1);
+    expect(testSuite.scenarios![1]).toEqual(scenario2);
+  });
+
+  it('should handle multiple scenario files with glob patterns', async () => {
+    const scenarios = [
+      {
+        description: 'Scenario A',
+        config: [{ vars: { test: 'A' } }],
+        tests: [{ vars: { input: '1' } }],
+      },
+      {
+        description: 'Scenario B',
+        config: [{ vars: { test: 'B' } }],
+        tests: [{ vars: { input: '2' } }],
+      },
+      {
+        description: 'Scenario C',
+        config: [{ vars: { test: 'C' } }],
+        tests: [{ vars: { input: '3' } }],
+      },
+    ];
+
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fsPromises.readFile).mockImplementation((filePath) => {
+      if (filePath === 'config.yaml') {
+        return Promise.resolve(
+          yaml.dump({
+            prompts: ['Test prompt'],
+            providers: ['openai:gpt-3.5-turbo'],
+            scenarios: ['file://group1/*.yaml', 'file://group2/*.yaml'],
+          }),
+        );
+      }
+      return Promise.resolve('');
+    });
+
+    vi.mocked(globSync).mockReturnValue(['config.yaml']);
+
+    vi.mocked(maybeLoadFromExternalFile).mockImplementation((input) => {
+      if (Array.isArray(input)) {
+        // Simulate two glob patterns each returning different scenarios
+        return [
+          [scenarios[0], scenarios[1]], // group1/*.yaml
+          [scenarios[2]], // group2/*.yaml
+        ];
+      }
+      return input;
+    });
+
+    const cmdObj = { config: ['config.yaml'] };
+    const defaultConfig = {};
+
+    const { testSuite } = await resolveConfigs(cmdObj, defaultConfig);
+
+    // Verify all scenarios are flattened into a single array
+    expect(testSuite.scenarios).toHaveLength(3);
+    expect(testSuite.scenarios).toEqual(scenarios);
+  });
+
+  it('should handle mixed scenario loading (direct and glob)', async () => {
+    const directScenario = {
+      description: 'Direct scenario',
+      config: [{ vars: { type: 'direct' } }],
+      tests: [{ vars: { test: 'direct' } }],
+    };
+
+    const globScenarios = [
+      {
+        description: 'Glob scenario 1',
+        config: [{ vars: { type: 'glob' } }],
+        tests: [{ vars: { test: 'glob1' } }],
+      },
+      {
+        description: 'Glob scenario 2',
+        config: [{ vars: { type: 'glob' } }],
+        tests: [{ vars: { test: 'glob2' } }],
+      },
+    ];
+
+    vi.mocked(fs.existsSync).mockReturnValue(true);
+    vi.mocked(fsPromises.readFile).mockImplementation((filePath) => {
+      if (filePath === 'config.yaml') {
+        return Promise.resolve(
+          yaml.dump({
+            prompts: ['Test prompt'],
+            providers: ['openai:gpt-3.5-turbo'],
+            scenarios: [directScenario, 'file://scenarios/*.yaml'],
+          }),
+        );
+      }
+      return Promise.resolve('');
+    });
+
+    vi.mocked(globSync).mockReturnValue(['config.yaml']);
+
+    vi.mocked(maybeLoadFromExternalFile).mockImplementation((input) => {
+      if (Array.isArray(input)) {
+        // First element is direct scenario, second is glob pattern
+        return [directScenario, globScenarios];
+      }
+      return input;
+    });
+
+    const cmdObj = { config: ['config.yaml'] };
+    const defaultConfig = {};
+
+    const { testSuite } = await resolveConfigs(cmdObj, defaultConfig);
+
+    // Verify mixed scenarios are flattened correctly
+    expect(testSuite.scenarios).toHaveLength(3);
+    expect(testSuite.scenarios![0]).toEqual(directScenario);
+    expect(testSuite.scenarios![1]).toEqual(globScenarios[0]);
+    expect(testSuite.scenarios![2]).toEqual(globScenarios[1]);
+  });
+});