chore: import upstream snapshot with attribution

test/commands/validate-exit-codes.test.ts

@@ -0,0 +1,188 @@
+import { Command } from 'commander';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { doValidate, validateCommand } from '../../src/commands/validate';
+import logger from '../../src/logger';
+import { ConfigResolutionError, resolveConfigs } from '../../src/util/config/load';
+
+import type { UnifiedConfig } from '../../src/types/index';
+
+vi.mock('../../src/logger');
+vi.mock('../../src/util/config/load', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('../../src/util/config/load')>()),
+  resolveConfigs: vi.fn(),
+}));
+vi.mock('../../src/telemetry', () => ({
+  default: {
+    record: vi.fn(),
+    send: vi.fn(),
+  },
+}));
+
+describe('Validate Command Exit Codes', () => {
+  let program: Command;
+  const defaultConfig = {} as UnifiedConfig;
+  const defaultConfigPath = 'config.yaml';
+
+  beforeEach(() => {
+    program = new Command();
+    vi.clearAllMocks();
+
+    // Reset exit code before each test
+    process.exitCode = 0;
+  });
+
+  afterEach(() => {
+    vi.mocked(resolveConfigs).mockReset();
+  });
+
+  describe('Success scenarios - should set exit code 0', () => {
+    it('should set exit code 0 when configuration is valid', async () => {
+      // Mock successful config resolution and validation
+      const mockValidConfig = {
+        prompts: ['test prompt'],
+        providers: ['test-provider'],
+        tests: [{ vars: { test: 'value' } }],
+      };
+
+      const mockValidTestSuite = {
+        prompts: [{ raw: 'test prompt', label: 'test' }],
+        providers: [{ id: () => 'test-provider', callApi: () => Promise.resolve({}) }],
+        tests: [{ vars: { test: 'value' } }],
+      };
+
+      vi.mocked(resolveConfigs).mockResolvedValue({
+        config: mockValidConfig as any,
+        testSuite: mockValidTestSuite as any,
+        basePath: '/test',
+      });
+
+      await doValidate({ config: ['test-config.yaml'] }, defaultConfig, defaultConfigPath);
+
+      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('Configuration is valid'));
+      expect(process.exitCode).toBe(0);
+    });
+
+    it('should set exit code 0 when validating with default config path', async () => {
+      const mockValidConfig = {
+        prompts: ['test prompt'],
+        providers: ['test-provider'],
+      };
+
+      const mockValidTestSuite = {
+        prompts: [{ raw: 'test prompt', label: 'test' }],
+        providers: [{ id: () => 'test-provider', callApi: () => Promise.resolve({}) }],
+      };
+
+      vi.mocked(resolveConfigs).mockResolvedValue({
+        config: mockValidConfig as any,
+        testSuite: mockValidTestSuite as any,
+        basePath: '/test',
+      });
+
+      await doValidate({}, defaultConfig, defaultConfigPath);
+
+      expect(logger.info).toHaveBeenCalledWith(expect.stringContaining('Configuration is valid'));
+      expect(process.exitCode).toBe(0);
+    });
+  });
+
+  describe('Failure scenarios - should set exit code 1', () => {
+    it('should set exit code 1 when configuration validation fails', async () => {
+      // Mock invalid config that fails schema validation
+      const mockInvalidConfig = {
+        // Missing required fields to trigger validation error
+        invalidField: 'invalid value',
+      };
+
+      const mockValidTestSuite = {
+        prompts: [{ raw: 'test prompt', label: 'test' }],
+        providers: [{ id: () => 'test-provider', callApi: () => Promise.resolve({}) }],
+      };
+
+      vi.mocked(resolveConfigs).mockResolvedValue({
+        config: mockInvalidConfig as any,
+        testSuite: mockValidTestSuite as any,
+        basePath: '/test',
+      });
+
+      await doValidate({ config: ['invalid-config.yaml'] }, defaultConfig, defaultConfigPath);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Configuration validation error'),
+      );
+      expect(process.exitCode).toBe(1);
+    });
+
+    it('should set exit code 1 when test suite validation fails', async () => {
+      // Mock valid config but invalid test suite
+      const mockValidConfig = {
+        prompts: ['test prompt'],
+        providers: ['test-provider'],
+      };
+
+      const mockInvalidTestSuite = {
+        // Invalid test suite structure to trigger validation error
+        prompts: 'invalid prompts format', // Should be an array
+        providers: [{ id: () => 'test-provider', callApi: () => Promise.resolve({}) }],
+      };
+
+      vi.mocked(resolveConfigs).mockResolvedValue({
+        config: mockValidConfig as any,
+        testSuite: mockInvalidTestSuite as any,
+        basePath: '/test',
+      });
+
+      await doValidate({ config: ['test-config.yaml'] }, defaultConfig, defaultConfigPath);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Test suite validation error'),
+      );
+      expect(process.exitCode).toBe(1);
+    });
+
+    it('should set exit code 1 when config resolution throws an error', async () => {
+      // Mock resolveConfigs to throw an error
+      vi.mocked(resolveConfigs).mockRejectedValue(new Error('Failed to load configuration'));
+
+      await doValidate({ config: ['non-existent-config.yaml'] }, defaultConfig, defaultConfigPath);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        expect.stringContaining('Failed to validate configuration: Failed to load configuration'),
+      );
+      expect(process.exitCode).toBe(1);
+    });
+
+    it('should prefix config resolution messages with the validate context', async () => {
+      vi.mocked(resolveConfigs).mockRejectedValue(
+        new ConfigResolutionError('You must provide at least 1 prompt'),
+      );
+
+      await doValidate({ config: ['invalid-config.yaml'] }, defaultConfig, defaultConfigPath);
+
+      expect(logger.error).toHaveBeenCalledWith(
+        'Failed to validate configuration: You must provide at least 1 prompt',
+      );
+      expect(process.exitCode).toBe(1);
+    });
+  });
+
+  describe('Command registration', () => {
+    it('should register validate command correctly', () => {
+      validateCommand(program, defaultConfig, defaultConfigPath);
+
+      const validateCmd = program.commands.find((cmd) => cmd.name() === 'validate');
+
+      expect(validateCmd).toBeDefined();
+      expect(validateCmd?.name()).toBe('validate');
+      expect(validateCmd?.description()).toBe('Validate configuration files and test providers');
+
+      // Check that the config subcommand is registered
+      const configSubCmd = validateCmd?.commands.find((cmd) => cmd.name() === 'config');
+      expect(configSubCmd).toBeDefined();
+
+      // Check that the config option is registered on the config subcommand
+      const configOption = configSubCmd?.options.find((opt) => opt.long === '--config');
+      expect(configOption).toBeDefined();
+    });
+  });
+});