chore: import upstream snapshot with attribution

test/examples/integrationLangchain.test.ts

@@ -0,0 +1,143 @@
+import { spawnSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const EXAMPLE_PATH = path.join(
+  process.cwd(),
+  'examples',
+  'integration-langchain',
+  'langchain_example.py',
+);
+
+function findPythonPath(): string | undefined {
+  const probe = 'import sys; print(sys.executable)';
+  const candidates: Array<[string, string[]]> = [];
+
+  if (process.env.PROMPTFOO_PYTHON) {
+    candidates.push([process.env.PROMPTFOO_PYTHON, ['-c', probe]]);
+  }
+  if (process.platform === 'win32') {
+    candidates.push(['py', ['-3', '-c', probe]]);
+  }
+  candidates.push(['python3', ['-c', probe]], ['python', ['-c', probe]]);
+
+  for (const [command, args] of candidates) {
+    const result = spawnSync(command, args, { encoding: 'utf8', timeout: 5000 });
+    if (result.status === 0 && result.stdout.trim()) {
+      return result.stdout.trim();
+    }
+  }
+
+  return undefined;
+}
+
+const PYTHON_PATH = findPythonPath();
+
+// Skip these subprocess cases when no Python interpreter is available (CI always has one).
+const itPy = PYTHON_PATH ? it : it.skip;
+
+describe('integration-langchain example', () => {
+  let stubRoot: string;
+
+  beforeAll(() => {
+    const scratchRoot = path.join(process.cwd(), 'scratch', 'langchain-example-tests');
+    fs.mkdirSync(scratchRoot, { recursive: true });
+    stubRoot = fs.mkdtempSync(path.join(scratchRoot, 'case-'));
+
+    const langchainCore = path.join(stubRoot, 'langchain_core');
+    fs.mkdirSync(langchainCore, { recursive: true });
+    fs.writeFileSync(path.join(langchainCore, '__init__.py'), '');
+    fs.writeFileSync(
+      path.join(langchainCore, 'output_parsers.py'),
+      'class StrOutputParser:\n    pass\n',
+    );
+    fs.writeFileSync(
+      path.join(langchainCore, 'prompts.py'),
+      `
+import os
+
+class _FakeChain:
+    def __or__(self, _other):
+        return self
+
+    def invoke(self, _payload):
+        if os.getenv("PROMPTFOO_LANGCHAIN_STUB_ERROR"):
+            raise RuntimeError("stubbed invocation failure")
+        return "stubbed answer"
+
+class PromptTemplate:
+    @staticmethod
+    def from_template(_template):
+        return _FakeChain()
+`,
+    );
+    fs.writeFileSync(
+      path.join(stubRoot, 'langchain_openai.py'),
+      'class OpenAI:\n    def __init__(self, **_kwargs):\n        pass\n',
+    );
+  });
+
+  afterAll(() => {
+    fs.rmSync(stubRoot, { force: true, recursive: true });
+  });
+
+  function runExample(
+    args: string[],
+    envOverrides: NodeJS.ProcessEnv = {},
+  ): { status: number | null; stdout: string; stderr: string } {
+    if (!PYTHON_PATH) {
+      throw new Error('Python is not available');
+    }
+
+    const childEnv = { ...process.env };
+    delete childEnv.OPENAI_API_KEY;
+    delete childEnv.PROMPTFOO_LANGCHAIN_STUB_ERROR;
+    childEnv.PYTHONPATH = [stubRoot, childEnv.PYTHONPATH].filter(Boolean).join(path.delimiter);
+
+    const result = spawnSync(PYTHON_PATH, [EXAMPLE_PATH, ...args], {
+      encoding: 'utf8',
+      env: { ...childEnv, ...envOverrides },
+      timeout: 10_000,
+    });
+
+    return { status: result.status, stdout: result.stdout, stderr: result.stderr };
+  }
+
+  itPy('prints usage and exits non-zero when the question is missing', () => {
+    const result = runExample([]);
+
+    expect(result.status).toBe(1);
+    expect(result.stdout).toBe('');
+    expect(result.stderr).toContain('Usage:');
+    expect(result.stderr).toContain('<question>');
+  });
+
+  itPy('reports a missing API key without importing LangChain', () => {
+    const result = runExample(['What is 2 + 2?'], { PYTHONPATH: '' });
+
+    expect(result.status).toBe(1);
+    expect(result.stdout).toBe('');
+    expect(result.stderr.trim()).toBe('OPENAI_API_KEY environment variable is required.');
+  });
+
+  itPy('prints the LangChain result to stdout', () => {
+    const result = runExample(['What is 2 + 2?'], { OPENAI_API_KEY: 'test-key' });
+
+    expect(result.status).toBe(0);
+    expect(result.stdout.trim()).toBe('stubbed answer');
+    expect(result.stderr).toBe('');
+  });
+
+  itPy('reports invocation failures on stderr and exits non-zero', () => {
+    const result = runExample(['What is 2 + 2?'], {
+      OPENAI_API_KEY: 'test-key',
+      PROMPTFOO_LANGCHAIN_STUB_ERROR: 'true',
+    });
+
+    expect(result.status).toBe(1);
+    expect(result.stdout).toBe('');
+    expect(result.stderr.trim()).toBe('Error invoking math chain: stubbed invocation failure');
+  });
+});

test/examples/readme.test.ts

@@ -0,0 +1,216 @@
+import fs from 'fs';
+import path from 'path';
+
+import { globSync } from 'glob';
+import { describe, expect, it } from 'vitest';
+
+const rootDir = path.join(__dirname, '../..');
+const examplesDir = path.join(rootDir, 'examples');
+
+const readmeFiles = globSync('**/README.md', { cwd: examplesDir }).sort();
+
+function getExampleName(readmePath: string): string {
+  return path.dirname(readmePath).split(path.sep).join('/');
+}
+
+function isLeafExample(dirPath: string): boolean {
+  return globSync('promptfooconfig.*', { cwd: dirPath }).length > 0;
+}
+
+function hasSubExamples(dirPath: string): boolean {
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+  return entries.some(
+    (e) => e.isDirectory() && fs.existsSync(path.join(dirPath, e.name, 'README.md')),
+  );
+}
+
+function getSubExampleDirs(dirPath: string): string[] {
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+  return entries
+    .filter((e) => e.isDirectory() && fs.existsSync(path.join(dirPath, e.name, 'README.md')))
+    .map((e) => e.name);
+}
+
+/**
+ * Parse code fences, returning opening fences with their line numbers and language.
+ * Carefully distinguishes opening vs closing fences.
+ */
+function getCodeFences(content: string): Array<{ line: number; lang: string | null }> {
+  const lines = content.split('\n');
+  const fences: Array<{ line: number; lang: string | null }> = [];
+  let inCodeBlock = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    if (trimmed.startsWith('```')) {
+      if (inCodeBlock) {
+        inCodeBlock = false;
+      } else {
+        const lang = trimmed.slice(3).trim() || null;
+        fences.push({ line: i + 1, lang });
+        inCodeBlock = true;
+      }
+    }
+  }
+  return fences;
+}
+
+/**
+ * Get H1 headings, excluding those inside code blocks.
+ */
+function getH1Lines(content: string): Array<{ line: number; text: string }> {
+  const lines = content.split('\n');
+  const h1s: Array<{ line: number; text: string }> = [];
+  let inCodeBlock = false;
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    if (trimmed.startsWith('```')) {
+      inCodeBlock = !inCodeBlock;
+      continue;
+    }
+    if (!inCodeBlock && /^# /.test(trimmed)) {
+      h1s.push({ line: i + 1, text: trimmed });
+    }
+  }
+  return h1s;
+}
+
+/**
+ * Extract backtick-quoted filenames that look like local file references.
+ * Only captures references OUTSIDE of code blocks to avoid false positives
+ * from illustrative YAML/code examples.
+ */
+function getFileReferences(content: string): string[] {
+  const pattern = /`([\w][\w.-]*\.(yaml|yml|json|js|ts|py|txt|csv|xlsx))`/g;
+  const refs = new Set<string>();
+
+  // Strip code blocks first to avoid false positives from illustrative examples
+  const strippedContent = content.replace(/```[\s\S]*?```/g, '');
+
+  let match;
+  while ((match = pattern.exec(strippedContent)) !== null) {
+    const filename = match[1];
+    // Skip things that look like package names, URLs, or glob patterns
+    if (filename.includes('*') || filename.includes('/')) {
+      continue;
+    }
+    refs.add(filename);
+  }
+  return [...refs];
+}
+
+// Known file references that appear illustratively in READMEs or are generated at runtime
+const FILE_REFERENCE_ALLOWLIST = new Set([
+  'output.json',
+  'output.html',
+  'output.csv',
+  'output.yaml',
+  'results.json',
+  'redteam.yaml',
+  '.env',
+  '.env.example',
+  'package.json',
+  'package-lock.json',
+  'requirements.txt',
+  'pyproject.toml',
+  '.gitignore',
+  'tsconfig.json',
+]);
+
+describe('Example README standards', () => {
+  it('should find README files', () => {
+    expect(readmeFiles.length).toBeGreaterThan(200);
+  });
+
+  describe.each(readmeFiles)('%s', (relativePath) => {
+    const fullPath = path.join(examplesDir, relativePath);
+    const dirPath = path.dirname(fullPath);
+    const exampleName = getExampleName(relativePath);
+    const content = fs.readFileSync(fullPath, 'utf-8');
+    const isLeaf = isLeafExample(dirPath);
+    const isParent = hasSubExamples(dirPath);
+
+    it('should have correct H1 format: # <folder-name> (<Human Readable Name>)', () => {
+      const firstLine = content.split('\n')[0].trim();
+      expect(firstLine).toMatch(
+        new RegExp(`^# ${exampleName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} \\(.+\\)$`),
+      );
+    });
+
+    it('should have exactly one H1 heading', () => {
+      const h1s = getH1Lines(content);
+      expect(h1s.length).toBe(1);
+    });
+
+    if (isLeaf) {
+      it('should have init command with correct example name', () => {
+        const expectedInit = `npx promptfoo@latest init --example ${exampleName}`;
+        expect(content).toContain(expectedInit);
+      });
+
+      it('should have at least one H2 section', () => {
+        const lines = content.split('\n');
+        let inCodeBlock = false;
+        const hasH2 = lines.some((line) => {
+          const trimmed = line.trim();
+          if (trimmed.startsWith('```')) {
+            inCodeBlock = !inCodeBlock;
+            return false;
+          }
+          return !inCodeBlock && /^## /.test(trimmed);
+        });
+        expect(hasH2).toBe(true);
+      });
+    }
+
+    it('should have language specifiers on all code blocks', () => {
+      const fences = getCodeFences(content);
+      const bareFences = fences.filter((f) => f.lang === null);
+      if (bareFences.length > 0) {
+        const lineNumbers = bareFences.map((f) => f.line).join(', ');
+        expect(
+          bareFences,
+          `Code blocks without language specifiers at lines: ${lineNumbers}`,
+        ).toHaveLength(0);
+      }
+    });
+
+    it('should not reference files that do not exist in the directory', () => {
+      const refs = getFileReferences(content);
+      const missing = refs.filter((ref) => {
+        if (FILE_REFERENCE_ALLOWLIST.has(ref)) {
+          return false;
+        }
+        // Check in the directory itself
+        if (fs.existsSync(path.join(dirPath, ref))) {
+          return false;
+        }
+        // Also check in immediate subdirectories (files may live in sub-folders)
+        const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+        for (const entry of entries) {
+          if (entry.isDirectory() && fs.existsSync(path.join(dirPath, entry.name, ref))) {
+            return false;
+          }
+        }
+        return true;
+      });
+      if (missing.length > 0) {
+        expect(missing, `Referenced files not found: ${missing.join(', ')}`).toEqual([]);
+      }
+    });
+
+    if (isParent) {
+      it('should list sub-example directories', () => {
+        const subDirs = getSubExampleDirs(dirPath);
+        const missingRefs = subDirs.filter((dir) => !content.includes(dir));
+        if (missingRefs.length > 0) {
+          expect(
+            missingRefs,
+            `Sub-example directories not mentioned: ${missingRefs.join(', ')}`,
+          ).toEqual([]);
+        }
+      });
+    }
+  });
+});

test/examples/yaml-aliases.test.ts

@@ -0,0 +1,35 @@
+import fs from 'fs';
+import path from 'path';
+
+import { globSync } from 'glob';
+import { describe, expect, it } from 'vitest';
+import { parseDocument, visit } from 'yaml';
+
+const rootDir = path.join(__dirname, '../..');
+const exampleYamlFiles = globSync('examples/**/*.{yaml,yml}', {
+  cwd: rootDir,
+  absolute: true,
+}).sort();
+
+describe('Example YAML compatibility', () => {
+  it('keeps YAML alias count within SnakeYAML parser limits', () => {
+    // CodeQL's JavaScript extractor parses YAML with SnakeYAML, which defaults to a
+    // max of 50 aliases for non-scalar nodes.
+    const filesOverLimit = exampleYamlFiles
+      .map((file) => {
+        const doc = parseDocument(fs.readFileSync(file, 'utf8'));
+        let aliasCount = 0;
+
+        visit(doc, {
+          Alias() {
+            aliasCount += 1;
+          },
+        });
+
+        return { aliasCount, file: path.relative(rootDir, file) };
+      })
+      .filter(({ aliasCount }) => aliasCount > 50);
+
+    expect(filesOverLimit).toEqual([]);
+  });
+});