chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:59:42 +08:00
commit 59f8f60dad
348 changed files with 139133 additions and 0 deletions
+38
View File
@@ -0,0 +1,38 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
temp
*.jar
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
# turbo
.turbo
# tsconfig
tsconfig.tsbuildinfo
# docs
THIRD_PARTY
NOTICE
README.md
LICENSE
View File
+7
View File
@@ -0,0 +1,7 @@
{
"singleQuote": true,
"trailingComma": "all",
"tabWidth": 2,
"semi": true,
"printWidth": 100
}
+32
View File
@@ -0,0 +1,32 @@
import eslint from "@eslint/js";
import tseslint from "@typescript-eslint/eslint-plugin";
import tsparser from "@typescript-eslint/parser";
import globals from "globals";
export default [
eslint.configs.recommended,
{
files: ["src/**/*.ts"],
languageOptions: {
globals: {
...globals.node,
},
parser: tsparser,
parserOptions: {
ecmaVersion: "latest",
sourceType: "module",
},
},
plugins: {
"@typescript-eslint": tseslint,
},
rules: {
...tseslint.configs.recommended.rules,
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": "warn",
},
},
{
ignores: ["dist/**", "lib/**", "node_modules/**"],
},
];
+86
View File
@@ -0,0 +1,86 @@
{
"name": "@opendataloader/pdf",
"version": "0.0.0",
"description": "A Node.js wrapper for the opendataloader-pdf Java CLI.",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"type": "module",
"bin": {
"opendataloader-pdf": "./dist/cli.js"
},
"exports": {
".": {
"import": "./dist/index.js",
"require": "./dist/index.cjs"
}
},
"scripts": {
"setup": "node ./scripts/setup.cjs",
"build": "pnpm run setup && tsup",
"test": "vitest --run",
"format": "prettier --write \"**/*.{ts,js,json,md}\"",
"lint": "eslint \"src/**/*.ts\"",
"lint:fix": "eslint \"src/**/*.ts\" --fix"
},
"repository": {
"type": "git",
"url": "git+https://github.com/opendataloader-project/opendataloader-pdf.git"
},
"keywords": [
"pdf",
"markdown",
"html",
"convert",
"pdf-convert",
"pdf-parser",
"pdf-parsing",
"pdf-to-json",
"pdf-to-markdown",
"pdf-to-html"
],
"author": "opendataloader-project",
"license": "Apache-2.0",
"bugs": {
"url": "https://github.com/opendataloader-project/opendataloader-pdf/issues"
},
"homepage": "https://github.com/opendataloader-project/opendataloader-pdf#readme",
"engines": {
"node": ">=20.19.0"
},
"publishConfig": {
"access": "public"
},
"dependencies": {
"commander": "^15.0.0"
},
"devDependencies": {
"@eslint/js": "^10.0.1",
"@types/node": "^26.0.0",
"@typescript-eslint/eslint-plugin": "^8.58.2",
"@typescript-eslint/parser": "^8.58.2",
"eslint": "^10.2.0",
"glob": "^13.0.6",
"globals": "^17.5.0",
"prettier": "^3.8.3",
"tsup": "^8.5.1",
"typescript": "^6.0.2",
"vite": "^8.0.8",
"vitest": "^4.1.4"
},
"pnpm": {
"overrides": {
"minimatch@<10.2.3": ">=10.2.3",
"flatted@<3.4.2": ">=3.4.2",
"postcss@<8.5.10": ">=8.5.10"
}
},
"files": [
"dist",
"lib",
"LICENSE",
"NOTICE",
"README.md",
"THIRD_PARTY"
]
}
File diff suppressed because it is too large Load Diff
+62
View File
@@ -0,0 +1,62 @@
const fs = require('fs');
const path = require('path');
const { globSync } = require('glob');
const rootDir = path.resolve(__dirname, '..');
const javaDir = path.resolve(rootDir, '../../java');
const sourceJarGlob = path
.join(javaDir, 'opendataloader-pdf-cli/target/opendataloader-pdf-cli-*.jar')
.replace(/\\/g, '/');
console.log(`Searching for JAR file in: ${sourceJarGlob}`);
const sourceJarPaths = globSync(sourceJarGlob);
if (sourceJarPaths.length === 0) {
console.error(
"Could not find the JAR file. Please run 'mvn package' in the 'java/' directory first.",
);
process.exit(1);
}
if (sourceJarPaths.length > 1) {
console.error(`Found multiple JAR files, expected one: ${sourceJarPaths}`);
process.exit(1);
}
const sourceJarPath = sourceJarPaths[0];
console.log(`Found source JAR: ${sourceJarPath}`);
const destJarDir = path.join(rootDir, 'lib').replace(/\\/g, '/');
if (!fs.existsSync(destJarDir)) {
fs.mkdirSync(destJarDir, { recursive: true });
}
const destJarPath = path.join(destJarDir, 'opendataloader-pdf-cli.jar').replace(/\\/g, '/');
console.log(`Copying JAR to ${destJarPath}`);
fs.copyFileSync(sourceJarPath, destJarPath);
// Copy README.md, LICENSE, NOTICE, and THIRD_PARTY
const readmeSrc = path.resolve(rootDir, '../../README.md');
const licenseSrc = path.resolve(rootDir, '../../LICENSE');
const noticeSrc = path.resolve(rootDir, '../../NOTICE');
const thirdPartySrc = path.resolve(rootDir, '../../THIRD_PARTY');
const readmeDest = path.join(rootDir, 'README.md').replace(/\\/g, '/');
const licenseDest = path.join(rootDir, 'LICENSE').replace(/\\/g, '/');
const noticeDest = path.join(rootDir, 'NOTICE').replace(/\\/g, '/');
const thirdPartyDest = path.join(rootDir, 'THIRD_PARTY').replace(/\\/g, '/');
console.log(`Copying README.md to ${readmeDest}`);
fs.copyFileSync(readmeSrc, readmeDest);
console.log(`Copying LICENSE to ${licenseDest}`);
fs.copyFileSync(licenseSrc, licenseDest);
console.log(`Copying NOTICE to ${noticeDest}`);
fs.copyFileSync(noticeSrc, noticeDest);
console.log(`Copying THIRD_PARTY directory to ${thirdPartyDest}`);
if (fs.existsSync(thirdPartyDest)) {
fs.rmSync(thirdPartyDest, { recursive: true, force: true });
}
fs.cpSync(thirdPartySrc, thirdPartyDest, { recursive: true });
console.log('Package preparation complete.');
@@ -0,0 +1,41 @@
// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
// Run `npm run generate-options` to regenerate
import { Command } from 'commander';
/**
* Register all CLI options on the given Commander program.
*/
export function registerCliOptions(program: Command): void {
program.option('-o, --output-dir <value>', 'Directory where output files are written. Default: input file directory');
program.option('-p, --password <value>', 'Password for encrypted PDF files');
program.option('-f, --format <value>', 'Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output.');
program.option('-q, --quiet', 'Suppress console logging output');
program.option('--content-safety-off <value>', 'Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg');
program.option('--sanitize', 'Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders');
program.option('--keep-line-breaks', 'Preserve original line breaks in extracted text');
program.option('--replace-invalid-chars <value>', 'Replacement character for invalid/unrecognized characters. Default: space');
program.option('--use-struct-tree', 'Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality');
program.option('--table-method <value>', 'Table detection method. Values: default (border-based), cluster (border + cluster). Default: default');
program.option('--reading-order <value>', 'Reading order algorithm. Values: off, xycut. Default: xycut');
program.option('--markdown-page-separator <value>', 'Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none');
program.option('--markdown-with-html', 'Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown.');
program.option('--text-page-separator <value>', 'Separator between pages in text output. Use %page-number% for page numbers. Default: none');
program.option('--html-page-separator <value>', 'Separator between pages in HTML output. Use %page-number% for page numbers. Default: none');
program.option('--image-output <value>', 'Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external');
program.option('--image-format <value>', 'Output format for extracted images. Values: png, jpeg. Default: png');
program.option('--image-dir <value>', 'Directory for extracted images (applies only with --image-output external)');
program.option('--pages <value>', 'Pages to extract (e.g., "1,3,5-7"). Default: all pages');
program.option('--include-header-footer', 'Include page headers and footers in output');
program.option('--detect-strikethrough', 'Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)');
program.option('--hybrid <value>', 'Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai');
program.option('--hybrid-mode <value>', 'Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)');
program.option('--hybrid-url <value>', 'Hybrid backend server URL (overrides default)');
program.option('--hybrid-timeout <value>', 'Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0');
program.option('--hybrid-fallback', 'Opt in to Java fallback on hybrid backend error (default: disabled)');
program.option('--hybrid-hancom-ai-regionlist-strategy <value>', 'DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)');
program.option('--hybrid-hancom-ai-ocr-strategy <value>', 'OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)');
program.option('--hybrid-hancom-ai-image-cache <value>', 'Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk');
program.option('--to-stdout', 'Write output to stdout instead of file (single format only)');
program.option('--threads <value>', 'Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode');
}
+83
View File
@@ -0,0 +1,83 @@
#!/usr/bin/env node
import { Command, CommanderError } from 'commander';
import { _runForCli } from './index.js';
import { CliOptions, buildConvertOptions } from './convert-options.generated.js';
import { registerCliOptions } from './cli-options.generated.js';
function createProgram(): Command {
const program = new Command();
program
.name('opendataloader-pdf')
.usage('[options] <input...>')
.description('Convert PDFs using the OpenDataLoader CLI.')
.showHelpAfterError("Use '--help' to see available options.")
.showSuggestionAfterError(false)
.argument('<input...>', 'Input files or directories to convert');
// Register CLI options from auto-generated file
registerCliOptions(program);
program.configureOutput({
writeErr: (str) => {
console.error(str.trimEnd());
},
outputError: (str, write) => {
write(str);
},
});
return program;
}
async function main(): Promise<number> {
const program = createProgram();
program.exitOverride();
try {
program.parse(process.argv);
} catch (err) {
if (err instanceof CommanderError) {
if (err.code === 'commander.helpDisplayed') {
return 0;
}
return err.exitCode ?? 1;
}
const message = err instanceof Error ? err.message : String(err);
console.error(message);
console.error("Use '--help' to see available options.");
return 1;
}
const cliOptions = program.opts<CliOptions>();
const inputPaths = program.args;
const convertOptions = buildConvertOptions(cliOptions);
try {
// _runForCli streams stdout/stderr to the parent process as they arrive;
// we deliberately do not re-print anything here. (Issue #398.)
await _runForCli(inputPaths, convertOptions);
return 0;
} catch (err) {
// Subprocess-exit errors are already on the user's terminal via the live
// stderr stream — re-printing would duplicate output and risk leaking
// anything sensitive Java logged (e.g. a --password value echoed by an
// underlying library). Wrapper-side failures (JAR not found, java not in
// PATH, bad input path) still need to be surfaced.
const isJavaExit =
err instanceof Error && (err as Error & { isJavaExit?: boolean }).isJavaExit === true;
if (!isJavaExit) {
const message = err instanceof Error ? err.message : String(err);
console.error(message);
}
return 1;
}
}
main().then((code) => {
if (code !== 0) {
process.exit(code);
}
});
@@ -0,0 +1,325 @@
// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
// Run `npm run generate-options` to regenerate
/**
* Options for the convert function.
*/
export interface ConvertOptions {
/** Directory where output files are written. Default: input file directory */
outputDir?: string;
/** Password for encrypted PDF files */
password?: string;
/** Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output. */
format?: string | string[];
/** Suppress console logging output */
quiet?: boolean;
/** Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg */
contentSafetyOff?: string | string[];
/** Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders */
sanitize?: boolean;
/** Preserve original line breaks in extracted text */
keepLineBreaks?: boolean;
/** Replacement character for invalid/unrecognized characters. Default: space */
replaceInvalidChars?: string;
/** Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality */
useStructTree?: boolean;
/** Table detection method. Values: default (border-based), cluster (border + cluster). Default: default */
tableMethod?: string;
/** Reading order algorithm. Values: off, xycut. Default: xycut */
readingOrder?: string;
/** Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none */
markdownPageSeparator?: string;
/** Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown. */
markdownWithHtml?: boolean;
/** Separator between pages in text output. Use %page-number% for page numbers. Default: none */
textPageSeparator?: string;
/** Separator between pages in HTML output. Use %page-number% for page numbers. Default: none */
htmlPageSeparator?: string;
/** Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external */
imageOutput?: string;
/** Output format for extracted images. Values: png, jpeg. Default: png */
imageFormat?: string;
/** Directory for extracted images (applies only with --image-output external) */
imageDir?: string;
/** Pages to extract (e.g., "1,3,5-7"). Default: all pages */
pages?: string;
/** Include page headers and footers in output */
includeHeaderFooter?: boolean;
/** Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental) */
detectStrikethrough?: boolean;
/** Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai */
hybrid?: string;
/** Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend) */
hybridMode?: string;
/** Hybrid backend server URL (overrides default) */
hybridUrl?: string;
/** Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0 */
hybridTimeout?: string;
/** Opt in to Java fallback on hybrid backend error (default: disabled) */
hybridFallback?: boolean;
/** DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list) */
hybridHancomAiRegionlistStrategy?: string;
/** OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only) */
hybridHancomAiOcrStrategy?: string;
/** Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk */
hybridHancomAiImageCache?: string;
/** Write output to stdout instead of file (single format only) */
toStdout?: boolean;
/** Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode */
threads?: string;
}
/**
* Options as parsed from CLI (all values are strings from commander).
*/
export interface CliOptions {
outputDir?: string;
password?: string;
format?: string;
quiet?: boolean;
contentSafetyOff?: string;
sanitize?: boolean;
keepLineBreaks?: boolean;
replaceInvalidChars?: string;
useStructTree?: boolean;
tableMethod?: string;
readingOrder?: string;
markdownPageSeparator?: string;
markdownWithHtml?: boolean;
textPageSeparator?: string;
htmlPageSeparator?: string;
imageOutput?: string;
imageFormat?: string;
imageDir?: string;
pages?: string;
includeHeaderFooter?: boolean;
detectStrikethrough?: boolean;
hybrid?: string;
hybridMode?: string;
hybridUrl?: string;
hybridTimeout?: string;
hybridFallback?: boolean;
hybridHancomAiRegionlistStrategy?: string;
hybridHancomAiOcrStrategy?: string;
hybridHancomAiImageCache?: string;
toStdout?: boolean;
threads?: string;
}
/**
* Convert CLI options to ConvertOptions.
*/
export function buildConvertOptions(cliOptions: CliOptions): ConvertOptions {
const convertOptions: ConvertOptions = {};
if (cliOptions.outputDir) {
convertOptions.outputDir = cliOptions.outputDir;
}
if (cliOptions.password) {
convertOptions.password = cliOptions.password;
}
if (cliOptions.format) {
convertOptions.format = cliOptions.format;
}
if (cliOptions.quiet) {
convertOptions.quiet = true;
}
if (cliOptions.contentSafetyOff) {
convertOptions.contentSafetyOff = cliOptions.contentSafetyOff;
}
if (cliOptions.sanitize) {
convertOptions.sanitize = true;
}
if (cliOptions.keepLineBreaks) {
convertOptions.keepLineBreaks = true;
}
if (cliOptions.replaceInvalidChars) {
convertOptions.replaceInvalidChars = cliOptions.replaceInvalidChars;
}
if (cliOptions.useStructTree) {
convertOptions.useStructTree = true;
}
if (cliOptions.tableMethod) {
convertOptions.tableMethod = cliOptions.tableMethod;
}
if (cliOptions.readingOrder) {
convertOptions.readingOrder = cliOptions.readingOrder;
}
if (cliOptions.markdownPageSeparator) {
convertOptions.markdownPageSeparator = cliOptions.markdownPageSeparator;
}
if (cliOptions.markdownWithHtml) {
convertOptions.markdownWithHtml = true;
}
if (cliOptions.textPageSeparator) {
convertOptions.textPageSeparator = cliOptions.textPageSeparator;
}
if (cliOptions.htmlPageSeparator) {
convertOptions.htmlPageSeparator = cliOptions.htmlPageSeparator;
}
if (cliOptions.imageOutput) {
convertOptions.imageOutput = cliOptions.imageOutput;
}
if (cliOptions.imageFormat) {
convertOptions.imageFormat = cliOptions.imageFormat;
}
if (cliOptions.imageDir) {
convertOptions.imageDir = cliOptions.imageDir;
}
if (cliOptions.pages) {
convertOptions.pages = cliOptions.pages;
}
if (cliOptions.includeHeaderFooter) {
convertOptions.includeHeaderFooter = true;
}
if (cliOptions.detectStrikethrough) {
convertOptions.detectStrikethrough = true;
}
if (cliOptions.hybrid) {
convertOptions.hybrid = cliOptions.hybrid;
}
if (cliOptions.hybridMode) {
convertOptions.hybridMode = cliOptions.hybridMode;
}
if (cliOptions.hybridUrl) {
convertOptions.hybridUrl = cliOptions.hybridUrl;
}
if (cliOptions.hybridTimeout) {
convertOptions.hybridTimeout = cliOptions.hybridTimeout;
}
if (cliOptions.hybridFallback) {
convertOptions.hybridFallback = true;
}
if (cliOptions.hybridHancomAiRegionlistStrategy) {
convertOptions.hybridHancomAiRegionlistStrategy = cliOptions.hybridHancomAiRegionlistStrategy;
}
if (cliOptions.hybridHancomAiOcrStrategy) {
convertOptions.hybridHancomAiOcrStrategy = cliOptions.hybridHancomAiOcrStrategy;
}
if (cliOptions.hybridHancomAiImageCache) {
convertOptions.hybridHancomAiImageCache = cliOptions.hybridHancomAiImageCache;
}
if (cliOptions.toStdout) {
convertOptions.toStdout = true;
}
if (cliOptions.threads) {
convertOptions.threads = cliOptions.threads;
}
return convertOptions;
}
/**
* Build CLI arguments array from ConvertOptions.
*/
export function buildArgs(options: ConvertOptions): string[] {
const args: string[] = [];
if (options.outputDir) {
args.push('--output-dir', options.outputDir);
}
if (options.password) {
args.push('--password', options.password);
}
if (options.format) {
if (Array.isArray(options.format)) {
if (options.format.length > 0) {
args.push('--format', options.format.join(','));
}
} else {
args.push('--format', options.format);
}
}
if (options.quiet) {
args.push('--quiet');
}
if (options.contentSafetyOff) {
if (Array.isArray(options.contentSafetyOff)) {
if (options.contentSafetyOff.length > 0) {
args.push('--content-safety-off', options.contentSafetyOff.join(','));
}
} else {
args.push('--content-safety-off', options.contentSafetyOff);
}
}
if (options.sanitize) {
args.push('--sanitize');
}
if (options.keepLineBreaks) {
args.push('--keep-line-breaks');
}
if (options.replaceInvalidChars) {
args.push('--replace-invalid-chars', options.replaceInvalidChars);
}
if (options.useStructTree) {
args.push('--use-struct-tree');
}
if (options.tableMethod) {
args.push('--table-method', options.tableMethod);
}
if (options.readingOrder) {
args.push('--reading-order', options.readingOrder);
}
if (options.markdownPageSeparator) {
args.push('--markdown-page-separator', options.markdownPageSeparator);
}
if (options.markdownWithHtml) {
args.push('--markdown-with-html');
}
if (options.textPageSeparator) {
args.push('--text-page-separator', options.textPageSeparator);
}
if (options.htmlPageSeparator) {
args.push('--html-page-separator', options.htmlPageSeparator);
}
if (options.imageOutput) {
args.push('--image-output', options.imageOutput);
}
if (options.imageFormat) {
args.push('--image-format', options.imageFormat);
}
if (options.imageDir) {
args.push('--image-dir', options.imageDir);
}
if (options.pages) {
args.push('--pages', options.pages);
}
if (options.includeHeaderFooter) {
args.push('--include-header-footer');
}
if (options.detectStrikethrough) {
args.push('--detect-strikethrough');
}
if (options.hybrid) {
args.push('--hybrid', options.hybrid);
}
if (options.hybridMode) {
args.push('--hybrid-mode', options.hybridMode);
}
if (options.hybridUrl) {
args.push('--hybrid-url', options.hybridUrl);
}
if (options.hybridTimeout) {
args.push('--hybrid-timeout', options.hybridTimeout);
}
if (options.hybridFallback) {
args.push('--hybrid-fallback');
}
if (options.hybridHancomAiRegionlistStrategy) {
args.push('--hybrid-hancom-ai-regionlist-strategy', options.hybridHancomAiRegionlistStrategy);
}
if (options.hybridHancomAiOcrStrategy) {
args.push('--hybrid-hancom-ai-ocr-strategy', options.hybridHancomAiOcrStrategy);
}
if (options.hybridHancomAiImageCache) {
args.push('--hybrid-hancom-ai-image-cache', options.hybridHancomAiImageCache);
}
if (options.toStdout) {
args.push('--to-stdout');
}
if (options.threads) {
args.push('--threads', options.threads);
}
return args;
}
+246
View File
@@ -0,0 +1,246 @@
import { spawn } from 'child_process';
import * as path from 'path';
import * as fs from 'fs';
import { StringDecoder } from 'string_decoder';
import { fileURLToPath } from 'url';
// Re-export types and utilities from auto-generated file
export type { ConvertOptions } from './convert-options.generated.js';
export { buildArgs } from './convert-options.generated.js';
import type { ConvertOptions } from './convert-options.generated.js';
import { buildArgs } from './convert-options.generated.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const JAR_NAME = 'opendataloader-pdf-cli.jar';
interface JarExecutionOptions {
/**
* When true, forwards Java's stdout and stderr chunks to the parent
* process in real time as well as accumulating them. Used by the bundled
* CLI so long-running conversions show progress as it happens.
*/
streamOutput?: boolean;
}
function executeJar(args: string[], executionOptions: JarExecutionOptions = {}): Promise<string> {
const { streamOutput = false } = executionOptions;
return new Promise((resolve, reject) => {
const jarPath = path.join(__dirname, '..', 'lib', JAR_NAME);
if (!fs.existsSync(jarPath)) {
return reject(
new Error(`JAR file not found at ${jarPath}. Please run the build script first.`),
);
}
const command = 'java';
// Force headless AWT so macOS doesn't surface a Dock icon (and steal focus)
// every time the JVM touches ImageIO/PDFBox rendering. Safe on all OSes —
// the CLI never opens a UI window, only manipulates BufferedImages.
const commandArgs = [
'-Djava.awt.headless=true',
'-Dapple.awt.UIElement=true',
'-jar',
jarPath,
...args,
];
const javaProcess = spawn(command, commandArgs);
let stdout = '';
let stderr = '';
// StringDecoder buffers incomplete multi-byte UTF-8 sequences across
// chunk boundaries — Buffer.toString() alone would emit replacement
// characters when, e.g., a 3-byte Korean codepoint splits across two
// 'data' events. One decoder per stream so they don't share state.
const stdoutDecoder = new StringDecoder('utf8');
const stderrDecoder = new StringDecoder('utf8');
javaProcess.stdout.on('data', (data: Buffer) => {
const chunk = stdoutDecoder.write(data);
if (chunk.length === 0) return;
if (streamOutput) {
// Stream-only: don't also accumulate, or a multi-hour conversion
// would buffer its entire (potentially gigabyte) stdout in memory
// for no consumer.
process.stdout.write(chunk);
} else {
stdout += chunk;
}
});
javaProcess.stderr.on('data', (data: Buffer) => {
const chunk = stderrDecoder.write(data);
if (chunk.length === 0) return;
if (streamOutput) {
process.stderr.write(chunk);
}
// stderr is always accumulated (progress logs are small and we need
// them for error messages on non-zero exit).
stderr += chunk;
});
javaProcess.on('close', (code) => {
// Flush any trailing bytes the decoder is still holding (always emit
// them — if we drop them on error paths, error messages with non-ASCII
// characters lose their tail).
const stdoutTail = stdoutDecoder.end();
const stderrTail = stderrDecoder.end();
if (stdoutTail.length > 0) {
if (streamOutput) {
process.stdout.write(stdoutTail);
} else {
stdout += stdoutTail;
}
}
if (stderrTail.length > 0) {
if (streamOutput) process.stderr.write(stderrTail);
stderr += stderrTail;
}
if (code === 0) {
resolve(stdout);
} else {
const errorOutput = stderr || stdout;
const error = new Error(
`The opendataloader-pdf CLI exited with code ${code}.\n\n${errorOutput}`,
);
// Tag so the CLI can suppress re-printing this message — Java's
// stderr was already streamed live to the parent in CLI mode, and
// re-printing risks leaking anything sensitive Java logged
// (e.g. a --password value echoed by an underlying library).
(error as Error & { isJavaExit?: boolean }).isJavaExit = true;
reject(error);
}
});
javaProcess.on('error', (err: Error) => {
if (err.message.includes('ENOENT')) {
reject(
new Error(
"'java' command not found. Please ensure Java is installed and in your system's PATH.",
),
);
} else {
reject(err);
}
});
});
}
function buildJarArgs(
inputPaths: string | string[],
options: ConvertOptions,
): string[] | Error {
const inputList = Array.isArray(inputPaths) ? inputPaths : [inputPaths];
if (inputList.length === 0) {
return new Error('At least one input path must be provided.');
}
for (const input of inputList) {
if (!fs.existsSync(input)) {
return new Error(`Input file or folder not found: ${input}`);
}
}
return [...inputList, ...buildArgs(options)];
}
export function convert(
inputPaths: string | string[],
options: ConvertOptions = {},
): Promise<string> {
const argsOrError = buildJarArgs(inputPaths, options);
if (argsOrError instanceof Error) {
return Promise.reject(argsOrError);
}
// Library API: never streams to the parent process. Returns the full stdout
// string so callers can do `const out = await convert(...)` without surprise
// side-effects on process.stdout / process.stderr.
return executeJar(argsOrError, { streamOutput: false });
}
/**
* Internal entry point used by the bundled CLI. Streams Java's stdout and
* stderr to the parent process in real time (so long-running conversions like
* hybrid mode show progress as it happens) and resolves without a stdout
* payload — preventing the caller from re-printing what was already streamed.
*
* Not part of the public API: do not import this from application code. Use
* {@link convert} instead.
*
* @internal
*/
export async function _runForCli(
inputPaths: string | string[],
options: ConvertOptions = {},
): Promise<void> {
const argsOrError = buildJarArgs(inputPaths, options);
if (argsOrError instanceof Error) {
throw argsOrError;
}
await executeJar(argsOrError, { streamOutput: true });
}
/**
* @deprecated Use `convert()` and `ConvertOptions` instead. This function will be removed in a future version.
*/
export interface RunOptions {
outputFolder?: string;
password?: string;
replaceInvalidChars?: string;
generateMarkdown?: boolean;
generateHtml?: boolean;
generateAnnotatedPdf?: boolean;
keepLineBreaks?: boolean;
contentSafetyOff?: string;
htmlInMarkdown?: boolean;
addImageToMarkdown?: boolean;
noJson?: boolean;
debug?: boolean;
useStructTree?: boolean;
}
/**
* @deprecated Use `convert()` instead. This function will be removed in a future version.
*/
export function run(inputPath: string, options: RunOptions = {}): Promise<string> {
console.warn(
'Warning: run() is deprecated and will be removed in a future version. Use convert() instead.',
);
// Build format array based on legacy boolean options
const formats: string[] = [];
if (!options.noJson) {
formats.push('json');
}
if (options.generateMarkdown) {
if (options.addImageToMarkdown) {
formats.push('markdown-with-images');
} else if (options.htmlInMarkdown) {
formats.push('markdown-with-html');
} else {
formats.push('markdown');
}
}
if (options.generateHtml) {
formats.push('html');
}
if (options.generateAnnotatedPdf) {
formats.push('pdf');
}
return convert(inputPath, {
outputDir: options.outputFolder,
password: options.password,
replaceInvalidChars: options.replaceInvalidChars,
keepLineBreaks: options.keepLineBreaks,
contentSafetyOff: options.contentSafetyOff,
useStructTree: options.useStructTree,
format: formats.length > 0 ? formats : undefined,
quiet: !options.debug,
});
}
@@ -0,0 +1,227 @@
/**
* Unit tests for auto-generated convert-options functions
*/
import { describe, it, expect } from 'vitest';
import {
buildArgs,
buildConvertOptions,
ConvertOptions,
CliOptions,
} from '../src/convert-options.generated';
describe('buildArgs()', () => {
it('should return empty array for empty options', () => {
const args = buildArgs({});
expect(args).toEqual([]);
});
it('should handle string options', () => {
const args = buildArgs({
outputDir: '/output',
password: 'secret',
readingOrder: 'xycut',
});
expect(args).toEqual([
'--output-dir', '/output',
'--password', 'secret',
'--reading-order', 'xycut',
]);
});
it('should handle boolean options', () => {
const args = buildArgs({
quiet: true,
keepLineBreaks: true,
});
expect(args).toEqual([
'--quiet',
'--keep-line-breaks',
]);
});
it('should handle sanitize boolean', () => {
const options: ConvertOptions = { sanitize: true };
const args = buildArgs(options);
expect(args).toEqual(['--sanitize']);
});
it('should not include sanitize when false', () => {
const options: ConvertOptions = { sanitize: false };
const args = buildArgs(options);
expect(args).toEqual([]);
});
it('should handle list options with string value', () => {
const args = buildArgs({
format: 'json,markdown',
contentSafetyOff: 'all',
});
expect(args).toEqual([
'--format', 'json,markdown',
'--content-safety-off', 'all',
]);
});
it('should handle list options with array value', () => {
const args = buildArgs({
format: ['json', 'markdown', 'html'],
contentSafetyOff: ['hidden-text', 'off-page'],
});
expect(args).toEqual([
'--format', 'json,markdown,html',
'--content-safety-off', 'hidden-text,off-page',
]);
});
it('should handle all options together', () => {
const options: ConvertOptions = {
outputDir: '/output',
password: 'secret',
format: ['json', 'markdown'],
quiet: true,
contentSafetyOff: 'all',
keepLineBreaks: true,
replaceInvalidChars: '_',
useStructTree: true,
tableMethod: 'cluster',
readingOrder: 'xycut',
markdownPageSeparator: '---',
textPageSeparator: '\\n\\n',
htmlPageSeparator: '<hr>',
imageOutput: 'external',
imageFormat: 'jpeg',
sanitize: true,
};
const args = buildArgs(options);
expect(args).toContain('--output-dir');
expect(args).toContain('/output');
expect(args).toContain('--password');
expect(args).toContain('secret');
expect(args).toContain('--format');
expect(args).toContain('json,markdown');
expect(args).toContain('--quiet');
expect(args).toContain('--content-safety-off');
expect(args).toContain('all');
expect(args).toContain('--keep-line-breaks');
expect(args).toContain('--replace-invalid-chars');
expect(args).toContain('_');
expect(args).toContain('--use-struct-tree');
expect(args).toContain('--table-method');
expect(args).toContain('cluster');
expect(args).toContain('--reading-order');
expect(args).toContain('xycut');
expect(args).toContain('--markdown-page-separator');
expect(args).toContain('---');
expect(args).toContain('--image-output');
expect(args).toContain('external');
expect(args).toContain('--image-format');
expect(args).toContain('jpeg');
expect(args).toContain('--sanitize');
});
it('should not include undefined options', () => {
const args = buildArgs({
outputDir: '/output',
quiet: false,
keepLineBreaks: false,
});
expect(args).toEqual(['--output-dir', '/output']);
});
it('should skip empty arrays for list options', () => {
const args = buildArgs({
format: [],
contentSafetyOff: [],
outputDir: '/output',
});
expect(args).toEqual(['--output-dir', '/output']);
});
});
describe('buildConvertOptions()', () => {
it('should return empty object for empty CLI options', () => {
const result = buildConvertOptions({});
expect(result).toEqual({});
});
it('should convert string options', () => {
const cliOptions: CliOptions = {
outputDir: '/output',
password: 'secret',
readingOrder: 'xycut',
imageFormat: 'png',
};
const result = buildConvertOptions(cliOptions);
expect(result).toEqual({
outputDir: '/output',
password: 'secret',
readingOrder: 'xycut',
imageFormat: 'png',
});
});
it('should convert boolean options', () => {
const cliOptions: CliOptions = {
quiet: true,
keepLineBreaks: true,
useStructTree: true,
};
const result = buildConvertOptions(cliOptions);
expect(result).toEqual({
quiet: true,
keepLineBreaks: true,
useStructTree: true,
});
});
it('should not include false boolean options', () => {
const cliOptions: CliOptions = {
outputDir: '/output',
quiet: false,
keepLineBreaks: false,
};
const result = buildConvertOptions(cliOptions);
expect(result).toEqual({
outputDir: '/output',
});
});
it('should pass through all provided options', () => {
const cliOptions: CliOptions = {
outputDir: '/output',
password: 'secret',
format: 'json,markdown',
quiet: true,
contentSafetyOff: 'all',
keepLineBreaks: true,
replaceInvalidChars: '_',
useStructTree: true,
tableMethod: 'cluster',
readingOrder: 'xycut',
markdownPageSeparator: '---',
textPageSeparator: '\\n',
htmlPageSeparator: '<hr>',
imageOutput: 'external',
imageFormat: 'jpeg',
};
const result = buildConvertOptions(cliOptions);
expect(result.outputDir).toBe('/output');
expect(result.password).toBe('secret');
expect(result.format).toBe('json,markdown');
expect(result.quiet).toBe(true);
expect(result.contentSafetyOff).toBe('all');
expect(result.keepLineBreaks).toBe(true);
expect(result.replaceInvalidChars).toBe('_');
expect(result.useStructTree).toBe(true);
expect(result.tableMethod).toBe('cluster');
expect(result.readingOrder).toBe('xycut');
expect(result.markdownPageSeparator).toBe('---');
expect(result.textPageSeparator).toBe('\\n');
expect(result.htmlPageSeparator).toBe('<hr>');
expect(result.imageOutput).toBe('external');
expect(result.imageFormat).toBe('jpeg');
});
});
@@ -0,0 +1,43 @@
/**
* Integration tests that actually run the JAR (slow)
*/
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { convert } from '../src/index';
import * as path from 'path';
import * as fs from 'fs';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, '..', '..', '..');
const inputPdf = path.join(rootDir, 'samples', 'pdf', '1901.03003.pdf');
const tempDir = path.join(__dirname, 'temp', 'convert');
describe('convert() integration', () => {
beforeAll(() => {
if (fs.existsSync(tempDir)) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
fs.mkdirSync(tempDir, { recursive: true });
});
afterAll(() => {
if (fs.existsSync(tempDir)) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it('should generate output file', async () => {
await convert(inputPdf, {
outputDir: tempDir,
format: 'json',
quiet: true,
});
const outputFile = path.join(tempDir, '1901.03003.json');
expect(fs.existsSync(outputFile)).toBe(true);
expect(fs.statSync(outputFile).size).toBeGreaterThan(0);
}, 30000);
});
@@ -0,0 +1,288 @@
/**
* Unit tests for the JAR execution layer's stdout/stderr handling.
*
* These tests use a fake child process (no real Java) so we can deterministically
* assert the streaming/buffering contract:
* - convert() — library API: never writes to process.stdout, returns full stdout
* - _runForCli() — CLI helper: streams stdout/stderr to the parent in real time,
* does not return the stdout payload (the caller must not re-print it)
*
* Issue #398 reproducer: a long-running conversion (think hybrid mode, 1h+) must
* surface progress via stderr without the CLI double-printing the result on close.
*/
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { EventEmitter } from 'events';
// Mock child_process so we control spawn() entirely (no real Java process).
vi.mock('child_process', () => ({
spawn: vi.fn(),
}));
// Mock fs so JAR-file-exists and input-file-exists checks always pass.
vi.mock('fs', () => ({
existsSync: vi.fn().mockReturnValue(true),
}));
import { spawn } from 'child_process';
import { convert, _runForCli } from '../src/index';
class FakeProcess extends EventEmitter {
stdout = new EventEmitter() as EventEmitter & { pipe?: unknown };
stderr = new EventEmitter() as EventEmitter & { pipe?: unknown };
}
function makeFakeSpawn(): {
proc: FakeProcess;
spawnMock: ReturnType<typeof vi.fn>;
} {
const proc = new FakeProcess();
const spawnMock = spawn as unknown as ReturnType<typeof vi.fn>;
spawnMock.mockReturnValue(proc as unknown as ReturnType<typeof spawn>);
return { proc, spawnMock };
}
/** Yield to the microtask queue so Promise callbacks fire. */
const tick = () => new Promise<void>((r) => setImmediate(r));
describe('executeJar — library API (convert)', () => {
let stdoutSpy: ReturnType<typeof vi.spyOn>;
let stderrSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
});
afterEach(() => {
stdoutSpy.mockRestore();
stderrSpy.mockRestore();
vi.clearAllMocks();
});
it('returns the full stdout string on success', async () => {
const { proc } = makeFakeSpawn();
const promise = convert('input.pdf', { quiet: true });
proc.stdout.emit('data', Buffer.from('Lorem '));
proc.stdout.emit('data', Buffer.from('ipsum dolor'));
proc.emit('close', 0);
await expect(promise).resolves.toBe('Lorem ipsum dolor');
});
it('does not write to process.stdout (no streaming side-effect)', async () => {
const { proc } = makeFakeSpawn();
const promise = convert('input.pdf', { quiet: true });
proc.stdout.emit('data', Buffer.from('result text'));
proc.emit('close', 0);
await promise;
expect(stdoutSpy).not.toHaveBeenCalled();
});
it('does not write to process.stderr (no streaming side-effect)', async () => {
// Library callers should not have stderr leaked into their parent process.
// Java's --quiet silences stderr at the source for typical library use,
// but the contract holds even if Java emits to stderr.
const { proc } = makeFakeSpawn();
const promise = convert('input.pdf', { quiet: true });
proc.stderr.emit('data', Buffer.from('정보: progress'));
proc.stdout.emit('data', Buffer.from('result'));
proc.emit('close', 0);
await promise;
expect(stderrSpy).not.toHaveBeenCalled();
});
it('rejects with stderr in the error message on non-zero exit', async () => {
const { proc } = makeFakeSpawn();
const promise = convert('input.pdf', { quiet: true });
proc.stderr.emit('data', Buffer.from('Java stack trace'));
proc.emit('close', 1);
await expect(promise).rejects.toThrow(/exited with code 1/);
await expect(promise).rejects.toThrow(/Java stack trace/);
});
it('tags the rejection with isJavaExit so the CLI can suppress re-printing', async () => {
// The CLI relies on this tag to avoid duplicating stderr that was already
// streamed live (and to avoid re-surfacing anything sensitive Java logged).
// Library callers can ignore the tag — message and behavior are unchanged.
const { proc } = makeFakeSpawn();
const promise = convert('input.pdf', { quiet: true });
proc.stderr.emit('data', Buffer.from('Bad password: hunter2'));
proc.emit('close', 1);
await expect(promise).rejects.toMatchObject({ isJavaExit: true });
});
it('reassembles multi-byte UTF-8 codepoints split across chunks', async () => {
// Java's progress logs are localized; '정' = 0xEC 0xA0 0x95 (3 bytes).
// If the OS hands us this codepoint split across two 'data' events, a
// naive Buffer.toString() emits two replacement characters. Streaming
// is meant to be byte-faithful to what Java printed, so we must
// reassemble across the boundary.
const { proc } = makeFakeSpawn();
const promise = convert('input.pdf', { quiet: true });
proc.stdout.emit('data', Buffer.from([0xEC, 0xA0])); // first 2 bytes of '정'
proc.stdout.emit('data', Buffer.from([0x95])); // last byte of '정'
proc.stdout.emit('data', Buffer.from('보', 'utf8')); // a clean codepoint
proc.emit('close', 0);
await expect(promise).resolves.toBe('정보');
});
});
describe('executeJar — CLI helper (_runForCli)', () => {
let stdoutSpy: ReturnType<typeof vi.spyOn>;
let stderrSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
});
afterEach(() => {
stdoutSpy.mockRestore();
stderrSpy.mockRestore();
vi.clearAllMocks();
});
it('streams stdout chunks to process.stdout in real time', async () => {
// The defining requirement for hybrid-mode-style long runs: the user must
// see output as it arrives, not buffered until the process closes.
const { proc } = makeFakeSpawn();
const promise = _runForCli(['input.pdf']);
proc.stdout.emit('data', Buffer.from('chunk1'));
await tick();
expect(stdoutSpy).toHaveBeenCalledWith('chunk1');
expect(stdoutSpy).toHaveBeenCalledTimes(1);
proc.stdout.emit('data', Buffer.from('chunk2'));
await tick();
expect(stdoutSpy).toHaveBeenCalledTimes(2);
proc.emit('close', 0);
await promise;
// Critical: process.stdout was called exactly twice — once per chunk.
// No extra "final flush" call. That's how we avoid the #398 double-write.
expect(stdoutSpy).toHaveBeenCalledTimes(2);
});
it('streams stderr chunks to process.stderr in real time', async () => {
const { proc } = makeFakeSpawn();
const promise = _runForCli(['input.pdf']);
proc.stderr.emit('data', Buffer.from('정보: Number of pages: 14\n'));
await tick();
expect(stderrSpy).toHaveBeenCalledWith('정보: Number of pages: 14\n');
proc.stderr.emit('data', Buffer.from('정보: Processing 14 pages\n'));
await tick();
expect(stderrSpy).toHaveBeenCalledTimes(2);
proc.emit('close', 0);
await promise;
});
it('preserves true interleaving of stderr and stdout chunks as they arrive', async () => {
// Hybrid mode emits progress logs on stderr *throughout* the run, with
// result chunks landing on stdout in between. Both streams must reach the
// parent in arrival order without one buffering the other.
const { proc } = makeFakeSpawn();
const promise = _runForCli(['input.pdf']);
const calls: string[] = [];
stdoutSpy.mockImplementation(((chunk: Buffer) => {
calls.push('OUT:' + chunk.toString());
return true;
}) as never);
stderrSpy.mockImplementation(((chunk: Buffer) => {
calls.push('ERR:' + chunk.toString());
return true;
}) as never);
proc.stderr.emit('data', Buffer.from('progress 1'));
await tick();
proc.stdout.emit('data', Buffer.from('result A'));
await tick();
proc.stderr.emit('data', Buffer.from('progress 2'));
await tick();
proc.stdout.emit('data', Buffer.from('result B'));
await tick();
proc.emit('close', 0);
await promise;
expect(calls).toEqual([
'ERR:progress 1',
'OUT:result A',
'ERR:progress 2',
'OUT:result B',
]);
});
it('resolves without returning the stdout payload (caller must not re-print)', async () => {
// _runForCli's contract: streaming has the side-effect, the resolved value
// carries no stdout text. This is what blocks the CLI from double-printing.
const { proc } = makeFakeSpawn();
const promise = _runForCli(['input.pdf']);
proc.stdout.emit('data', Buffer.from('result text'));
proc.emit('close', 0);
const resolved = await promise;
expect(resolved).toBeUndefined();
});
it('rejects with stderr in the error message on non-zero exit', async () => {
const { proc } = makeFakeSpawn();
const promise = _runForCli(['input.pdf']);
proc.stderr.emit('data', Buffer.from('boom'));
proc.emit('close', 2);
await expect(promise).rejects.toThrow(/exited with code 2/);
await expect(promise).rejects.toThrow(/boom/);
});
it('forwards multi-byte UTF-8 to process.stderr without splitting codepoints', async () => {
// Defining the streaming contract for non-ASCII progress logs. Without a
// StringDecoder, splitting '정' (0xEC 0xA0 0x95) across two chunks would
// emit two replacement characters to the user's terminal — exactly the
// failure mode the comment in index.ts warns about.
const { proc } = makeFakeSpawn();
const promise = _runForCli(['input.pdf']);
const forwarded: string[] = [];
stderrSpy.mockImplementation(((chunk: string) => {
forwarded.push(chunk);
return true;
}) as never);
proc.stderr.emit('data', Buffer.from([0xEC, 0xA0])); // partial '정'
await tick();
proc.stderr.emit('data', Buffer.from([0x95])); // completes '정'
await tick();
proc.stderr.emit('data', Buffer.from('보', 'utf8'));
await tick();
proc.emit('close', 0);
await promise;
// No replacement characters anywhere in what the user saw.
const joined = forwarded.join('');
expect(joined).toBe('정보');
expect(joined).not.toContain('');
});
});
@@ -0,0 +1,67 @@
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
import { run, convert } from '../src/index';
import * as path from 'path';
import * as fs from 'fs';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, '..', '..', '..');
const inputPdf = path.join(rootDir, 'samples', 'pdf', '1901.03003.pdf');
const tempDir = path.join(__dirname, 'temp', 'run');
describe('opendataloader-pdf', () => {
beforeAll(() => {
// Clean up previous test runs
if (fs.existsSync(tempDir)) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
fs.mkdirSync(tempDir, { recursive: true });
});
afterAll(() => {
// Clean up after tests
if (fs.existsSync(tempDir)) {
fs.rmSync(tempDir, { recursive: true, force: true });
}
});
it('should process PDF and generate markdown output', async () => {
console.log(`[TEST] Running opendataloader-pdf test...`);
console.log(`[TEST] Input PDF: ${inputPdf}`);
console.log(`[TEST] Output directory: ${tempDir}`);
await run(inputPdf, {
outputFolder: tempDir,
generateMarkdown: true,
generateHtml: true,
generateAnnotatedPdf: true,
debug: true,
});
expect(fs.existsSync(path.join(tempDir, '1901.03003.json'))).toBe(true);
expect(fs.existsSync(path.join(tempDir, '1901.03003.md'))).toBe(true);
expect(fs.existsSync(path.join(tempDir, '1901.03003.html'))).toBe(true);
expect(fs.existsSync(path.join(tempDir, '1901.03003_annotated.pdf'))).toBe(true);
}, 30000); // 30 second timeout for this test
it('should convert PDF with explicit formats using quiet mode', async () => {
const convertDir = path.join(tempDir, 'convert');
if (fs.existsSync(convertDir)) {
fs.rmSync(convertDir, { recursive: true, force: true });
}
fs.mkdirSync(convertDir);
await convert([inputPdf], {
outputDir: convertDir,
format: ['json', 'text', 'html', 'pdf', 'markdown'],
});
expect(fs.existsSync(path.join(convertDir, '1901.03003.json'))).toBe(true);
expect(fs.existsSync(path.join(convertDir, '1901.03003.txt'))).toBe(true);
expect(fs.existsSync(path.join(convertDir, '1901.03003.html'))).toBe(true);
expect(fs.existsSync(path.join(convertDir, '1901.03003.md'))).toBe(true);
expect(fs.existsSync(path.join(convertDir, '1901.03003_annotated.pdf'))).toBe(true);
}, 30000);
});
@@ -0,0 +1,169 @@
/**
* Integration tests for the stdout/stderr streaming contract.
*
* Runs the bundled CLI as a real subprocess against a multi-page sample PDF.
* Asserts the user-facing behavior that mock unit tests cannot prove:
* - CLI never double-prints stdout (regression test for #398)
* - Java's progress logs reach the parent's stderr in real time, before the
* stdout payload finishes — the property that makes hour-long hybrid runs
* observable
* - Library API does not leak Java's output to the parent's stdout
*/
import { describe, it, expect, beforeAll } from 'vitest';
import { spawn, spawnSync } from 'child_process';
import * as path from 'path';
import * as fs from 'fs';
import * as os from 'os';
import { fileURLToPath, pathToFileURL } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, '..', '..', '..');
const cliPath = path.resolve(__dirname, '..', 'dist', 'cli.js');
const jarPath = path.resolve(__dirname, '..', 'lib', 'opendataloader-pdf-cli.jar');
const samplePdf = path.join(rootDir, 'samples', 'pdf', '2408.02509v1.pdf');
interface Capture {
stdout: string;
stderr: string;
/** Wall-clock ms (since process spawn) of each chunk we received. */
timeline: Array<{ stream: 'out' | 'err'; size: number; at: number }>;
exitCode: number | null;
}
function captureSubprocess(command: string, args: string[]): Promise<Capture> {
return new Promise((resolve, reject) => {
const proc = spawn(command, args);
const start = Date.now();
const cap: Capture = { stdout: '', stderr: '', timeline: [], exitCode: null };
proc.stdout.on('data', (chunk: Buffer) => {
cap.stdout += chunk.toString();
cap.timeline.push({ stream: 'out', size: chunk.length, at: Date.now() - start });
});
proc.stderr.on('data', (chunk: Buffer) => {
cap.stderr += chunk.toString();
cap.timeline.push({ stream: 'err', size: chunk.length, at: Date.now() - start });
});
proc.on('close', (code) => {
cap.exitCode = code;
resolve(cap);
});
proc.on('error', reject);
});
}
const runCli = (args: string[]) => captureSubprocess('node', [cliPath, ...args]);
const runJar = (args: string[]) => captureSubprocess('java', ['-jar', jarPath, ...args]);
describe('CLI streaming contract', () => {
beforeAll(() => {
// Self-contained: build dist on demand so `pnpm test` works on a fresh
// checkout without a separate `pnpm build` step. The JAR has to come from
// the Maven build (it's not something Vitest should rebuild), so we still
// surface a clear error if it's missing.
if (!fs.existsSync(cliPath)) {
const result = spawnSync(
'pnpm',
['exec', 'tsup', '--no-dts', 'src/index.ts', 'src/cli.ts',
'--format', 'esm,cjs', '--shims', '--out-dir', 'dist'],
{ cwd: path.resolve(__dirname, '..'), stdio: 'inherit' },
);
if (result.status !== 0) {
throw new Error('Failed to build dist for streaming integration test');
}
}
if (!fs.existsSync(jarPath)) {
throw new Error(
`Bundled JAR not found at ${jarPath} — build the Java module first ` +
`(\`mvn package\` in java/, then \`pnpm run setup\` in this package).`,
);
}
if (!fs.existsSync(samplePdf)) {
throw new Error(`Sample PDF not found at ${samplePdf}`);
}
}, 60000);
it('prints --to-stdout output exactly once (regression for #398)', async () => {
// Ground truth: invoke the JAR directly with --quiet so stderr is empty
// and stdout carries only the result payload. The Node CLI must produce
// the same bytes — no more, no less. Comparing against the JAR rather
// than another Node-CLI run prevents a symmetric-mutation regression
// (e.g., trimming/appending in both code paths) from sneaking past.
const referenceCap = await runJar([samplePdf, '--quiet', '--format', 'text', '--to-stdout']);
expect(referenceCap.exitCode).toBe(0);
const referenceStdout = referenceCap.stdout;
expect(referenceStdout.length).toBeGreaterThan(1000);
const cap = await runCli([samplePdf, '--format', 'text', '--to-stdout']);
expect(cap.exitCode).toBe(0);
// Defining check: the no-flag Node CLI must produce the *exact same*
// stdout bytes as the JAR. Pre-fix this came out at 2x because of
// double-write. Length-only comparison would miss byte-substituting
// mutations of identical length, so we compare the full payload.
expect(cap.stdout).toBe(referenceStdout);
}, 60000);
it('forwards Java progress logs to stderr in real time before stdout completes', async () => {
const cap = await runCli([samplePdf, '--format', 'text', '--to-stdout']);
expect(cap.exitCode).toBe(0);
// Java emits a "Number of pages" line during preprocessing — long before
// text extraction finishes — to stderr.
expect(cap.stderr).toMatch(/Number of pages/);
// The property that makes long runs observable: at least one stderr chunk
// arrives before stdout finishes streaming. We compare *event indices*
// (arrival order in cap.timeline) rather than millisecond timestamps to
// stay deterministic — two events scheduled in the same tick will share
// an `at` value but always have distinct indices.
const firstErrIdx = cap.timeline.findIndex((e) => e.stream === 'err');
let lastOutIdx = -1;
for (let i = cap.timeline.length - 1; i >= 0; i--) {
if (cap.timeline[i].stream === 'out') {
lastOutIdx = i;
break;
}
}
expect(firstErrIdx).toBeGreaterThanOrEqual(0);
expect(lastOutIdx).toBeGreaterThanOrEqual(0);
expect(firstErrIdx).toBeLessThan(lastOutIdx);
}, 60000);
it('library convert() does not leak to the parent process stdio', async () => {
// Spawn a tiny Node script that imports convert() and calls it. We capture
// *that* process's stdio: convert() must not write to stdout/stderr itself.
// The import specifier is a file:// URL so the harness works on Windows
// (where path.resolve yields backslashes that ESM rejects).
const distIndexUrl = pathToFileURL(path.resolve(__dirname, '..', 'dist', 'index.js')).href;
const harness = `
import { convert } from '${distIndexUrl}';
const out = await convert(${JSON.stringify(samplePdf)}, {
format: ['text'], toStdout: true, quiet: true,
});
// One intended write at the end so we can distinguish "the marker"
// from any leakage caused by executeJar forwarding to process.stdout.
process.stdout.write('LEN=' + out.length + '\\n');
`;
// Place the harness in an isolated tmpdir — survives Ctrl-C / OOM cleanly
// (OS reaps tmpdir trees) instead of polluting the source tree.
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'odl-pdf-issue398-'));
const tmpScript = path.join(tmpDir, 'harness.mjs');
fs.writeFileSync(tmpScript, harness);
try {
const cap = await captureSubprocess('node', [tmpScript]);
expect(cap.exitCode).toBe(0);
// The only line on stdout must be our LEN= marker. Anything else means
// executeJar leaked Java's stdout into the parent.
expect(cap.stdout.trim().split('\n')).toHaveLength(1);
expect(cap.stdout).toMatch(/^LEN=\d+\n$/);
expect(cap.stderr).toBe('');
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
}, 60000);
});
+17
View File
@@ -0,0 +1,17 @@
{
"compilerOptions": {
"target": "es2023",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"declaration": true,
"skipLibCheck": true,
"strict": true,
"stripInternal": true,
"types": ["node"],
"ignoreDeprecations": "6.0"
},
"include": ["src/**/*.ts"]
}
+12
View File
@@ -0,0 +1,12 @@
import { defineConfig } from 'tsup';
export default defineConfig({
clean: true,
dts: true,
entry: ['src/index.ts', 'src/cli.ts'],
format: ['esm', 'cjs'],
shims: true,
sourcemap: true,
outDir: 'dist',
splitting: false,
});
+8
View File
@@ -0,0 +1,8 @@
/// <reference types="vitest" />
import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
// Your test configuration options go here
},
});