chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
pnpm-debug.log*
|
||||
lerna-debug.log*
|
||||
|
||||
node_modules
|
||||
dist
|
||||
dist-ssr
|
||||
temp
|
||||
*.jar
|
||||
*.local
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
.idea
|
||||
.DS_Store
|
||||
*.suo
|
||||
*.ntvs*
|
||||
*.njsproj
|
||||
*.sln
|
||||
*.sw?
|
||||
|
||||
# turbo
|
||||
.turbo
|
||||
|
||||
# tsconfig
|
||||
tsconfig.tsbuildinfo
|
||||
|
||||
# docs
|
||||
THIRD_PARTY
|
||||
NOTICE
|
||||
README.md
|
||||
LICENSE
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"singleQuote": true,
|
||||
"trailingComma": "all",
|
||||
"tabWidth": 2,
|
||||
"semi": true,
|
||||
"printWidth": 100
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import eslint from "@eslint/js";
|
||||
import tseslint from "@typescript-eslint/eslint-plugin";
|
||||
import tsparser from "@typescript-eslint/parser";
|
||||
import globals from "globals";
|
||||
|
||||
export default [
|
||||
eslint.configs.recommended,
|
||||
{
|
||||
files: ["src/**/*.ts"],
|
||||
languageOptions: {
|
||||
globals: {
|
||||
...globals.node,
|
||||
},
|
||||
parser: tsparser,
|
||||
parserOptions: {
|
||||
ecmaVersion: "latest",
|
||||
sourceType: "module",
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
"@typescript-eslint": tseslint,
|
||||
},
|
||||
rules: {
|
||||
...tseslint.configs.recommended.rules,
|
||||
"no-unused-vars": "off",
|
||||
"@typescript-eslint/no-unused-vars": "warn",
|
||||
},
|
||||
},
|
||||
{
|
||||
ignores: ["dist/**", "lib/**", "node_modules/**"],
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"name": "@opendataloader/pdf",
|
||||
"version": "0.0.0",
|
||||
"description": "A Node.js wrapper for the opendataloader-pdf Java CLI.",
|
||||
"main": "./dist/index.cjs",
|
||||
"module": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"type": "module",
|
||||
"bin": {
|
||||
"opendataloader-pdf": "./dist/cli.js"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"import": "./dist/index.js",
|
||||
"require": "./dist/index.cjs"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"setup": "node ./scripts/setup.cjs",
|
||||
"build": "pnpm run setup && tsup",
|
||||
"test": "vitest --run",
|
||||
"format": "prettier --write \"**/*.{ts,js,json,md}\"",
|
||||
"lint": "eslint \"src/**/*.ts\"",
|
||||
"lint:fix": "eslint \"src/**/*.ts\" --fix"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/opendataloader-project/opendataloader-pdf.git"
|
||||
},
|
||||
"keywords": [
|
||||
"pdf",
|
||||
"markdown",
|
||||
"html",
|
||||
"convert",
|
||||
"pdf-convert",
|
||||
"pdf-parser",
|
||||
"pdf-parsing",
|
||||
"pdf-to-json",
|
||||
"pdf-to-markdown",
|
||||
"pdf-to-html"
|
||||
],
|
||||
"author": "opendataloader-project",
|
||||
"license": "Apache-2.0",
|
||||
"bugs": {
|
||||
"url": "https://github.com/opendataloader-project/opendataloader-pdf/issues"
|
||||
},
|
||||
"homepage": "https://github.com/opendataloader-project/opendataloader-pdf#readme",
|
||||
"engines": {
|
||||
"node": ">=20.19.0"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"dependencies": {
|
||||
"commander": "^15.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^10.0.1",
|
||||
"@types/node": "^26.0.0",
|
||||
"@typescript-eslint/eslint-plugin": "^8.58.2",
|
||||
"@typescript-eslint/parser": "^8.58.2",
|
||||
"eslint": "^10.2.0",
|
||||
"glob": "^13.0.6",
|
||||
"globals": "^17.5.0",
|
||||
"prettier": "^3.8.3",
|
||||
"tsup": "^8.5.1",
|
||||
"typescript": "^6.0.2",
|
||||
"vite": "^8.0.8",
|
||||
"vitest": "^4.1.4"
|
||||
},
|
||||
"pnpm": {
|
||||
"overrides": {
|
||||
"minimatch@<10.2.3": ">=10.2.3",
|
||||
"flatted@<3.4.2": ">=3.4.2",
|
||||
"postcss@<8.5.10": ">=8.5.10"
|
||||
}
|
||||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"lib",
|
||||
"LICENSE",
|
||||
"NOTICE",
|
||||
"README.md",
|
||||
"THIRD_PARTY"
|
||||
]
|
||||
}
|
||||
Generated
+2673
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,62 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { globSync } = require('glob');
|
||||
|
||||
const rootDir = path.resolve(__dirname, '..');
|
||||
const javaDir = path.resolve(rootDir, '../../java');
|
||||
const sourceJarGlob = path
|
||||
.join(javaDir, 'opendataloader-pdf-cli/target/opendataloader-pdf-cli-*.jar')
|
||||
.replace(/\\/g, '/');
|
||||
|
||||
console.log(`Searching for JAR file in: ${sourceJarGlob}`);
|
||||
|
||||
const sourceJarPaths = globSync(sourceJarGlob);
|
||||
if (sourceJarPaths.length === 0) {
|
||||
console.error(
|
||||
"Could not find the JAR file. Please run 'mvn package' in the 'java/' directory first.",
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (sourceJarPaths.length > 1) {
|
||||
console.error(`Found multiple JAR files, expected one: ${sourceJarPaths}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const sourceJarPath = sourceJarPaths[0];
|
||||
console.log(`Found source JAR: ${sourceJarPath}`);
|
||||
|
||||
const destJarDir = path.join(rootDir, 'lib').replace(/\\/g, '/');
|
||||
if (!fs.existsSync(destJarDir)) {
|
||||
fs.mkdirSync(destJarDir, { recursive: true });
|
||||
}
|
||||
const destJarPath = path.join(destJarDir, 'opendataloader-pdf-cli.jar').replace(/\\/g, '/');
|
||||
console.log(`Copying JAR to ${destJarPath}`);
|
||||
fs.copyFileSync(sourceJarPath, destJarPath);
|
||||
|
||||
// Copy README.md, LICENSE, NOTICE, and THIRD_PARTY
|
||||
const readmeSrc = path.resolve(rootDir, '../../README.md');
|
||||
const licenseSrc = path.resolve(rootDir, '../../LICENSE');
|
||||
const noticeSrc = path.resolve(rootDir, '../../NOTICE');
|
||||
const thirdPartySrc = path.resolve(rootDir, '../../THIRD_PARTY');
|
||||
|
||||
const readmeDest = path.join(rootDir, 'README.md').replace(/\\/g, '/');
|
||||
const licenseDest = path.join(rootDir, 'LICENSE').replace(/\\/g, '/');
|
||||
const noticeDest = path.join(rootDir, 'NOTICE').replace(/\\/g, '/');
|
||||
const thirdPartyDest = path.join(rootDir, 'THIRD_PARTY').replace(/\\/g, '/');
|
||||
|
||||
console.log(`Copying README.md to ${readmeDest}`);
|
||||
fs.copyFileSync(readmeSrc, readmeDest);
|
||||
|
||||
console.log(`Copying LICENSE to ${licenseDest}`);
|
||||
fs.copyFileSync(licenseSrc, licenseDest);
|
||||
|
||||
console.log(`Copying NOTICE to ${noticeDest}`);
|
||||
fs.copyFileSync(noticeSrc, noticeDest);
|
||||
|
||||
console.log(`Copying THIRD_PARTY directory to ${thirdPartyDest}`);
|
||||
if (fs.existsSync(thirdPartyDest)) {
|
||||
fs.rmSync(thirdPartyDest, { recursive: true, force: true });
|
||||
}
|
||||
fs.cpSync(thirdPartySrc, thirdPartyDest, { recursive: true });
|
||||
|
||||
console.log('Package preparation complete.');
|
||||
@@ -0,0 +1,41 @@
|
||||
// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
|
||||
// Run `npm run generate-options` to regenerate
|
||||
|
||||
import { Command } from 'commander';
|
||||
|
||||
/**
|
||||
* Register all CLI options on the given Commander program.
|
||||
*/
|
||||
export function registerCliOptions(program: Command): void {
|
||||
program.option('-o, --output-dir <value>', 'Directory where output files are written. Default: input file directory');
|
||||
program.option('-p, --password <value>', 'Password for encrypted PDF files');
|
||||
program.option('-f, --format <value>', 'Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output.');
|
||||
program.option('-q, --quiet', 'Suppress console logging output');
|
||||
program.option('--content-safety-off <value>', 'Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg');
|
||||
program.option('--sanitize', 'Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders');
|
||||
program.option('--keep-line-breaks', 'Preserve original line breaks in extracted text');
|
||||
program.option('--replace-invalid-chars <value>', 'Replacement character for invalid/unrecognized characters. Default: space');
|
||||
program.option('--use-struct-tree', 'Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality');
|
||||
program.option('--table-method <value>', 'Table detection method. Values: default (border-based), cluster (border + cluster). Default: default');
|
||||
program.option('--reading-order <value>', 'Reading order algorithm. Values: off, xycut. Default: xycut');
|
||||
program.option('--markdown-page-separator <value>', 'Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none');
|
||||
program.option('--markdown-with-html', 'Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown.');
|
||||
program.option('--text-page-separator <value>', 'Separator between pages in text output. Use %page-number% for page numbers. Default: none');
|
||||
program.option('--html-page-separator <value>', 'Separator between pages in HTML output. Use %page-number% for page numbers. Default: none');
|
||||
program.option('--image-output <value>', 'Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external');
|
||||
program.option('--image-format <value>', 'Output format for extracted images. Values: png, jpeg. Default: png');
|
||||
program.option('--image-dir <value>', 'Directory for extracted images (applies only with --image-output external)');
|
||||
program.option('--pages <value>', 'Pages to extract (e.g., "1,3,5-7"). Default: all pages');
|
||||
program.option('--include-header-footer', 'Include page headers and footers in output');
|
||||
program.option('--detect-strikethrough', 'Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)');
|
||||
program.option('--hybrid <value>', 'Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai');
|
||||
program.option('--hybrid-mode <value>', 'Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)');
|
||||
program.option('--hybrid-url <value>', 'Hybrid backend server URL (overrides default)');
|
||||
program.option('--hybrid-timeout <value>', 'Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0');
|
||||
program.option('--hybrid-fallback', 'Opt in to Java fallback on hybrid backend error (default: disabled)');
|
||||
program.option('--hybrid-hancom-ai-regionlist-strategy <value>', 'DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)');
|
||||
program.option('--hybrid-hancom-ai-ocr-strategy <value>', 'OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)');
|
||||
program.option('--hybrid-hancom-ai-image-cache <value>', 'Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk');
|
||||
program.option('--to-stdout', 'Write output to stdout instead of file (single format only)');
|
||||
program.option('--threads <value>', 'Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode');
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
import { Command, CommanderError } from 'commander';
|
||||
import { _runForCli } from './index.js';
|
||||
import { CliOptions, buildConvertOptions } from './convert-options.generated.js';
|
||||
import { registerCliOptions } from './cli-options.generated.js';
|
||||
|
||||
function createProgram(): Command {
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('opendataloader-pdf')
|
||||
.usage('[options] <input...>')
|
||||
.description('Convert PDFs using the OpenDataLoader CLI.')
|
||||
.showHelpAfterError("Use '--help' to see available options.")
|
||||
.showSuggestionAfterError(false)
|
||||
.argument('<input...>', 'Input files or directories to convert');
|
||||
|
||||
// Register CLI options from auto-generated file
|
||||
registerCliOptions(program);
|
||||
|
||||
program.configureOutput({
|
||||
writeErr: (str) => {
|
||||
console.error(str.trimEnd());
|
||||
},
|
||||
outputError: (str, write) => {
|
||||
write(str);
|
||||
},
|
||||
});
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const program = createProgram();
|
||||
|
||||
program.exitOverride();
|
||||
|
||||
try {
|
||||
program.parse(process.argv);
|
||||
} catch (err) {
|
||||
if (err instanceof CommanderError) {
|
||||
if (err.code === 'commander.helpDisplayed') {
|
||||
return 0;
|
||||
}
|
||||
return err.exitCode ?? 1;
|
||||
}
|
||||
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(message);
|
||||
console.error("Use '--help' to see available options.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const cliOptions = program.opts<CliOptions>();
|
||||
const inputPaths = program.args;
|
||||
const convertOptions = buildConvertOptions(cliOptions);
|
||||
|
||||
try {
|
||||
// _runForCli streams stdout/stderr to the parent process as they arrive;
|
||||
// we deliberately do not re-print anything here. (Issue #398.)
|
||||
await _runForCli(inputPaths, convertOptions);
|
||||
return 0;
|
||||
} catch (err) {
|
||||
// Subprocess-exit errors are already on the user's terminal via the live
|
||||
// stderr stream — re-printing would duplicate output and risk leaking
|
||||
// anything sensitive Java logged (e.g. a --password value echoed by an
|
||||
// underlying library). Wrapper-side failures (JAR not found, java not in
|
||||
// PATH, bad input path) still need to be surfaced.
|
||||
const isJavaExit =
|
||||
err instanceof Error && (err as Error & { isJavaExit?: boolean }).isJavaExit === true;
|
||||
if (!isJavaExit) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(message);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
main().then((code) => {
|
||||
if (code !== 0) {
|
||||
process.exit(code);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,325 @@
|
||||
// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
|
||||
// Run `npm run generate-options` to regenerate
|
||||
|
||||
/**
|
||||
* Options for the convert function.
|
||||
*/
|
||||
export interface ConvertOptions {
|
||||
/** Directory where output files are written. Default: input file directory */
|
||||
outputDir?: string;
|
||||
/** Password for encrypted PDF files */
|
||||
password?: string;
|
||||
/** Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output. */
|
||||
format?: string | string[];
|
||||
/** Suppress console logging output */
|
||||
quiet?: boolean;
|
||||
/** Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg */
|
||||
contentSafetyOff?: string | string[];
|
||||
/** Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders */
|
||||
sanitize?: boolean;
|
||||
/** Preserve original line breaks in extracted text */
|
||||
keepLineBreaks?: boolean;
|
||||
/** Replacement character for invalid/unrecognized characters. Default: space */
|
||||
replaceInvalidChars?: string;
|
||||
/** Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality */
|
||||
useStructTree?: boolean;
|
||||
/** Table detection method. Values: default (border-based), cluster (border + cluster). Default: default */
|
||||
tableMethod?: string;
|
||||
/** Reading order algorithm. Values: off, xycut. Default: xycut */
|
||||
readingOrder?: string;
|
||||
/** Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none */
|
||||
markdownPageSeparator?: string;
|
||||
/** Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown. */
|
||||
markdownWithHtml?: boolean;
|
||||
/** Separator between pages in text output. Use %page-number% for page numbers. Default: none */
|
||||
textPageSeparator?: string;
|
||||
/** Separator between pages in HTML output. Use %page-number% for page numbers. Default: none */
|
||||
htmlPageSeparator?: string;
|
||||
/** Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external */
|
||||
imageOutput?: string;
|
||||
/** Output format for extracted images. Values: png, jpeg. Default: png */
|
||||
imageFormat?: string;
|
||||
/** Directory for extracted images (applies only with --image-output external) */
|
||||
imageDir?: string;
|
||||
/** Pages to extract (e.g., "1,3,5-7"). Default: all pages */
|
||||
pages?: string;
|
||||
/** Include page headers and footers in output */
|
||||
includeHeaderFooter?: boolean;
|
||||
/** Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental) */
|
||||
detectStrikethrough?: boolean;
|
||||
/** Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai */
|
||||
hybrid?: string;
|
||||
/** Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend) */
|
||||
hybridMode?: string;
|
||||
/** Hybrid backend server URL (overrides default) */
|
||||
hybridUrl?: string;
|
||||
/** Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0 */
|
||||
hybridTimeout?: string;
|
||||
/** Opt in to Java fallback on hybrid backend error (default: disabled) */
|
||||
hybridFallback?: boolean;
|
||||
/** DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list) */
|
||||
hybridHancomAiRegionlistStrategy?: string;
|
||||
/** OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only) */
|
||||
hybridHancomAiOcrStrategy?: string;
|
||||
/** Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk */
|
||||
hybridHancomAiImageCache?: string;
|
||||
/** Write output to stdout instead of file (single format only) */
|
||||
toStdout?: boolean;
|
||||
/** Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode */
|
||||
threads?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options as parsed from CLI (all values are strings from commander).
|
||||
*/
|
||||
export interface CliOptions {
|
||||
outputDir?: string;
|
||||
password?: string;
|
||||
format?: string;
|
||||
quiet?: boolean;
|
||||
contentSafetyOff?: string;
|
||||
sanitize?: boolean;
|
||||
keepLineBreaks?: boolean;
|
||||
replaceInvalidChars?: string;
|
||||
useStructTree?: boolean;
|
||||
tableMethod?: string;
|
||||
readingOrder?: string;
|
||||
markdownPageSeparator?: string;
|
||||
markdownWithHtml?: boolean;
|
||||
textPageSeparator?: string;
|
||||
htmlPageSeparator?: string;
|
||||
imageOutput?: string;
|
||||
imageFormat?: string;
|
||||
imageDir?: string;
|
||||
pages?: string;
|
||||
includeHeaderFooter?: boolean;
|
||||
detectStrikethrough?: boolean;
|
||||
hybrid?: string;
|
||||
hybridMode?: string;
|
||||
hybridUrl?: string;
|
||||
hybridTimeout?: string;
|
||||
hybridFallback?: boolean;
|
||||
hybridHancomAiRegionlistStrategy?: string;
|
||||
hybridHancomAiOcrStrategy?: string;
|
||||
hybridHancomAiImageCache?: string;
|
||||
toStdout?: boolean;
|
||||
threads?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert CLI options to ConvertOptions.
|
||||
*/
|
||||
export function buildConvertOptions(cliOptions: CliOptions): ConvertOptions {
|
||||
const convertOptions: ConvertOptions = {};
|
||||
|
||||
if (cliOptions.outputDir) {
|
||||
convertOptions.outputDir = cliOptions.outputDir;
|
||||
}
|
||||
if (cliOptions.password) {
|
||||
convertOptions.password = cliOptions.password;
|
||||
}
|
||||
if (cliOptions.format) {
|
||||
convertOptions.format = cliOptions.format;
|
||||
}
|
||||
if (cliOptions.quiet) {
|
||||
convertOptions.quiet = true;
|
||||
}
|
||||
if (cliOptions.contentSafetyOff) {
|
||||
convertOptions.contentSafetyOff = cliOptions.contentSafetyOff;
|
||||
}
|
||||
if (cliOptions.sanitize) {
|
||||
convertOptions.sanitize = true;
|
||||
}
|
||||
if (cliOptions.keepLineBreaks) {
|
||||
convertOptions.keepLineBreaks = true;
|
||||
}
|
||||
if (cliOptions.replaceInvalidChars) {
|
||||
convertOptions.replaceInvalidChars = cliOptions.replaceInvalidChars;
|
||||
}
|
||||
if (cliOptions.useStructTree) {
|
||||
convertOptions.useStructTree = true;
|
||||
}
|
||||
if (cliOptions.tableMethod) {
|
||||
convertOptions.tableMethod = cliOptions.tableMethod;
|
||||
}
|
||||
if (cliOptions.readingOrder) {
|
||||
convertOptions.readingOrder = cliOptions.readingOrder;
|
||||
}
|
||||
if (cliOptions.markdownPageSeparator) {
|
||||
convertOptions.markdownPageSeparator = cliOptions.markdownPageSeparator;
|
||||
}
|
||||
if (cliOptions.markdownWithHtml) {
|
||||
convertOptions.markdownWithHtml = true;
|
||||
}
|
||||
if (cliOptions.textPageSeparator) {
|
||||
convertOptions.textPageSeparator = cliOptions.textPageSeparator;
|
||||
}
|
||||
if (cliOptions.htmlPageSeparator) {
|
||||
convertOptions.htmlPageSeparator = cliOptions.htmlPageSeparator;
|
||||
}
|
||||
if (cliOptions.imageOutput) {
|
||||
convertOptions.imageOutput = cliOptions.imageOutput;
|
||||
}
|
||||
if (cliOptions.imageFormat) {
|
||||
convertOptions.imageFormat = cliOptions.imageFormat;
|
||||
}
|
||||
if (cliOptions.imageDir) {
|
||||
convertOptions.imageDir = cliOptions.imageDir;
|
||||
}
|
||||
if (cliOptions.pages) {
|
||||
convertOptions.pages = cliOptions.pages;
|
||||
}
|
||||
if (cliOptions.includeHeaderFooter) {
|
||||
convertOptions.includeHeaderFooter = true;
|
||||
}
|
||||
if (cliOptions.detectStrikethrough) {
|
||||
convertOptions.detectStrikethrough = true;
|
||||
}
|
||||
if (cliOptions.hybrid) {
|
||||
convertOptions.hybrid = cliOptions.hybrid;
|
||||
}
|
||||
if (cliOptions.hybridMode) {
|
||||
convertOptions.hybridMode = cliOptions.hybridMode;
|
||||
}
|
||||
if (cliOptions.hybridUrl) {
|
||||
convertOptions.hybridUrl = cliOptions.hybridUrl;
|
||||
}
|
||||
if (cliOptions.hybridTimeout) {
|
||||
convertOptions.hybridTimeout = cliOptions.hybridTimeout;
|
||||
}
|
||||
if (cliOptions.hybridFallback) {
|
||||
convertOptions.hybridFallback = true;
|
||||
}
|
||||
if (cliOptions.hybridHancomAiRegionlistStrategy) {
|
||||
convertOptions.hybridHancomAiRegionlistStrategy = cliOptions.hybridHancomAiRegionlistStrategy;
|
||||
}
|
||||
if (cliOptions.hybridHancomAiOcrStrategy) {
|
||||
convertOptions.hybridHancomAiOcrStrategy = cliOptions.hybridHancomAiOcrStrategy;
|
||||
}
|
||||
if (cliOptions.hybridHancomAiImageCache) {
|
||||
convertOptions.hybridHancomAiImageCache = cliOptions.hybridHancomAiImageCache;
|
||||
}
|
||||
if (cliOptions.toStdout) {
|
||||
convertOptions.toStdout = true;
|
||||
}
|
||||
if (cliOptions.threads) {
|
||||
convertOptions.threads = cliOptions.threads;
|
||||
}
|
||||
|
||||
return convertOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build CLI arguments array from ConvertOptions.
|
||||
*/
|
||||
export function buildArgs(options: ConvertOptions): string[] {
|
||||
const args: string[] = [];
|
||||
|
||||
if (options.outputDir) {
|
||||
args.push('--output-dir', options.outputDir);
|
||||
}
|
||||
if (options.password) {
|
||||
args.push('--password', options.password);
|
||||
}
|
||||
if (options.format) {
|
||||
if (Array.isArray(options.format)) {
|
||||
if (options.format.length > 0) {
|
||||
args.push('--format', options.format.join(','));
|
||||
}
|
||||
} else {
|
||||
args.push('--format', options.format);
|
||||
}
|
||||
}
|
||||
if (options.quiet) {
|
||||
args.push('--quiet');
|
||||
}
|
||||
if (options.contentSafetyOff) {
|
||||
if (Array.isArray(options.contentSafetyOff)) {
|
||||
if (options.contentSafetyOff.length > 0) {
|
||||
args.push('--content-safety-off', options.contentSafetyOff.join(','));
|
||||
}
|
||||
} else {
|
||||
args.push('--content-safety-off', options.contentSafetyOff);
|
||||
}
|
||||
}
|
||||
if (options.sanitize) {
|
||||
args.push('--sanitize');
|
||||
}
|
||||
if (options.keepLineBreaks) {
|
||||
args.push('--keep-line-breaks');
|
||||
}
|
||||
if (options.replaceInvalidChars) {
|
||||
args.push('--replace-invalid-chars', options.replaceInvalidChars);
|
||||
}
|
||||
if (options.useStructTree) {
|
||||
args.push('--use-struct-tree');
|
||||
}
|
||||
if (options.tableMethod) {
|
||||
args.push('--table-method', options.tableMethod);
|
||||
}
|
||||
if (options.readingOrder) {
|
||||
args.push('--reading-order', options.readingOrder);
|
||||
}
|
||||
if (options.markdownPageSeparator) {
|
||||
args.push('--markdown-page-separator', options.markdownPageSeparator);
|
||||
}
|
||||
if (options.markdownWithHtml) {
|
||||
args.push('--markdown-with-html');
|
||||
}
|
||||
if (options.textPageSeparator) {
|
||||
args.push('--text-page-separator', options.textPageSeparator);
|
||||
}
|
||||
if (options.htmlPageSeparator) {
|
||||
args.push('--html-page-separator', options.htmlPageSeparator);
|
||||
}
|
||||
if (options.imageOutput) {
|
||||
args.push('--image-output', options.imageOutput);
|
||||
}
|
||||
if (options.imageFormat) {
|
||||
args.push('--image-format', options.imageFormat);
|
||||
}
|
||||
if (options.imageDir) {
|
||||
args.push('--image-dir', options.imageDir);
|
||||
}
|
||||
if (options.pages) {
|
||||
args.push('--pages', options.pages);
|
||||
}
|
||||
if (options.includeHeaderFooter) {
|
||||
args.push('--include-header-footer');
|
||||
}
|
||||
if (options.detectStrikethrough) {
|
||||
args.push('--detect-strikethrough');
|
||||
}
|
||||
if (options.hybrid) {
|
||||
args.push('--hybrid', options.hybrid);
|
||||
}
|
||||
if (options.hybridMode) {
|
||||
args.push('--hybrid-mode', options.hybridMode);
|
||||
}
|
||||
if (options.hybridUrl) {
|
||||
args.push('--hybrid-url', options.hybridUrl);
|
||||
}
|
||||
if (options.hybridTimeout) {
|
||||
args.push('--hybrid-timeout', options.hybridTimeout);
|
||||
}
|
||||
if (options.hybridFallback) {
|
||||
args.push('--hybrid-fallback');
|
||||
}
|
||||
if (options.hybridHancomAiRegionlistStrategy) {
|
||||
args.push('--hybrid-hancom-ai-regionlist-strategy', options.hybridHancomAiRegionlistStrategy);
|
||||
}
|
||||
if (options.hybridHancomAiOcrStrategy) {
|
||||
args.push('--hybrid-hancom-ai-ocr-strategy', options.hybridHancomAiOcrStrategy);
|
||||
}
|
||||
if (options.hybridHancomAiImageCache) {
|
||||
args.push('--hybrid-hancom-ai-image-cache', options.hybridHancomAiImageCache);
|
||||
}
|
||||
if (options.toStdout) {
|
||||
args.push('--to-stdout');
|
||||
}
|
||||
if (options.threads) {
|
||||
args.push('--threads', options.threads);
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
import { spawn } from 'child_process';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import { StringDecoder } from 'string_decoder';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Re-export types and utilities from auto-generated file
|
||||
export type { ConvertOptions } from './convert-options.generated.js';
|
||||
export { buildArgs } from './convert-options.generated.js';
|
||||
import type { ConvertOptions } from './convert-options.generated.js';
|
||||
import { buildArgs } from './convert-options.generated.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const JAR_NAME = 'opendataloader-pdf-cli.jar';
|
||||
|
||||
interface JarExecutionOptions {
|
||||
/**
|
||||
* When true, forwards Java's stdout and stderr chunks to the parent
|
||||
* process in real time as well as accumulating them. Used by the bundled
|
||||
* CLI so long-running conversions show progress as it happens.
|
||||
*/
|
||||
streamOutput?: boolean;
|
||||
}
|
||||
|
||||
function executeJar(args: string[], executionOptions: JarExecutionOptions = {}): Promise<string> {
|
||||
const { streamOutput = false } = executionOptions;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const jarPath = path.join(__dirname, '..', 'lib', JAR_NAME);
|
||||
|
||||
if (!fs.existsSync(jarPath)) {
|
||||
return reject(
|
||||
new Error(`JAR file not found at ${jarPath}. Please run the build script first.`),
|
||||
);
|
||||
}
|
||||
|
||||
const command = 'java';
|
||||
// Force headless AWT so macOS doesn't surface a Dock icon (and steal focus)
|
||||
// every time the JVM touches ImageIO/PDFBox rendering. Safe on all OSes —
|
||||
// the CLI never opens a UI window, only manipulates BufferedImages.
|
||||
const commandArgs = [
|
||||
'-Djava.awt.headless=true',
|
||||
'-Dapple.awt.UIElement=true',
|
||||
'-jar',
|
||||
jarPath,
|
||||
...args,
|
||||
];
|
||||
|
||||
const javaProcess = spawn(command, commandArgs);
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
// StringDecoder buffers incomplete multi-byte UTF-8 sequences across
|
||||
// chunk boundaries — Buffer.toString() alone would emit replacement
|
||||
// characters when, e.g., a 3-byte Korean codepoint splits across two
|
||||
// 'data' events. One decoder per stream so they don't share state.
|
||||
const stdoutDecoder = new StringDecoder('utf8');
|
||||
const stderrDecoder = new StringDecoder('utf8');
|
||||
|
||||
javaProcess.stdout.on('data', (data: Buffer) => {
|
||||
const chunk = stdoutDecoder.write(data);
|
||||
if (chunk.length === 0) return;
|
||||
if (streamOutput) {
|
||||
// Stream-only: don't also accumulate, or a multi-hour conversion
|
||||
// would buffer its entire (potentially gigabyte) stdout in memory
|
||||
// for no consumer.
|
||||
process.stdout.write(chunk);
|
||||
} else {
|
||||
stdout += chunk;
|
||||
}
|
||||
});
|
||||
|
||||
javaProcess.stderr.on('data', (data: Buffer) => {
|
||||
const chunk = stderrDecoder.write(data);
|
||||
if (chunk.length === 0) return;
|
||||
if (streamOutput) {
|
||||
process.stderr.write(chunk);
|
||||
}
|
||||
// stderr is always accumulated (progress logs are small and we need
|
||||
// them for error messages on non-zero exit).
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
javaProcess.on('close', (code) => {
|
||||
// Flush any trailing bytes the decoder is still holding (always emit
|
||||
// them — if we drop them on error paths, error messages with non-ASCII
|
||||
// characters lose their tail).
|
||||
const stdoutTail = stdoutDecoder.end();
|
||||
const stderrTail = stderrDecoder.end();
|
||||
if (stdoutTail.length > 0) {
|
||||
if (streamOutput) {
|
||||
process.stdout.write(stdoutTail);
|
||||
} else {
|
||||
stdout += stdoutTail;
|
||||
}
|
||||
}
|
||||
if (stderrTail.length > 0) {
|
||||
if (streamOutput) process.stderr.write(stderrTail);
|
||||
stderr += stderrTail;
|
||||
}
|
||||
|
||||
if (code === 0) {
|
||||
resolve(stdout);
|
||||
} else {
|
||||
const errorOutput = stderr || stdout;
|
||||
const error = new Error(
|
||||
`The opendataloader-pdf CLI exited with code ${code}.\n\n${errorOutput}`,
|
||||
);
|
||||
// Tag so the CLI can suppress re-printing this message — Java's
|
||||
// stderr was already streamed live to the parent in CLI mode, and
|
||||
// re-printing risks leaking anything sensitive Java logged
|
||||
// (e.g. a --password value echoed by an underlying library).
|
||||
(error as Error & { isJavaExit?: boolean }).isJavaExit = true;
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
javaProcess.on('error', (err: Error) => {
|
||||
if (err.message.includes('ENOENT')) {
|
||||
reject(
|
||||
new Error(
|
||||
"'java' command not found. Please ensure Java is installed and in your system's PATH.",
|
||||
),
|
||||
);
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function buildJarArgs(
|
||||
inputPaths: string | string[],
|
||||
options: ConvertOptions,
|
||||
): string[] | Error {
|
||||
const inputList = Array.isArray(inputPaths) ? inputPaths : [inputPaths];
|
||||
if (inputList.length === 0) {
|
||||
return new Error('At least one input path must be provided.');
|
||||
}
|
||||
|
||||
for (const input of inputList) {
|
||||
if (!fs.existsSync(input)) {
|
||||
return new Error(`Input file or folder not found: ${input}`);
|
||||
}
|
||||
}
|
||||
|
||||
return [...inputList, ...buildArgs(options)];
|
||||
}
|
||||
|
||||
export function convert(
|
||||
inputPaths: string | string[],
|
||||
options: ConvertOptions = {},
|
||||
): Promise<string> {
|
||||
const argsOrError = buildJarArgs(inputPaths, options);
|
||||
if (argsOrError instanceof Error) {
|
||||
return Promise.reject(argsOrError);
|
||||
}
|
||||
// Library API: never streams to the parent process. Returns the full stdout
|
||||
// string so callers can do `const out = await convert(...)` without surprise
|
||||
// side-effects on process.stdout / process.stderr.
|
||||
return executeJar(argsOrError, { streamOutput: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal entry point used by the bundled CLI. Streams Java's stdout and
|
||||
* stderr to the parent process in real time (so long-running conversions like
|
||||
* hybrid mode show progress as it happens) and resolves without a stdout
|
||||
* payload — preventing the caller from re-printing what was already streamed.
|
||||
*
|
||||
* Not part of the public API: do not import this from application code. Use
|
||||
* {@link convert} instead.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
export async function _runForCli(
|
||||
inputPaths: string | string[],
|
||||
options: ConvertOptions = {},
|
||||
): Promise<void> {
|
||||
const argsOrError = buildJarArgs(inputPaths, options);
|
||||
if (argsOrError instanceof Error) {
|
||||
throw argsOrError;
|
||||
}
|
||||
await executeJar(argsOrError, { streamOutput: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use `convert()` and `ConvertOptions` instead. This function will be removed in a future version.
|
||||
*/
|
||||
export interface RunOptions {
|
||||
outputFolder?: string;
|
||||
password?: string;
|
||||
replaceInvalidChars?: string;
|
||||
generateMarkdown?: boolean;
|
||||
generateHtml?: boolean;
|
||||
generateAnnotatedPdf?: boolean;
|
||||
keepLineBreaks?: boolean;
|
||||
contentSafetyOff?: string;
|
||||
htmlInMarkdown?: boolean;
|
||||
addImageToMarkdown?: boolean;
|
||||
noJson?: boolean;
|
||||
debug?: boolean;
|
||||
useStructTree?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use `convert()` instead. This function will be removed in a future version.
|
||||
*/
|
||||
export function run(inputPath: string, options: RunOptions = {}): Promise<string> {
|
||||
console.warn(
|
||||
'Warning: run() is deprecated and will be removed in a future version. Use convert() instead.',
|
||||
);
|
||||
|
||||
// Build format array based on legacy boolean options
|
||||
const formats: string[] = [];
|
||||
if (!options.noJson) {
|
||||
formats.push('json');
|
||||
}
|
||||
if (options.generateMarkdown) {
|
||||
if (options.addImageToMarkdown) {
|
||||
formats.push('markdown-with-images');
|
||||
} else if (options.htmlInMarkdown) {
|
||||
formats.push('markdown-with-html');
|
||||
} else {
|
||||
formats.push('markdown');
|
||||
}
|
||||
}
|
||||
if (options.generateHtml) {
|
||||
formats.push('html');
|
||||
}
|
||||
if (options.generateAnnotatedPdf) {
|
||||
formats.push('pdf');
|
||||
}
|
||||
|
||||
return convert(inputPath, {
|
||||
outputDir: options.outputFolder,
|
||||
password: options.password,
|
||||
replaceInvalidChars: options.replaceInvalidChars,
|
||||
keepLineBreaks: options.keepLineBreaks,
|
||||
contentSafetyOff: options.contentSafetyOff,
|
||||
useStructTree: options.useStructTree,
|
||||
format: formats.length > 0 ? formats : undefined,
|
||||
quiet: !options.debug,
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* Unit tests for auto-generated convert-options functions
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
buildArgs,
|
||||
buildConvertOptions,
|
||||
ConvertOptions,
|
||||
CliOptions,
|
||||
} from '../src/convert-options.generated';
|
||||
|
||||
describe('buildArgs()', () => {
|
||||
it('should return empty array for empty options', () => {
|
||||
const args = buildArgs({});
|
||||
expect(args).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle string options', () => {
|
||||
const args = buildArgs({
|
||||
outputDir: '/output',
|
||||
password: 'secret',
|
||||
readingOrder: 'xycut',
|
||||
});
|
||||
expect(args).toEqual([
|
||||
'--output-dir', '/output',
|
||||
'--password', 'secret',
|
||||
'--reading-order', 'xycut',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should handle boolean options', () => {
|
||||
const args = buildArgs({
|
||||
quiet: true,
|
||||
keepLineBreaks: true,
|
||||
});
|
||||
expect(args).toEqual([
|
||||
'--quiet',
|
||||
'--keep-line-breaks',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should handle sanitize boolean', () => {
|
||||
const options: ConvertOptions = { sanitize: true };
|
||||
const args = buildArgs(options);
|
||||
expect(args).toEqual(['--sanitize']);
|
||||
});
|
||||
|
||||
it('should not include sanitize when false', () => {
|
||||
const options: ConvertOptions = { sanitize: false };
|
||||
const args = buildArgs(options);
|
||||
expect(args).toEqual([]);
|
||||
});
|
||||
|
||||
it('should handle list options with string value', () => {
|
||||
const args = buildArgs({
|
||||
format: 'json,markdown',
|
||||
contentSafetyOff: 'all',
|
||||
});
|
||||
expect(args).toEqual([
|
||||
'--format', 'json,markdown',
|
||||
'--content-safety-off', 'all',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should handle list options with array value', () => {
|
||||
const args = buildArgs({
|
||||
format: ['json', 'markdown', 'html'],
|
||||
contentSafetyOff: ['hidden-text', 'off-page'],
|
||||
});
|
||||
expect(args).toEqual([
|
||||
'--format', 'json,markdown,html',
|
||||
'--content-safety-off', 'hidden-text,off-page',
|
||||
]);
|
||||
});
|
||||
|
||||
it('should handle all options together', () => {
|
||||
const options: ConvertOptions = {
|
||||
outputDir: '/output',
|
||||
password: 'secret',
|
||||
format: ['json', 'markdown'],
|
||||
quiet: true,
|
||||
contentSafetyOff: 'all',
|
||||
keepLineBreaks: true,
|
||||
replaceInvalidChars: '_',
|
||||
useStructTree: true,
|
||||
tableMethod: 'cluster',
|
||||
readingOrder: 'xycut',
|
||||
markdownPageSeparator: '---',
|
||||
textPageSeparator: '\\n\\n',
|
||||
htmlPageSeparator: '<hr>',
|
||||
imageOutput: 'external',
|
||||
imageFormat: 'jpeg',
|
||||
sanitize: true,
|
||||
};
|
||||
|
||||
const args = buildArgs(options);
|
||||
|
||||
expect(args).toContain('--output-dir');
|
||||
expect(args).toContain('/output');
|
||||
expect(args).toContain('--password');
|
||||
expect(args).toContain('secret');
|
||||
expect(args).toContain('--format');
|
||||
expect(args).toContain('json,markdown');
|
||||
expect(args).toContain('--quiet');
|
||||
expect(args).toContain('--content-safety-off');
|
||||
expect(args).toContain('all');
|
||||
expect(args).toContain('--keep-line-breaks');
|
||||
expect(args).toContain('--replace-invalid-chars');
|
||||
expect(args).toContain('_');
|
||||
expect(args).toContain('--use-struct-tree');
|
||||
expect(args).toContain('--table-method');
|
||||
expect(args).toContain('cluster');
|
||||
expect(args).toContain('--reading-order');
|
||||
expect(args).toContain('xycut');
|
||||
expect(args).toContain('--markdown-page-separator');
|
||||
expect(args).toContain('---');
|
||||
expect(args).toContain('--image-output');
|
||||
expect(args).toContain('external');
|
||||
expect(args).toContain('--image-format');
|
||||
expect(args).toContain('jpeg');
|
||||
expect(args).toContain('--sanitize');
|
||||
});
|
||||
|
||||
it('should not include undefined options', () => {
|
||||
const args = buildArgs({
|
||||
outputDir: '/output',
|
||||
quiet: false,
|
||||
keepLineBreaks: false,
|
||||
});
|
||||
expect(args).toEqual(['--output-dir', '/output']);
|
||||
});
|
||||
|
||||
it('should skip empty arrays for list options', () => {
|
||||
const args = buildArgs({
|
||||
format: [],
|
||||
contentSafetyOff: [],
|
||||
outputDir: '/output',
|
||||
});
|
||||
expect(args).toEqual(['--output-dir', '/output']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildConvertOptions()', () => {
|
||||
it('should return empty object for empty CLI options', () => {
|
||||
const result = buildConvertOptions({});
|
||||
expect(result).toEqual({});
|
||||
});
|
||||
|
||||
it('should convert string options', () => {
|
||||
const cliOptions: CliOptions = {
|
||||
outputDir: '/output',
|
||||
password: 'secret',
|
||||
readingOrder: 'xycut',
|
||||
imageFormat: 'png',
|
||||
};
|
||||
const result = buildConvertOptions(cliOptions);
|
||||
expect(result).toEqual({
|
||||
outputDir: '/output',
|
||||
password: 'secret',
|
||||
readingOrder: 'xycut',
|
||||
imageFormat: 'png',
|
||||
});
|
||||
});
|
||||
|
||||
it('should convert boolean options', () => {
|
||||
const cliOptions: CliOptions = {
|
||||
quiet: true,
|
||||
keepLineBreaks: true,
|
||||
useStructTree: true,
|
||||
};
|
||||
const result = buildConvertOptions(cliOptions);
|
||||
expect(result).toEqual({
|
||||
quiet: true,
|
||||
keepLineBreaks: true,
|
||||
useStructTree: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should not include false boolean options', () => {
|
||||
const cliOptions: CliOptions = {
|
||||
outputDir: '/output',
|
||||
quiet: false,
|
||||
keepLineBreaks: false,
|
||||
};
|
||||
const result = buildConvertOptions(cliOptions);
|
||||
expect(result).toEqual({
|
||||
outputDir: '/output',
|
||||
});
|
||||
});
|
||||
|
||||
it('should pass through all provided options', () => {
|
||||
const cliOptions: CliOptions = {
|
||||
outputDir: '/output',
|
||||
password: 'secret',
|
||||
format: 'json,markdown',
|
||||
quiet: true,
|
||||
contentSafetyOff: 'all',
|
||||
keepLineBreaks: true,
|
||||
replaceInvalidChars: '_',
|
||||
useStructTree: true,
|
||||
tableMethod: 'cluster',
|
||||
readingOrder: 'xycut',
|
||||
markdownPageSeparator: '---',
|
||||
textPageSeparator: '\\n',
|
||||
htmlPageSeparator: '<hr>',
|
||||
imageOutput: 'external',
|
||||
imageFormat: 'jpeg',
|
||||
};
|
||||
const result = buildConvertOptions(cliOptions);
|
||||
expect(result.outputDir).toBe('/output');
|
||||
expect(result.password).toBe('secret');
|
||||
expect(result.format).toBe('json,markdown');
|
||||
expect(result.quiet).toBe(true);
|
||||
expect(result.contentSafetyOff).toBe('all');
|
||||
expect(result.keepLineBreaks).toBe(true);
|
||||
expect(result.replaceInvalidChars).toBe('_');
|
||||
expect(result.useStructTree).toBe(true);
|
||||
expect(result.tableMethod).toBe('cluster');
|
||||
expect(result.readingOrder).toBe('xycut');
|
||||
expect(result.markdownPageSeparator).toBe('---');
|
||||
expect(result.textPageSeparator).toBe('\\n');
|
||||
expect(result.htmlPageSeparator).toBe('<hr>');
|
||||
expect(result.imageOutput).toBe('external');
|
||||
expect(result.imageFormat).toBe('jpeg');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
/**
|
||||
* Integration tests that actually run the JAR (slow)
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { convert } from '../src/index';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const rootDir = path.resolve(__dirname, '..', '..', '..');
|
||||
const inputPdf = path.join(rootDir, 'samples', 'pdf', '1901.03003.pdf');
|
||||
const tempDir = path.join(__dirname, 'temp', 'convert');
|
||||
|
||||
describe('convert() integration', () => {
|
||||
beforeAll(() => {
|
||||
if (fs.existsSync(tempDir)) {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
if (fs.existsSync(tempDir)) {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('should generate output file', async () => {
|
||||
await convert(inputPdf, {
|
||||
outputDir: tempDir,
|
||||
format: 'json',
|
||||
quiet: true,
|
||||
});
|
||||
|
||||
const outputFile = path.join(tempDir, '1901.03003.json');
|
||||
expect(fs.existsSync(outputFile)).toBe(true);
|
||||
expect(fs.statSync(outputFile).size).toBeGreaterThan(0);
|
||||
}, 30000);
|
||||
});
|
||||
@@ -0,0 +1,288 @@
|
||||
/**
|
||||
* Unit tests for the JAR execution layer's stdout/stderr handling.
|
||||
*
|
||||
* These tests use a fake child process (no real Java) so we can deterministically
|
||||
* assert the streaming/buffering contract:
|
||||
* - convert() — library API: never writes to process.stdout, returns full stdout
|
||||
* - _runForCli() — CLI helper: streams stdout/stderr to the parent in real time,
|
||||
* does not return the stdout payload (the caller must not re-print it)
|
||||
*
|
||||
* Issue #398 reproducer: a long-running conversion (think hybrid mode, 1h+) must
|
||||
* surface progress via stderr without the CLI double-printing the result on close.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
|
||||
import { EventEmitter } from 'events';
|
||||
|
||||
// Mock child_process so we control spawn() entirely (no real Java process).
|
||||
vi.mock('child_process', () => ({
|
||||
spawn: vi.fn(),
|
||||
}));
|
||||
|
||||
// Mock fs so JAR-file-exists and input-file-exists checks always pass.
|
||||
vi.mock('fs', () => ({
|
||||
existsSync: vi.fn().mockReturnValue(true),
|
||||
}));
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { convert, _runForCli } from '../src/index';
|
||||
|
||||
class FakeProcess extends EventEmitter {
|
||||
stdout = new EventEmitter() as EventEmitter & { pipe?: unknown };
|
||||
stderr = new EventEmitter() as EventEmitter & { pipe?: unknown };
|
||||
}
|
||||
|
||||
function makeFakeSpawn(): {
|
||||
proc: FakeProcess;
|
||||
spawnMock: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
const proc = new FakeProcess();
|
||||
const spawnMock = spawn as unknown as ReturnType<typeof vi.fn>;
|
||||
spawnMock.mockReturnValue(proc as unknown as ReturnType<typeof spawn>);
|
||||
return { proc, spawnMock };
|
||||
}
|
||||
|
||||
/** Yield to the microtask queue so Promise callbacks fire. */
|
||||
const tick = () => new Promise<void>((r) => setImmediate(r));
|
||||
|
||||
describe('executeJar — library API (convert)', () => {
|
||||
let stdoutSpy: ReturnType<typeof vi.spyOn>;
|
||||
let stderrSpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(() => {
|
||||
stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
||||
stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
stdoutSpy.mockRestore();
|
||||
stderrSpy.mockRestore();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('returns the full stdout string on success', async () => {
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = convert('input.pdf', { quiet: true });
|
||||
|
||||
proc.stdout.emit('data', Buffer.from('Lorem '));
|
||||
proc.stdout.emit('data', Buffer.from('ipsum dolor'));
|
||||
proc.emit('close', 0);
|
||||
|
||||
await expect(promise).resolves.toBe('Lorem ipsum dolor');
|
||||
});
|
||||
|
||||
it('does not write to process.stdout (no streaming side-effect)', async () => {
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = convert('input.pdf', { quiet: true });
|
||||
|
||||
proc.stdout.emit('data', Buffer.from('result text'));
|
||||
proc.emit('close', 0);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(stdoutSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not write to process.stderr (no streaming side-effect)', async () => {
|
||||
// Library callers should not have stderr leaked into their parent process.
|
||||
// Java's --quiet silences stderr at the source for typical library use,
|
||||
// but the contract holds even if Java emits to stderr.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = convert('input.pdf', { quiet: true });
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('정보: progress'));
|
||||
proc.stdout.emit('data', Buffer.from('result'));
|
||||
proc.emit('close', 0);
|
||||
|
||||
await promise;
|
||||
|
||||
expect(stderrSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('rejects with stderr in the error message on non-zero exit', async () => {
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = convert('input.pdf', { quiet: true });
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('Java stack trace'));
|
||||
proc.emit('close', 1);
|
||||
|
||||
await expect(promise).rejects.toThrow(/exited with code 1/);
|
||||
await expect(promise).rejects.toThrow(/Java stack trace/);
|
||||
});
|
||||
|
||||
it('tags the rejection with isJavaExit so the CLI can suppress re-printing', async () => {
|
||||
// The CLI relies on this tag to avoid duplicating stderr that was already
|
||||
// streamed live (and to avoid re-surfacing anything sensitive Java logged).
|
||||
// Library callers can ignore the tag — message and behavior are unchanged.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = convert('input.pdf', { quiet: true });
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('Bad password: hunter2'));
|
||||
proc.emit('close', 1);
|
||||
|
||||
await expect(promise).rejects.toMatchObject({ isJavaExit: true });
|
||||
});
|
||||
|
||||
it('reassembles multi-byte UTF-8 codepoints split across chunks', async () => {
|
||||
// Java's progress logs are localized; '정' = 0xEC 0xA0 0x95 (3 bytes).
|
||||
// If the OS hands us this codepoint split across two 'data' events, a
|
||||
// naive Buffer.toString() emits two replacement characters. Streaming
|
||||
// is meant to be byte-faithful to what Java printed, so we must
|
||||
// reassemble across the boundary.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = convert('input.pdf', { quiet: true });
|
||||
|
||||
proc.stdout.emit('data', Buffer.from([0xEC, 0xA0])); // first 2 bytes of '정'
|
||||
proc.stdout.emit('data', Buffer.from([0x95])); // last byte of '정'
|
||||
proc.stdout.emit('data', Buffer.from('보', 'utf8')); // a clean codepoint
|
||||
proc.emit('close', 0);
|
||||
|
||||
await expect(promise).resolves.toBe('정보');
|
||||
});
|
||||
});
|
||||
|
||||
describe('executeJar — CLI helper (_runForCli)', () => {
|
||||
let stdoutSpy: ReturnType<typeof vi.spyOn>;
|
||||
let stderrSpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
beforeEach(() => {
|
||||
stdoutSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
||||
stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
stdoutSpy.mockRestore();
|
||||
stderrSpy.mockRestore();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('streams stdout chunks to process.stdout in real time', async () => {
|
||||
// The defining requirement for hybrid-mode-style long runs: the user must
|
||||
// see output as it arrives, not buffered until the process closes.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = _runForCli(['input.pdf']);
|
||||
|
||||
proc.stdout.emit('data', Buffer.from('chunk1'));
|
||||
await tick();
|
||||
expect(stdoutSpy).toHaveBeenCalledWith('chunk1');
|
||||
expect(stdoutSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
proc.stdout.emit('data', Buffer.from('chunk2'));
|
||||
await tick();
|
||||
expect(stdoutSpy).toHaveBeenCalledTimes(2);
|
||||
|
||||
proc.emit('close', 0);
|
||||
await promise;
|
||||
|
||||
// Critical: process.stdout was called exactly twice — once per chunk.
|
||||
// No extra "final flush" call. That's how we avoid the #398 double-write.
|
||||
expect(stdoutSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('streams stderr chunks to process.stderr in real time', async () => {
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = _runForCli(['input.pdf']);
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('정보: Number of pages: 14\n'));
|
||||
await tick();
|
||||
expect(stderrSpy).toHaveBeenCalledWith('정보: Number of pages: 14\n');
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('정보: Processing 14 pages\n'));
|
||||
await tick();
|
||||
expect(stderrSpy).toHaveBeenCalledTimes(2);
|
||||
|
||||
proc.emit('close', 0);
|
||||
await promise;
|
||||
});
|
||||
|
||||
it('preserves true interleaving of stderr and stdout chunks as they arrive', async () => {
|
||||
// Hybrid mode emits progress logs on stderr *throughout* the run, with
|
||||
// result chunks landing on stdout in between. Both streams must reach the
|
||||
// parent in arrival order without one buffering the other.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = _runForCli(['input.pdf']);
|
||||
|
||||
const calls: string[] = [];
|
||||
stdoutSpy.mockImplementation(((chunk: Buffer) => {
|
||||
calls.push('OUT:' + chunk.toString());
|
||||
return true;
|
||||
}) as never);
|
||||
stderrSpy.mockImplementation(((chunk: Buffer) => {
|
||||
calls.push('ERR:' + chunk.toString());
|
||||
return true;
|
||||
}) as never);
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('progress 1'));
|
||||
await tick();
|
||||
proc.stdout.emit('data', Buffer.from('result A'));
|
||||
await tick();
|
||||
proc.stderr.emit('data', Buffer.from('progress 2'));
|
||||
await tick();
|
||||
proc.stdout.emit('data', Buffer.from('result B'));
|
||||
await tick();
|
||||
proc.emit('close', 0);
|
||||
await promise;
|
||||
|
||||
expect(calls).toEqual([
|
||||
'ERR:progress 1',
|
||||
'OUT:result A',
|
||||
'ERR:progress 2',
|
||||
'OUT:result B',
|
||||
]);
|
||||
});
|
||||
|
||||
it('resolves without returning the stdout payload (caller must not re-print)', async () => {
|
||||
// _runForCli's contract: streaming has the side-effect, the resolved value
|
||||
// carries no stdout text. This is what blocks the CLI from double-printing.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = _runForCli(['input.pdf']);
|
||||
|
||||
proc.stdout.emit('data', Buffer.from('result text'));
|
||||
proc.emit('close', 0);
|
||||
|
||||
const resolved = await promise;
|
||||
|
||||
expect(resolved).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects with stderr in the error message on non-zero exit', async () => {
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = _runForCli(['input.pdf']);
|
||||
|
||||
proc.stderr.emit('data', Buffer.from('boom'));
|
||||
proc.emit('close', 2);
|
||||
|
||||
await expect(promise).rejects.toThrow(/exited with code 2/);
|
||||
await expect(promise).rejects.toThrow(/boom/);
|
||||
});
|
||||
|
||||
it('forwards multi-byte UTF-8 to process.stderr without splitting codepoints', async () => {
|
||||
// Defining the streaming contract for non-ASCII progress logs. Without a
|
||||
// StringDecoder, splitting '정' (0xEC 0xA0 0x95) across two chunks would
|
||||
// emit two replacement characters to the user's terminal — exactly the
|
||||
// failure mode the comment in index.ts warns about.
|
||||
const { proc } = makeFakeSpawn();
|
||||
const promise = _runForCli(['input.pdf']);
|
||||
|
||||
const forwarded: string[] = [];
|
||||
stderrSpy.mockImplementation(((chunk: string) => {
|
||||
forwarded.push(chunk);
|
||||
return true;
|
||||
}) as never);
|
||||
|
||||
proc.stderr.emit('data', Buffer.from([0xEC, 0xA0])); // partial '정'
|
||||
await tick();
|
||||
proc.stderr.emit('data', Buffer.from([0x95])); // completes '정'
|
||||
await tick();
|
||||
proc.stderr.emit('data', Buffer.from('보', 'utf8'));
|
||||
await tick();
|
||||
proc.emit('close', 0);
|
||||
await promise;
|
||||
|
||||
// No replacement characters anywhere in what the user saw.
|
||||
const joined = forwarded.join('');
|
||||
expect(joined).toBe('정보');
|
||||
expect(joined).not.toContain('�');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import { run, convert } from '../src/index';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const rootDir = path.resolve(__dirname, '..', '..', '..');
|
||||
const inputPdf = path.join(rootDir, 'samples', 'pdf', '1901.03003.pdf');
|
||||
const tempDir = path.join(__dirname, 'temp', 'run');
|
||||
|
||||
describe('opendataloader-pdf', () => {
|
||||
beforeAll(() => {
|
||||
// Clean up previous test runs
|
||||
if (fs.existsSync(tempDir)) {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
fs.mkdirSync(tempDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
// Clean up after tests
|
||||
if (fs.existsSync(tempDir)) {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('should process PDF and generate markdown output', async () => {
|
||||
console.log(`[TEST] Running opendataloader-pdf test...`);
|
||||
console.log(`[TEST] Input PDF: ${inputPdf}`);
|
||||
console.log(`[TEST] Output directory: ${tempDir}`);
|
||||
|
||||
await run(inputPdf, {
|
||||
outputFolder: tempDir,
|
||||
generateMarkdown: true,
|
||||
generateHtml: true,
|
||||
generateAnnotatedPdf: true,
|
||||
debug: true,
|
||||
});
|
||||
|
||||
expect(fs.existsSync(path.join(tempDir, '1901.03003.json'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tempDir, '1901.03003.md'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tempDir, '1901.03003.html'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tempDir, '1901.03003_annotated.pdf'))).toBe(true);
|
||||
}, 30000); // 30 second timeout for this test
|
||||
|
||||
it('should convert PDF with explicit formats using quiet mode', async () => {
|
||||
const convertDir = path.join(tempDir, 'convert');
|
||||
if (fs.existsSync(convertDir)) {
|
||||
fs.rmSync(convertDir, { recursive: true, force: true });
|
||||
}
|
||||
fs.mkdirSync(convertDir);
|
||||
|
||||
await convert([inputPdf], {
|
||||
outputDir: convertDir,
|
||||
format: ['json', 'text', 'html', 'pdf', 'markdown'],
|
||||
});
|
||||
|
||||
expect(fs.existsSync(path.join(convertDir, '1901.03003.json'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(convertDir, '1901.03003.txt'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(convertDir, '1901.03003.html'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(convertDir, '1901.03003.md'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(convertDir, '1901.03003_annotated.pdf'))).toBe(true);
|
||||
}, 30000);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Integration tests for the stdout/stderr streaming contract.
|
||||
*
|
||||
* Runs the bundled CLI as a real subprocess against a multi-page sample PDF.
|
||||
* Asserts the user-facing behavior that mock unit tests cannot prove:
|
||||
* - CLI never double-prints stdout (regression test for #398)
|
||||
* - Java's progress logs reach the parent's stderr in real time, before the
|
||||
* stdout payload finishes — the property that makes hour-long hybrid runs
|
||||
* observable
|
||||
* - Library API does not leak Java's output to the parent's stdout
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeAll } from 'vitest';
|
||||
import { spawn, spawnSync } from 'child_process';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import { fileURLToPath, pathToFileURL } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const rootDir = path.resolve(__dirname, '..', '..', '..');
|
||||
const cliPath = path.resolve(__dirname, '..', 'dist', 'cli.js');
|
||||
const jarPath = path.resolve(__dirname, '..', 'lib', 'opendataloader-pdf-cli.jar');
|
||||
const samplePdf = path.join(rootDir, 'samples', 'pdf', '2408.02509v1.pdf');
|
||||
|
||||
interface Capture {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
/** Wall-clock ms (since process spawn) of each chunk we received. */
|
||||
timeline: Array<{ stream: 'out' | 'err'; size: number; at: number }>;
|
||||
exitCode: number | null;
|
||||
}
|
||||
|
||||
function captureSubprocess(command: string, args: string[]): Promise<Capture> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = spawn(command, args);
|
||||
const start = Date.now();
|
||||
const cap: Capture = { stdout: '', stderr: '', timeline: [], exitCode: null };
|
||||
|
||||
proc.stdout.on('data', (chunk: Buffer) => {
|
||||
cap.stdout += chunk.toString();
|
||||
cap.timeline.push({ stream: 'out', size: chunk.length, at: Date.now() - start });
|
||||
});
|
||||
proc.stderr.on('data', (chunk: Buffer) => {
|
||||
cap.stderr += chunk.toString();
|
||||
cap.timeline.push({ stream: 'err', size: chunk.length, at: Date.now() - start });
|
||||
});
|
||||
proc.on('close', (code) => {
|
||||
cap.exitCode = code;
|
||||
resolve(cap);
|
||||
});
|
||||
proc.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
const runCli = (args: string[]) => captureSubprocess('node', [cliPath, ...args]);
|
||||
const runJar = (args: string[]) => captureSubprocess('java', ['-jar', jarPath, ...args]);
|
||||
|
||||
describe('CLI streaming contract', () => {
|
||||
beforeAll(() => {
|
||||
// Self-contained: build dist on demand so `pnpm test` works on a fresh
|
||||
// checkout without a separate `pnpm build` step. The JAR has to come from
|
||||
// the Maven build (it's not something Vitest should rebuild), so we still
|
||||
// surface a clear error if it's missing.
|
||||
if (!fs.existsSync(cliPath)) {
|
||||
const result = spawnSync(
|
||||
'pnpm',
|
||||
['exec', 'tsup', '--no-dts', 'src/index.ts', 'src/cli.ts',
|
||||
'--format', 'esm,cjs', '--shims', '--out-dir', 'dist'],
|
||||
{ cwd: path.resolve(__dirname, '..'), stdio: 'inherit' },
|
||||
);
|
||||
if (result.status !== 0) {
|
||||
throw new Error('Failed to build dist for streaming integration test');
|
||||
}
|
||||
}
|
||||
if (!fs.existsSync(jarPath)) {
|
||||
throw new Error(
|
||||
`Bundled JAR not found at ${jarPath} — build the Java module first ` +
|
||||
`(\`mvn package\` in java/, then \`pnpm run setup\` in this package).`,
|
||||
);
|
||||
}
|
||||
if (!fs.existsSync(samplePdf)) {
|
||||
throw new Error(`Sample PDF not found at ${samplePdf}`);
|
||||
}
|
||||
}, 60000);
|
||||
|
||||
it('prints --to-stdout output exactly once (regression for #398)', async () => {
|
||||
// Ground truth: invoke the JAR directly with --quiet so stderr is empty
|
||||
// and stdout carries only the result payload. The Node CLI must produce
|
||||
// the same bytes — no more, no less. Comparing against the JAR rather
|
||||
// than another Node-CLI run prevents a symmetric-mutation regression
|
||||
// (e.g., trimming/appending in both code paths) from sneaking past.
|
||||
const referenceCap = await runJar([samplePdf, '--quiet', '--format', 'text', '--to-stdout']);
|
||||
expect(referenceCap.exitCode).toBe(0);
|
||||
const referenceStdout = referenceCap.stdout;
|
||||
expect(referenceStdout.length).toBeGreaterThan(1000);
|
||||
|
||||
const cap = await runCli([samplePdf, '--format', 'text', '--to-stdout']);
|
||||
expect(cap.exitCode).toBe(0);
|
||||
|
||||
// Defining check: the no-flag Node CLI must produce the *exact same*
|
||||
// stdout bytes as the JAR. Pre-fix this came out at 2x because of
|
||||
// double-write. Length-only comparison would miss byte-substituting
|
||||
// mutations of identical length, so we compare the full payload.
|
||||
expect(cap.stdout).toBe(referenceStdout);
|
||||
}, 60000);
|
||||
|
||||
it('forwards Java progress logs to stderr in real time before stdout completes', async () => {
|
||||
const cap = await runCli([samplePdf, '--format', 'text', '--to-stdout']);
|
||||
expect(cap.exitCode).toBe(0);
|
||||
|
||||
// Java emits a "Number of pages" line during preprocessing — long before
|
||||
// text extraction finishes — to stderr.
|
||||
expect(cap.stderr).toMatch(/Number of pages/);
|
||||
|
||||
// The property that makes long runs observable: at least one stderr chunk
|
||||
// arrives before stdout finishes streaming. We compare *event indices*
|
||||
// (arrival order in cap.timeline) rather than millisecond timestamps to
|
||||
// stay deterministic — two events scheduled in the same tick will share
|
||||
// an `at` value but always have distinct indices.
|
||||
const firstErrIdx = cap.timeline.findIndex((e) => e.stream === 'err');
|
||||
let lastOutIdx = -1;
|
||||
for (let i = cap.timeline.length - 1; i >= 0; i--) {
|
||||
if (cap.timeline[i].stream === 'out') {
|
||||
lastOutIdx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
expect(firstErrIdx).toBeGreaterThanOrEqual(0);
|
||||
expect(lastOutIdx).toBeGreaterThanOrEqual(0);
|
||||
expect(firstErrIdx).toBeLessThan(lastOutIdx);
|
||||
}, 60000);
|
||||
|
||||
it('library convert() does not leak to the parent process stdio', async () => {
|
||||
// Spawn a tiny Node script that imports convert() and calls it. We capture
|
||||
// *that* process's stdio: convert() must not write to stdout/stderr itself.
|
||||
// The import specifier is a file:// URL so the harness works on Windows
|
||||
// (where path.resolve yields backslashes that ESM rejects).
|
||||
const distIndexUrl = pathToFileURL(path.resolve(__dirname, '..', 'dist', 'index.js')).href;
|
||||
const harness = `
|
||||
import { convert } from '${distIndexUrl}';
|
||||
const out = await convert(${JSON.stringify(samplePdf)}, {
|
||||
format: ['text'], toStdout: true, quiet: true,
|
||||
});
|
||||
// One intended write at the end so we can distinguish "the marker"
|
||||
// from any leakage caused by executeJar forwarding to process.stdout.
|
||||
process.stdout.write('LEN=' + out.length + '\\n');
|
||||
`;
|
||||
|
||||
// Place the harness in an isolated tmpdir — survives Ctrl-C / OOM cleanly
|
||||
// (OS reaps tmpdir trees) instead of polluting the source tree.
|
||||
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'odl-pdf-issue398-'));
|
||||
const tmpScript = path.join(tmpDir, 'harness.mjs');
|
||||
fs.writeFileSync(tmpScript, harness);
|
||||
try {
|
||||
const cap = await captureSubprocess('node', [tmpScript]);
|
||||
expect(cap.exitCode).toBe(0);
|
||||
// The only line on stdout must be our LEN= marker. Anything else means
|
||||
// executeJar leaked Java's stdout into the parent.
|
||||
expect(cap.stdout.trim().split('\n')).toHaveLength(1);
|
||||
expect(cap.stdout).toMatch(/^LEN=\d+\n$/);
|
||||
expect(cap.stderr).toBe('');
|
||||
} finally {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
}, 60000);
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2023",
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"outDir": "./dist",
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"declaration": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"stripInternal": true,
|
||||
"types": ["node"],
|
||||
"ignoreDeprecations": "6.0"
|
||||
},
|
||||
"include": ["src/**/*.ts"]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { defineConfig } from 'tsup';
|
||||
|
||||
export default defineConfig({
|
||||
clean: true,
|
||||
dts: true,
|
||||
entry: ['src/index.ts', 'src/cli.ts'],
|
||||
format: ['esm', 'cjs'],
|
||||
shims: true,
|
||||
sourcemap: true,
|
||||
outDir: 'dist',
|
||||
splitting: false,
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
/// <reference types="vitest" />
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
// Your test configuration options go here
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user