chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,41 @@
|
||||
// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
|
||||
// Run `npm run generate-options` to regenerate
|
||||
|
||||
import { Command } from 'commander';
|
||||
|
||||
/**
|
||||
* Register all CLI options on the given Commander program.
|
||||
*/
|
||||
export function registerCliOptions(program: Command): void {
|
||||
program.option('-o, --output-dir <value>', 'Directory where output files are written. Default: input file directory');
|
||||
program.option('-p, --password <value>', 'Password for encrypted PDF files');
|
||||
program.option('-f, --format <value>', 'Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output.');
|
||||
program.option('-q, --quiet', 'Suppress console logging output');
|
||||
program.option('--content-safety-off <value>', 'Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg');
|
||||
program.option('--sanitize', 'Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders');
|
||||
program.option('--keep-line-breaks', 'Preserve original line breaks in extracted text');
|
||||
program.option('--replace-invalid-chars <value>', 'Replacement character for invalid/unrecognized characters. Default: space');
|
||||
program.option('--use-struct-tree', 'Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality');
|
||||
program.option('--table-method <value>', 'Table detection method. Values: default (border-based), cluster (border + cluster). Default: default');
|
||||
program.option('--reading-order <value>', 'Reading order algorithm. Values: off, xycut. Default: xycut');
|
||||
program.option('--markdown-page-separator <value>', 'Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none');
|
||||
program.option('--markdown-with-html', 'Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown.');
|
||||
program.option('--text-page-separator <value>', 'Separator between pages in text output. Use %page-number% for page numbers. Default: none');
|
||||
program.option('--html-page-separator <value>', 'Separator between pages in HTML output. Use %page-number% for page numbers. Default: none');
|
||||
program.option('--image-output <value>', 'Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external');
|
||||
program.option('--image-format <value>', 'Output format for extracted images. Values: png, jpeg. Default: png');
|
||||
program.option('--image-dir <value>', 'Directory for extracted images (applies only with --image-output external)');
|
||||
program.option('--pages <value>', 'Pages to extract (e.g., "1,3,5-7"). Default: all pages');
|
||||
program.option('--include-header-footer', 'Include page headers and footers in output');
|
||||
program.option('--detect-strikethrough', 'Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)');
|
||||
program.option('--hybrid <value>', 'Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai');
|
||||
program.option('--hybrid-mode <value>', 'Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)');
|
||||
program.option('--hybrid-url <value>', 'Hybrid backend server URL (overrides default)');
|
||||
program.option('--hybrid-timeout <value>', 'Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0');
|
||||
program.option('--hybrid-fallback', 'Opt in to Java fallback on hybrid backend error (default: disabled)');
|
||||
program.option('--hybrid-hancom-ai-regionlist-strategy <value>', 'DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)');
|
||||
program.option('--hybrid-hancom-ai-ocr-strategy <value>', 'OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)');
|
||||
program.option('--hybrid-hancom-ai-image-cache <value>', 'Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk');
|
||||
program.option('--to-stdout', 'Write output to stdout instead of file (single format only)');
|
||||
program.option('--threads <value>', 'Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode');
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
import { Command, CommanderError } from 'commander';
|
||||
import { _runForCli } from './index.js';
|
||||
import { CliOptions, buildConvertOptions } from './convert-options.generated.js';
|
||||
import { registerCliOptions } from './cli-options.generated.js';
|
||||
|
||||
function createProgram(): Command {
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
.name('opendataloader-pdf')
|
||||
.usage('[options] <input...>')
|
||||
.description('Convert PDFs using the OpenDataLoader CLI.')
|
||||
.showHelpAfterError("Use '--help' to see available options.")
|
||||
.showSuggestionAfterError(false)
|
||||
.argument('<input...>', 'Input files or directories to convert');
|
||||
|
||||
// Register CLI options from auto-generated file
|
||||
registerCliOptions(program);
|
||||
|
||||
program.configureOutput({
|
||||
writeErr: (str) => {
|
||||
console.error(str.trimEnd());
|
||||
},
|
||||
outputError: (str, write) => {
|
||||
write(str);
|
||||
},
|
||||
});
|
||||
|
||||
return program;
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const program = createProgram();
|
||||
|
||||
program.exitOverride();
|
||||
|
||||
try {
|
||||
program.parse(process.argv);
|
||||
} catch (err) {
|
||||
if (err instanceof CommanderError) {
|
||||
if (err.code === 'commander.helpDisplayed') {
|
||||
return 0;
|
||||
}
|
||||
return err.exitCode ?? 1;
|
||||
}
|
||||
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(message);
|
||||
console.error("Use '--help' to see available options.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const cliOptions = program.opts<CliOptions>();
|
||||
const inputPaths = program.args;
|
||||
const convertOptions = buildConvertOptions(cliOptions);
|
||||
|
||||
try {
|
||||
// _runForCli streams stdout/stderr to the parent process as they arrive;
|
||||
// we deliberately do not re-print anything here. (Issue #398.)
|
||||
await _runForCli(inputPaths, convertOptions);
|
||||
return 0;
|
||||
} catch (err) {
|
||||
// Subprocess-exit errors are already on the user's terminal via the live
|
||||
// stderr stream — re-printing would duplicate output and risk leaking
|
||||
// anything sensitive Java logged (e.g. a --password value echoed by an
|
||||
// underlying library). Wrapper-side failures (JAR not found, java not in
|
||||
// PATH, bad input path) still need to be surfaced.
|
||||
const isJavaExit =
|
||||
err instanceof Error && (err as Error & { isJavaExit?: boolean }).isJavaExit === true;
|
||||
if (!isJavaExit) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.error(message);
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
main().then((code) => {
|
||||
if (code !== 0) {
|
||||
process.exit(code);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,325 @@
|
||||
// AUTO-GENERATED FROM options.json - DO NOT EDIT DIRECTLY
|
||||
// Run `npm run generate-options` to regenerate
|
||||
|
||||
/**
|
||||
* Options for the convert function.
|
||||
*/
|
||||
export interface ConvertOptions {
|
||||
/** Directory where output files are written. Default: input file directory */
|
||||
outputDir?: string;
|
||||
/** Password for encrypted PDF files */
|
||||
password?: string;
|
||||
/** Output formats (comma-separated). Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. For HTML inside Markdown use --markdown-with-html. For image extraction control use --image-output. */
|
||||
format?: string | string[];
|
||||
/** Suppress console logging output */
|
||||
quiet?: boolean;
|
||||
/** Disable content safety filters. Values: all, hidden-text, off-page, tiny, hidden-ocg */
|
||||
contentSafetyOff?: string | string[];
|
||||
/** Enable sensitive data sanitization. Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders */
|
||||
sanitize?: boolean;
|
||||
/** Preserve original line breaks in extracted text */
|
||||
keepLineBreaks?: boolean;
|
||||
/** Replacement character for invalid/unrecognized characters. Default: space */
|
||||
replaceInvalidChars?: string;
|
||||
/** Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality */
|
||||
useStructTree?: boolean;
|
||||
/** Table detection method. Values: default (border-based), cluster (border + cluster). Default: default */
|
||||
tableMethod?: string;
|
||||
/** Reading order algorithm. Values: off, xycut. Default: xycut */
|
||||
readingOrder?: string;
|
||||
/** Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none */
|
||||
markdownPageSeparator?: string;
|
||||
/** Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. Implies --format markdown. */
|
||||
markdownWithHtml?: boolean;
|
||||
/** Separator between pages in text output. Use %page-number% for page numbers. Default: none */
|
||||
textPageSeparator?: string;
|
||||
/** Separator between pages in HTML output. Use %page-number% for page numbers. Default: none */
|
||||
htmlPageSeparator?: string;
|
||||
/** Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external */
|
||||
imageOutput?: string;
|
||||
/** Output format for extracted images. Values: png, jpeg. Default: png */
|
||||
imageFormat?: string;
|
||||
/** Directory for extracted images (applies only with --image-output external) */
|
||||
imageDir?: string;
|
||||
/** Pages to extract (e.g., "1,3,5-7"). Default: all pages */
|
||||
pages?: string;
|
||||
/** Include page headers and footers in output */
|
||||
includeHeaderFooter?: boolean;
|
||||
/** Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental) */
|
||||
detectStrikethrough?: boolean;
|
||||
/** Hybrid backend (requires a running server). Quick start: pip install "opendataloader-pdf[hybrid]" && opendataloader-pdf-hybrid --port 5002. For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai */
|
||||
hybrid?: string;
|
||||
/** Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend) */
|
||||
hybridMode?: string;
|
||||
/** Hybrid backend server URL (overrides default) */
|
||||
hybridUrl?: string;
|
||||
/** Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0 */
|
||||
hybridTimeout?: string;
|
||||
/** Opt in to Java fallback on hybrid backend error (default: disabled) */
|
||||
hybridFallback?: boolean;
|
||||
/** DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list) */
|
||||
hybridHancomAiRegionlistStrategy?: string;
|
||||
/** OCR strategy. Requires --hybrid=hancom-ai. Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only) */
|
||||
hybridHancomAiOcrStrategy?: string;
|
||||
/** Page image cache backing. Requires --hybrid=hancom-ai. Values: memory (default), disk */
|
||||
hybridHancomAiImageCache?: string;
|
||||
/** Write output to stdout instead of file (single format only) */
|
||||
toStdout?: boolean;
|
||||
/** Number of worker threads for per-page processing. Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; output may vary slightly on some PDFs. Capped at the number of available CPU cores. Applies to the native Java pipeline only; ignored in --hybrid mode */
|
||||
threads?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Options as parsed from CLI (all values are strings from commander).
|
||||
*/
|
||||
export interface CliOptions {
|
||||
outputDir?: string;
|
||||
password?: string;
|
||||
format?: string;
|
||||
quiet?: boolean;
|
||||
contentSafetyOff?: string;
|
||||
sanitize?: boolean;
|
||||
keepLineBreaks?: boolean;
|
||||
replaceInvalidChars?: string;
|
||||
useStructTree?: boolean;
|
||||
tableMethod?: string;
|
||||
readingOrder?: string;
|
||||
markdownPageSeparator?: string;
|
||||
markdownWithHtml?: boolean;
|
||||
textPageSeparator?: string;
|
||||
htmlPageSeparator?: string;
|
||||
imageOutput?: string;
|
||||
imageFormat?: string;
|
||||
imageDir?: string;
|
||||
pages?: string;
|
||||
includeHeaderFooter?: boolean;
|
||||
detectStrikethrough?: boolean;
|
||||
hybrid?: string;
|
||||
hybridMode?: string;
|
||||
hybridUrl?: string;
|
||||
hybridTimeout?: string;
|
||||
hybridFallback?: boolean;
|
||||
hybridHancomAiRegionlistStrategy?: string;
|
||||
hybridHancomAiOcrStrategy?: string;
|
||||
hybridHancomAiImageCache?: string;
|
||||
toStdout?: boolean;
|
||||
threads?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert CLI options to ConvertOptions.
|
||||
*/
|
||||
export function buildConvertOptions(cliOptions: CliOptions): ConvertOptions {
|
||||
const convertOptions: ConvertOptions = {};
|
||||
|
||||
if (cliOptions.outputDir) {
|
||||
convertOptions.outputDir = cliOptions.outputDir;
|
||||
}
|
||||
if (cliOptions.password) {
|
||||
convertOptions.password = cliOptions.password;
|
||||
}
|
||||
if (cliOptions.format) {
|
||||
convertOptions.format = cliOptions.format;
|
||||
}
|
||||
if (cliOptions.quiet) {
|
||||
convertOptions.quiet = true;
|
||||
}
|
||||
if (cliOptions.contentSafetyOff) {
|
||||
convertOptions.contentSafetyOff = cliOptions.contentSafetyOff;
|
||||
}
|
||||
if (cliOptions.sanitize) {
|
||||
convertOptions.sanitize = true;
|
||||
}
|
||||
if (cliOptions.keepLineBreaks) {
|
||||
convertOptions.keepLineBreaks = true;
|
||||
}
|
||||
if (cliOptions.replaceInvalidChars) {
|
||||
convertOptions.replaceInvalidChars = cliOptions.replaceInvalidChars;
|
||||
}
|
||||
if (cliOptions.useStructTree) {
|
||||
convertOptions.useStructTree = true;
|
||||
}
|
||||
if (cliOptions.tableMethod) {
|
||||
convertOptions.tableMethod = cliOptions.tableMethod;
|
||||
}
|
||||
if (cliOptions.readingOrder) {
|
||||
convertOptions.readingOrder = cliOptions.readingOrder;
|
||||
}
|
||||
if (cliOptions.markdownPageSeparator) {
|
||||
convertOptions.markdownPageSeparator = cliOptions.markdownPageSeparator;
|
||||
}
|
||||
if (cliOptions.markdownWithHtml) {
|
||||
convertOptions.markdownWithHtml = true;
|
||||
}
|
||||
if (cliOptions.textPageSeparator) {
|
||||
convertOptions.textPageSeparator = cliOptions.textPageSeparator;
|
||||
}
|
||||
if (cliOptions.htmlPageSeparator) {
|
||||
convertOptions.htmlPageSeparator = cliOptions.htmlPageSeparator;
|
||||
}
|
||||
if (cliOptions.imageOutput) {
|
||||
convertOptions.imageOutput = cliOptions.imageOutput;
|
||||
}
|
||||
if (cliOptions.imageFormat) {
|
||||
convertOptions.imageFormat = cliOptions.imageFormat;
|
||||
}
|
||||
if (cliOptions.imageDir) {
|
||||
convertOptions.imageDir = cliOptions.imageDir;
|
||||
}
|
||||
if (cliOptions.pages) {
|
||||
convertOptions.pages = cliOptions.pages;
|
||||
}
|
||||
if (cliOptions.includeHeaderFooter) {
|
||||
convertOptions.includeHeaderFooter = true;
|
||||
}
|
||||
if (cliOptions.detectStrikethrough) {
|
||||
convertOptions.detectStrikethrough = true;
|
||||
}
|
||||
if (cliOptions.hybrid) {
|
||||
convertOptions.hybrid = cliOptions.hybrid;
|
||||
}
|
||||
if (cliOptions.hybridMode) {
|
||||
convertOptions.hybridMode = cliOptions.hybridMode;
|
||||
}
|
||||
if (cliOptions.hybridUrl) {
|
||||
convertOptions.hybridUrl = cliOptions.hybridUrl;
|
||||
}
|
||||
if (cliOptions.hybridTimeout) {
|
||||
convertOptions.hybridTimeout = cliOptions.hybridTimeout;
|
||||
}
|
||||
if (cliOptions.hybridFallback) {
|
||||
convertOptions.hybridFallback = true;
|
||||
}
|
||||
if (cliOptions.hybridHancomAiRegionlistStrategy) {
|
||||
convertOptions.hybridHancomAiRegionlistStrategy = cliOptions.hybridHancomAiRegionlistStrategy;
|
||||
}
|
||||
if (cliOptions.hybridHancomAiOcrStrategy) {
|
||||
convertOptions.hybridHancomAiOcrStrategy = cliOptions.hybridHancomAiOcrStrategy;
|
||||
}
|
||||
if (cliOptions.hybridHancomAiImageCache) {
|
||||
convertOptions.hybridHancomAiImageCache = cliOptions.hybridHancomAiImageCache;
|
||||
}
|
||||
if (cliOptions.toStdout) {
|
||||
convertOptions.toStdout = true;
|
||||
}
|
||||
if (cliOptions.threads) {
|
||||
convertOptions.threads = cliOptions.threads;
|
||||
}
|
||||
|
||||
return convertOptions;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build CLI arguments array from ConvertOptions.
|
||||
*/
|
||||
export function buildArgs(options: ConvertOptions): string[] {
|
||||
const args: string[] = [];
|
||||
|
||||
if (options.outputDir) {
|
||||
args.push('--output-dir', options.outputDir);
|
||||
}
|
||||
if (options.password) {
|
||||
args.push('--password', options.password);
|
||||
}
|
||||
if (options.format) {
|
||||
if (Array.isArray(options.format)) {
|
||||
if (options.format.length > 0) {
|
||||
args.push('--format', options.format.join(','));
|
||||
}
|
||||
} else {
|
||||
args.push('--format', options.format);
|
||||
}
|
||||
}
|
||||
if (options.quiet) {
|
||||
args.push('--quiet');
|
||||
}
|
||||
if (options.contentSafetyOff) {
|
||||
if (Array.isArray(options.contentSafetyOff)) {
|
||||
if (options.contentSafetyOff.length > 0) {
|
||||
args.push('--content-safety-off', options.contentSafetyOff.join(','));
|
||||
}
|
||||
} else {
|
||||
args.push('--content-safety-off', options.contentSafetyOff);
|
||||
}
|
||||
}
|
||||
if (options.sanitize) {
|
||||
args.push('--sanitize');
|
||||
}
|
||||
if (options.keepLineBreaks) {
|
||||
args.push('--keep-line-breaks');
|
||||
}
|
||||
if (options.replaceInvalidChars) {
|
||||
args.push('--replace-invalid-chars', options.replaceInvalidChars);
|
||||
}
|
||||
if (options.useStructTree) {
|
||||
args.push('--use-struct-tree');
|
||||
}
|
||||
if (options.tableMethod) {
|
||||
args.push('--table-method', options.tableMethod);
|
||||
}
|
||||
if (options.readingOrder) {
|
||||
args.push('--reading-order', options.readingOrder);
|
||||
}
|
||||
if (options.markdownPageSeparator) {
|
||||
args.push('--markdown-page-separator', options.markdownPageSeparator);
|
||||
}
|
||||
if (options.markdownWithHtml) {
|
||||
args.push('--markdown-with-html');
|
||||
}
|
||||
if (options.textPageSeparator) {
|
||||
args.push('--text-page-separator', options.textPageSeparator);
|
||||
}
|
||||
if (options.htmlPageSeparator) {
|
||||
args.push('--html-page-separator', options.htmlPageSeparator);
|
||||
}
|
||||
if (options.imageOutput) {
|
||||
args.push('--image-output', options.imageOutput);
|
||||
}
|
||||
if (options.imageFormat) {
|
||||
args.push('--image-format', options.imageFormat);
|
||||
}
|
||||
if (options.imageDir) {
|
||||
args.push('--image-dir', options.imageDir);
|
||||
}
|
||||
if (options.pages) {
|
||||
args.push('--pages', options.pages);
|
||||
}
|
||||
if (options.includeHeaderFooter) {
|
||||
args.push('--include-header-footer');
|
||||
}
|
||||
if (options.detectStrikethrough) {
|
||||
args.push('--detect-strikethrough');
|
||||
}
|
||||
if (options.hybrid) {
|
||||
args.push('--hybrid', options.hybrid);
|
||||
}
|
||||
if (options.hybridMode) {
|
||||
args.push('--hybrid-mode', options.hybridMode);
|
||||
}
|
||||
if (options.hybridUrl) {
|
||||
args.push('--hybrid-url', options.hybridUrl);
|
||||
}
|
||||
if (options.hybridTimeout) {
|
||||
args.push('--hybrid-timeout', options.hybridTimeout);
|
||||
}
|
||||
if (options.hybridFallback) {
|
||||
args.push('--hybrid-fallback');
|
||||
}
|
||||
if (options.hybridHancomAiRegionlistStrategy) {
|
||||
args.push('--hybrid-hancom-ai-regionlist-strategy', options.hybridHancomAiRegionlistStrategy);
|
||||
}
|
||||
if (options.hybridHancomAiOcrStrategy) {
|
||||
args.push('--hybrid-hancom-ai-ocr-strategy', options.hybridHancomAiOcrStrategy);
|
||||
}
|
||||
if (options.hybridHancomAiImageCache) {
|
||||
args.push('--hybrid-hancom-ai-image-cache', options.hybridHancomAiImageCache);
|
||||
}
|
||||
if (options.toStdout) {
|
||||
args.push('--to-stdout');
|
||||
}
|
||||
if (options.threads) {
|
||||
args.push('--threads', options.threads);
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
import { spawn } from 'child_process';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import { StringDecoder } from 'string_decoder';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
// Re-export types and utilities from auto-generated file
|
||||
export type { ConvertOptions } from './convert-options.generated.js';
|
||||
export { buildArgs } from './convert-options.generated.js';
|
||||
import type { ConvertOptions } from './convert-options.generated.js';
|
||||
import { buildArgs } from './convert-options.generated.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const JAR_NAME = 'opendataloader-pdf-cli.jar';
|
||||
|
||||
interface JarExecutionOptions {
|
||||
/**
|
||||
* When true, forwards Java's stdout and stderr chunks to the parent
|
||||
* process in real time as well as accumulating them. Used by the bundled
|
||||
* CLI so long-running conversions show progress as it happens.
|
||||
*/
|
||||
streamOutput?: boolean;
|
||||
}
|
||||
|
||||
function executeJar(args: string[], executionOptions: JarExecutionOptions = {}): Promise<string> {
|
||||
const { streamOutput = false } = executionOptions;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const jarPath = path.join(__dirname, '..', 'lib', JAR_NAME);
|
||||
|
||||
if (!fs.existsSync(jarPath)) {
|
||||
return reject(
|
||||
new Error(`JAR file not found at ${jarPath}. Please run the build script first.`),
|
||||
);
|
||||
}
|
||||
|
||||
const command = 'java';
|
||||
// Force headless AWT so macOS doesn't surface a Dock icon (and steal focus)
|
||||
// every time the JVM touches ImageIO/PDFBox rendering. Safe on all OSes —
|
||||
// the CLI never opens a UI window, only manipulates BufferedImages.
|
||||
const commandArgs = [
|
||||
'-Djava.awt.headless=true',
|
||||
'-Dapple.awt.UIElement=true',
|
||||
'-jar',
|
||||
jarPath,
|
||||
...args,
|
||||
];
|
||||
|
||||
const javaProcess = spawn(command, commandArgs);
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
// StringDecoder buffers incomplete multi-byte UTF-8 sequences across
|
||||
// chunk boundaries — Buffer.toString() alone would emit replacement
|
||||
// characters when, e.g., a 3-byte Korean codepoint splits across two
|
||||
// 'data' events. One decoder per stream so they don't share state.
|
||||
const stdoutDecoder = new StringDecoder('utf8');
|
||||
const stderrDecoder = new StringDecoder('utf8');
|
||||
|
||||
javaProcess.stdout.on('data', (data: Buffer) => {
|
||||
const chunk = stdoutDecoder.write(data);
|
||||
if (chunk.length === 0) return;
|
||||
if (streamOutput) {
|
||||
// Stream-only: don't also accumulate, or a multi-hour conversion
|
||||
// would buffer its entire (potentially gigabyte) stdout in memory
|
||||
// for no consumer.
|
||||
process.stdout.write(chunk);
|
||||
} else {
|
||||
stdout += chunk;
|
||||
}
|
||||
});
|
||||
|
||||
javaProcess.stderr.on('data', (data: Buffer) => {
|
||||
const chunk = stderrDecoder.write(data);
|
||||
if (chunk.length === 0) return;
|
||||
if (streamOutput) {
|
||||
process.stderr.write(chunk);
|
||||
}
|
||||
// stderr is always accumulated (progress logs are small and we need
|
||||
// them for error messages on non-zero exit).
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
javaProcess.on('close', (code) => {
|
||||
// Flush any trailing bytes the decoder is still holding (always emit
|
||||
// them — if we drop them on error paths, error messages with non-ASCII
|
||||
// characters lose their tail).
|
||||
const stdoutTail = stdoutDecoder.end();
|
||||
const stderrTail = stderrDecoder.end();
|
||||
if (stdoutTail.length > 0) {
|
||||
if (streamOutput) {
|
||||
process.stdout.write(stdoutTail);
|
||||
} else {
|
||||
stdout += stdoutTail;
|
||||
}
|
||||
}
|
||||
if (stderrTail.length > 0) {
|
||||
if (streamOutput) process.stderr.write(stderrTail);
|
||||
stderr += stderrTail;
|
||||
}
|
||||
|
||||
if (code === 0) {
|
||||
resolve(stdout);
|
||||
} else {
|
||||
const errorOutput = stderr || stdout;
|
||||
const error = new Error(
|
||||
`The opendataloader-pdf CLI exited with code ${code}.\n\n${errorOutput}`,
|
||||
);
|
||||
// Tag so the CLI can suppress re-printing this message — Java's
|
||||
// stderr was already streamed live to the parent in CLI mode, and
|
||||
// re-printing risks leaking anything sensitive Java logged
|
||||
// (e.g. a --password value echoed by an underlying library).
|
||||
(error as Error & { isJavaExit?: boolean }).isJavaExit = true;
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
|
||||
javaProcess.on('error', (err: Error) => {
|
||||
if (err.message.includes('ENOENT')) {
|
||||
reject(
|
||||
new Error(
|
||||
"'java' command not found. Please ensure Java is installed and in your system's PATH.",
|
||||
),
|
||||
);
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function buildJarArgs(
|
||||
inputPaths: string | string[],
|
||||
options: ConvertOptions,
|
||||
): string[] | Error {
|
||||
const inputList = Array.isArray(inputPaths) ? inputPaths : [inputPaths];
|
||||
if (inputList.length === 0) {
|
||||
return new Error('At least one input path must be provided.');
|
||||
}
|
||||
|
||||
for (const input of inputList) {
|
||||
if (!fs.existsSync(input)) {
|
||||
return new Error(`Input file or folder not found: ${input}`);
|
||||
}
|
||||
}
|
||||
|
||||
return [...inputList, ...buildArgs(options)];
|
||||
}
|
||||
|
||||
export function convert(
|
||||
inputPaths: string | string[],
|
||||
options: ConvertOptions = {},
|
||||
): Promise<string> {
|
||||
const argsOrError = buildJarArgs(inputPaths, options);
|
||||
if (argsOrError instanceof Error) {
|
||||
return Promise.reject(argsOrError);
|
||||
}
|
||||
// Library API: never streams to the parent process. Returns the full stdout
|
||||
// string so callers can do `const out = await convert(...)` without surprise
|
||||
// side-effects on process.stdout / process.stderr.
|
||||
return executeJar(argsOrError, { streamOutput: false });
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal entry point used by the bundled CLI. Streams Java's stdout and
|
||||
* stderr to the parent process in real time (so long-running conversions like
|
||||
* hybrid mode show progress as it happens) and resolves without a stdout
|
||||
* payload — preventing the caller from re-printing what was already streamed.
|
||||
*
|
||||
* Not part of the public API: do not import this from application code. Use
|
||||
* {@link convert} instead.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
export async function _runForCli(
|
||||
inputPaths: string | string[],
|
||||
options: ConvertOptions = {},
|
||||
): Promise<void> {
|
||||
const argsOrError = buildJarArgs(inputPaths, options);
|
||||
if (argsOrError instanceof Error) {
|
||||
throw argsOrError;
|
||||
}
|
||||
await executeJar(argsOrError, { streamOutput: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use `convert()` and `ConvertOptions` instead. This function will be removed in a future version.
|
||||
*/
|
||||
export interface RunOptions {
|
||||
outputFolder?: string;
|
||||
password?: string;
|
||||
replaceInvalidChars?: string;
|
||||
generateMarkdown?: boolean;
|
||||
generateHtml?: boolean;
|
||||
generateAnnotatedPdf?: boolean;
|
||||
keepLineBreaks?: boolean;
|
||||
contentSafetyOff?: string;
|
||||
htmlInMarkdown?: boolean;
|
||||
addImageToMarkdown?: boolean;
|
||||
noJson?: boolean;
|
||||
debug?: boolean;
|
||||
useStructTree?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated Use `convert()` instead. This function will be removed in a future version.
|
||||
*/
|
||||
export function run(inputPath: string, options: RunOptions = {}): Promise<string> {
|
||||
console.warn(
|
||||
'Warning: run() is deprecated and will be removed in a future version. Use convert() instead.',
|
||||
);
|
||||
|
||||
// Build format array based on legacy boolean options
|
||||
const formats: string[] = [];
|
||||
if (!options.noJson) {
|
||||
formats.push('json');
|
||||
}
|
||||
if (options.generateMarkdown) {
|
||||
if (options.addImageToMarkdown) {
|
||||
formats.push('markdown-with-images');
|
||||
} else if (options.htmlInMarkdown) {
|
||||
formats.push('markdown-with-html');
|
||||
} else {
|
||||
formats.push('markdown');
|
||||
}
|
||||
}
|
||||
if (options.generateHtml) {
|
||||
formats.push('html');
|
||||
}
|
||||
if (options.generateAnnotatedPdf) {
|
||||
formats.push('pdf');
|
||||
}
|
||||
|
||||
return convert(inputPath, {
|
||||
outputDir: options.outputFolder,
|
||||
password: options.password,
|
||||
replaceInvalidChars: options.replaceInvalidChars,
|
||||
keepLineBreaks: options.keepLineBreaks,
|
||||
contentSafetyOff: options.contentSafetyOff,
|
||||
useStructTree: options.useStructTree,
|
||||
format: formats.length > 0 ? formats : undefined,
|
||||
quiet: !options.debug,
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user