wehub/opendataloader-project--opendataloader-pdf
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:59:42 +08:00
commit 59f8f60dad
348 changed files with 139133 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/.run/OpenDataLoaderCli.run.xml
+14
View File
@@ -0,0 +1,14 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="OpenDataLoaderCli" type="Application"
factoryName="Application">
<option name="ALTERNATIVE_JRE_PATH" value="corretto-21" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
<option name="MAIN_CLASS_NAME" value="org.opendataloader.pdf.cli.CLIMain" />
<module name="opendataloader-pdf-cli" />
<option name="PROGRAM_PARAMETERS"
value="../samples/pdf/1901.03003.pdf --markdown --html --pdf" />
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
</component>
java/checkstyle.xml
+13
View File
@@ -0,0 +1,13 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC "-//Checkstyle//DTD Checkstyle Configuration 1.3//EN" "https://checkstyle.org/dtds/configuration_1_3.dtd">
<module name="Checker">
<module name="TreeWalker">
<module name="UnusedImports"/>
<module name="WhitespaceAfter"/>
<module name="WhitespaceAround"/>
<module name="NoWhitespaceAfter"/>
<module name="NoWhitespaceBefore"/>
<module name="NeedBraces"/>
<module name="EmptyBlock"/>
</module>
</module>
java/opendataloader-pdf-cli/pom.xml
+137
View File
@@ -0,0 +1,137 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2025 Hancom Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-parent</artifactId>
<version>0.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>opendataloader-pdf-cli</artifactId>
<packaging>jar</packaging>
<name>OpenDataLoader PDF CLI</name>
<description>OpenDataLoader PDF CLI</description>
<dependencies>
<dependency>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>about.html</exclude>
<exclude>module-info.class</exclude>
<exclude>META-INF/*.MF</exclude>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/versions/**</exclude>
<exclude>META-INF/LICENSE</exclude>
<exclude>META-INF/LICENSE.txt</exclude>
<exclude>META-INF/LICENSE.md</exclude>
<exclude>META-INF/NOTICE</exclude>
<exclude>META-INF/NOTICE.txt</exclude>
<exclude>META-INF/NOTICE.md</exclude>
</excludes>
</filter>
<filter>
<artifact>com.sun.xml.bind:jaxb-impl</artifact>
<excludes>
<exclude>com/sun/xml/bind/**</exclude>
</excludes>
</filter>
<filter>
<artifact>org.jacoco:*</artifact>
<excludes>
<exclude>META-INF/maven/**</exclude>
</excludes>
</filter>
</filters>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.opendataloader.pdf.cli.CLIMain</mainClass>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ComponentsXmlResourceTransformer"/>
<transformer
implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/DEPENDENCIES</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/LICENSE</resource>
<file>${project.basedir}/../../LICENSE</file>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
<resource>META-INF/NOTICE</resource>
<file>${project.basedir}/../../NOTICE</file>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
java/opendataloader-pdf-cli/src/main/java/org/opendataloader/pdf/cli/CLIMain.java
+262
View File
@@ -0,0 +1,262 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.cli;
import org.apache.commons.cli.*;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.api.OpenDataLoaderPDF;
import org.opendataloader.pdf.api.cli.CLIOptions;
import org.opendataloader.pdf.exceptions.EncryptedTaggedPdfNotSupportedException;
import org.opendataloader.pdf.exceptions.InvalidPdfFileException;
import org.verapdf.exceptions.InvalidPasswordException;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.io.File;
import java.util.Locale;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.Logger;
public class CLIMain {
private static final Logger LOGGER = Logger.getLogger(CLIMain.class.getCanonicalName());
private static final String HELP = "[options] <INPUT FILE OR FOLDER>...\n Options:";
private enum InputSource { CLI_ARGUMENT, DIRECTORY_CHILD }
/**
* Result of processing a path: whether all files succeeded, and how many
* PDF files were processed under it (counted recursively for directories,
* 1 or 0 for a single file). Used by {@link #processDirectory} to print a
* clear summary when a user-supplied folder contains no PDFs (PDFDLOSP-15).
*/
private static final class PathResult {
final boolean allSucceeded;
final int pdfCount;
PathResult(boolean allSucceeded, int pdfCount) {
this.allSucceeded = allSucceeded;
this.pdfCount = pdfCount;
}
}
public static void main(String[] args) {
int exitCode = run(args);
if (exitCode != 0) {
System.exit(exitCode);
}
}
/**
* Runs the CLI with the given arguments and returns the exit code.
*
* @param args command-line arguments
* @return 0 on success, non-zero on failure
*/
static int run(String[] args) {
Options options = CLIOptions.defineOptions();
HelpFormatter formatter = new HelpFormatter();
CommandLine commandLine;
try {
commandLine = new DefaultParser().parse(options, args);
} catch (ParseException e) {
System.out.println(e.getMessage());
formatter.printHelp(HELP, options);
return 2;
}
// Handle --export-options before requiring input files
if (commandLine.hasOption(CLIOptions.EXPORT_OPTIONS_LONG_OPTION)) {
CLIOptions.exportOptionsAsJson(System.out);
return 0;
}
if (commandLine.getArgs().length < 1) {
formatter.printHelp(HELP, options);
return 0;
}
String[] arguments = commandLine.getArgs();
Config config;
boolean quiet;
try {
config = CLIOptions.createConfigFromCommandLine(commandLine);
quiet = commandLine.hasOption(CLIOptions.QUIET_OPTION) || commandLine.hasOption("quiet");
} catch (IllegalArgumentException exception) {
System.out.println(exception.getMessage());
formatter.printHelp(HELP, options);
return 2;
}
configureLogging(quiet);
boolean hasFailure = false;
try {
for (String argument : arguments) {
if (!processPath(new File(argument), config, InputSource.CLI_ARGUMENT).allSucceeded) {
hasFailure = true;
}
}
} finally {
// Release resources (e.g., hybrid client thread pools)
OpenDataLoaderPDF.shutdown();
}
return hasFailure ? 1 : 0;
}
private static void configureLogging(boolean quiet) {
if (!quiet) {
return;
}
Logger rootLogger = Logger.getLogger("");
rootLogger.setLevel(Level.OFF);
for (Handler handler : rootLogger.getHandlers()) {
handler.setLevel(Level.OFF);
}
LOGGER.setLevel(Level.OFF);
}
/**
* Processes a file or directory, returning true if all files succeeded.
*
* <p>{@code source} distinguishes user-provided arguments
* ({@link InputSource#CLI_ARGUMENT}) from files discovered during directory
* traversal ({@link InputSource#DIRECTORY_CHILD}): a non-PDF given directly
* on the command line is reported as an error, while non-PDF files inside a
* directory are silently skipped (preserves batch-folder processing).
*/
private static PathResult processPath(File file, Config config, InputSource source) {
if (!file.exists()) {
LOGGER.log(Level.WARNING, "File or folder " + file.getAbsolutePath() + " not found.");
return new PathResult(false, 0);
}
if (file.isDirectory()) {
return processDirectory(file, config, source);
}
if (file.isFile()) {
boolean isPdf = isPdfFile(file);
if (source == InputSource.CLI_ARGUMENT && !isPdf) {
System.out.println("Error: '" + file.getName()
+ "' is not a PDF file. Input must be a PDF file or a folder containing PDF files.");
return new PathResult(false, 0);
}
return new PathResult(processFile(file, config, source), isPdf ? 1 : 0);
}
return new PathResult(true, 0);
}
/**
* Counts PDF files processed under the directory rooted at {@code file}
* (recursively), so the CLI can surface a clear summary when a
* user-supplied folder contains no PDFs. Without this feedback the program
* would exit silently with status 0 and the user could not distinguish
* "wrong folder", "empty folder", and "successful run" (PDFDLOSP-15).
*
* <p>The summary is only printed for folders given directly on the command
* line ({@link InputSource#CLI_ARGUMENT}) — nested subdirectories aggregate
* upward into the top-level count rather than each printing their own line.
*
* <p>The summary line is the final <em>result</em> of the run, not a log
* entry, and is therefore intentionally emitted on stdout even under
* {@code --quiet}. {@code --quiet} suppresses processing logs; users
* still need to see whether anything was actually processed. The path
* shown is {@link File#getPath()} (the literal argument the user typed,
* e.g. {@code .} or {@code basic_images}) rather than {@link File#getName()},
* which would be empty for {@code .} or trailing-slash inputs.
*/
private static PathResult processDirectory(File file, Config config, InputSource source) {
File[] children = file.listFiles();
if (children == null) {
LOGGER.log(Level.WARNING, "Unable to read folder " + file.getAbsolutePath());
return new PathResult(false, 0);
}
boolean allSucceeded = true;
int pdfCount = 0;
for (File child : children) {
PathResult childResult = processPath(child, config, InputSource.DIRECTORY_CHILD);
if (!childResult.allSucceeded) {
allSucceeded = false;
}
pdfCount += childResult.pdfCount;
}
if (source == InputSource.CLI_ARGUMENT) {
if (pdfCount == 0) {
System.out.println("No PDF files found in '" + file.getPath() + "'.");
} else {
System.out.println("Processed " + pdfCount + " PDF file"
+ (pdfCount == 1 ? "" : "s") + " in '" + file.getPath() + "'.");
}
}
return new PathResult(allSucceeded, pdfCount);
}
/**
* Processes a single PDF file.
*
* <p>{@code source} controls how an {@link InvalidPdfFileException} from
* the magic-number guard is surfaced: {@link InputSource#CLI_ARGUMENT}
* routes it to stdout as a user-facing error and fails the run;
* {@link InputSource#DIRECTORY_CHILD} logs a WARNING and treats the file
* as silently skipped so batch-folder runs can still exit 0.
*
* @param file the file to process
* @param config the processing configuration
* @param source whether the file came from a CLI argument or directory traversal
* @return true if processing succeeded (or the file was a silently
* skipped non-PDF inside a directory), false on error.
*/
private static boolean processFile(File file, Config config, InputSource source) {
if (!isPdfFile(file)) {
LOGGER.log(Level.FINE, "Skipping non-PDF file " + file.getAbsolutePath());
return true;
}
try {
OpenDataLoaderPDF.processFile(file.getAbsolutePath(), config);
return true;
} catch (InvalidPdfFileException invalid) {
if (source == InputSource.CLI_ARGUMENT) {
System.out.println("Error: " + invalid.getMessage());
return false;
}
LOGGER.log(Level.WARNING, invalid.getMessage() + " Skipping.");
return true;
} catch (InvalidPasswordException exception) {
String password = config.getPassword();
String message = (password == null || password.isEmpty())
? "Error: '" + file.getName() + "' is password-protected. Use --password option."
: "Error: Incorrect password for '" + file.getName() + "'.";
System.out.println(message);
return false;
} catch (EncryptedTaggedPdfNotSupportedException exception) {
System.out.println("Error: " + exception.getMessage());
return false;
} catch (Exception exception) {
LOGGER.log(Level.SEVERE, "Exception during processing file " + file.getAbsolutePath() + ": " +
exception.getMessage(), exception);
return false;
} finally {
StaticContainers.closeImagesUtils();
}
}
private static boolean isPdfFile(File file) {
if (!file.isFile()) {
return false;
}
String name = file.getName();
return name.toLowerCase(Locale.ROOT).endsWith(".pdf");
}
}
java/opendataloader-pdf-cli/src/test/java/org/opendataloader/pdf/cli/CLIMainTest.java
+672
View File
@@ -0,0 +1,672 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.cli;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.apache.pdfbox.pdmodel.encryption.AccessPermission;
import org.apache.pdfbox.pdmodel.encryption.StandardProtectionPolicy;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import static org.junit.jupiter.api.Assertions.*;
class CLIMainTest {
@TempDir
Path tempDir;
/**
* When processing a PDF file throws any exception, CLIMain.run() must return
* a non-zero exit code. This test uses a malformed PDF with hybrid mode
* targeting an unreachable server, which triggers an exception during processing.
*
* <p>Before this fix, processFile() caught all exceptions and logged them at
* SEVERE level but never propagated the failure to the exit code.
*
* <p>Regression test for https://github.com/opendataloader-project/opendataloader-pdf/issues/287
*/
@Test
void testProcessingFailureReturnsNonZeroExitCode() throws IOException {
// Create a minimal PDF file so processFile is actually invoked
// (the file must exist and end in .pdf to pass the isPdfFile check)
Path testPdf = tempDir.resolve("test.pdf");
Files.write(testPdf, "%PDF-1.4 minimal".getBytes());
// Use an unreachable hybrid URL — the processing will fail either at
// the hybrid availability check or during PDF parsing, both of which
// must result in a non-zero exit code.
int exitCode = CLIMain.run(new String[]{
"--hybrid", "docling-fast",
"--hybrid-url", "http://127.0.0.1:59999",
testPdf.toString()
});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when file processing fails");
}
/**
* When a directory contains a file that fails processing, run() must return
* non-zero, even though other files in the directory may succeed. The
* "failure" here must be a genuine processing error (unreachable hybrid
* backend), not a corrupt-content classification — corrupt PDFs inside a
* directory are silently skipped by design (see
* {@link #testCorruptPdfInsideDirectoryIsSilentlySkipped()}), so the test
* uses a parseable PDF and lets the hybrid availability check fail.
*/
@Test
void testDirectoryWithFailingFileReturnsNonZeroExitCode() throws IOException {
Path dir = tempDir.resolve("docs");
Files.createDirectory(dir);
Path testPdf = dir.resolve("bad.pdf");
try (PDDocument doc = new PDDocument()) {
doc.addPage(new PDPage());
doc.save(testPdf.toFile());
}
int exitCode = CLIMain.run(new String[]{
"--hybrid", "docling-fast",
"--hybrid-url", "http://127.0.0.1:59999",
dir.toString()
});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when any file in directory fails");
}
/**
* Normal invocation with no arguments should return 0 (just prints help).
*/
@Test
void testNoArgumentsReturnsZero() {
int exitCode = CLIMain.run(new String[]{});
assertEquals(0, exitCode);
}
/**
* Invalid CLI arguments (e.g., unrecognized option) must return exit code 2,
* following POSIX convention for command-line usage errors.
*/
@Test
void testInvalidArgumentsReturnsExitCode2() {
int exitCode = CLIMain.run(new String[]{"--no-such-option"});
assertEquals(2, exitCode);
}
/**
* Non-existent input file must return non-zero exit code.
*/
@Test
void testNonExistentFileReturnsNonZeroExitCode() {
int exitCode = CLIMain.run(new String[]{"/nonexistent/path/file.pdf"});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when input file does not exist");
}
/**
* A non-PDF file given directly on the command line must produce a clear
* error message on stdout and a non-zero exit code, instead of silently
* exiting with success.
*
* <p>Regression test for PDFDLOSP-5.
*/
@Test
void testNonPdfTopLevelArgumentEmitsErrorAndFails() throws IOException {
Path png = tempDir.resolve("abcd.png");
Files.write(png, new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
String stdout = captureStdoutOf(() -> CLIMain.run(new String[]{png.toString()}));
assertTrue(stdout.contains("'abcd.png' is not a PDF file"),
"stdout must mention the offending file name and that it is not a PDF; got: " + stdout);
assertTrue(stdout.contains("Input must be a PDF file or a folder containing PDF files"),
"stdout must guide the user to valid input; got: " + stdout);
}
@Test
void testNonPdfTopLevelArgumentReturnsNonZeroExitCode() throws IOException {
Path png = tempDir.resolve("abcd.png");
Files.write(png, new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
int exitCode = CLIMain.run(new String[]{png.toString()});
assertNotEquals(0, exitCode,
"Exit code must be non-zero when the top-level argument is not a PDF");
}
/**
* Non-PDF files inside a directory must continue to be silently skipped —
* batch-folder processing with mixed file types is a legitimate use case
* and must not regress.
*/
@Test
void testNonPdfFileInsideDirectoryIsSilentlySkipped() throws IOException {
Path dir = tempDir.resolve("mixed");
Files.createDirectory(dir);
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
Files.write(dir.resolve("readme.txt"), "hello".getBytes(StandardCharsets.UTF_8));
String stdout = captureStdoutOf(() -> CLIMain.run(new String[]{dir.toString()}));
assertFalse(stdout.contains("is not a PDF file"),
"Files discovered during directory traversal must not trigger the top-level error; got: " + stdout);
}
@Test
void testDirectoryWithOnlyNonPdfFilesReturnsZero() throws IOException {
Path dir = tempDir.resolve("non-pdf-only");
Files.createDirectory(dir);
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
int exitCode = CLIMain.run(new String[]{dir.toString()});
assertEquals(0, exitCode,
"Directories containing only non-PDF files must succeed (silent skip)");
}
/**
* A folder containing zero processable PDFs must emit a clear "No PDF files
* found" message instead of exiting silently with status 0 — without this
* the user cannot distinguish "wrong folder", "empty folder", and
* "successful run". The path shown is the literal argument the user typed
* (File#getPath), so {@code .} and trailing-slash inputs render correctly.
*
* <p>Regression test for PDFDLOSP-15.
*/
@Test
void testEmptyDirectoryEmitsNoPdfFoundMessage() throws IOException {
Path dir = tempDir.resolve("empty");
Files.createDirectory(dir);
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder);
assertEquals(0, exitCode, "Empty folder is not an error");
assertTrue(stdoutHolder[0].contains("No PDF files found in '" + dir + "'"),
"stdout must report that no PDFs were found; got: " + stdoutHolder[0]);
}
@Test
void testDirectoryWithOnlyNonPdfFilesEmitsNoPdfFoundMessage() throws IOException {
Path dir = tempDir.resolve("basic_images");
Files.createDirectory(dir);
Files.write(dir.resolve("a.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
Files.write(dir.resolve("b.jpg"), new byte[]{(byte) 0xFF, (byte) 0xD8});
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder);
assertEquals(0, exitCode);
assertTrue(stdoutHolder[0].contains("No PDF files found in '" + dir + "'"),
"stdout must report that no PDFs were found; got: " + stdoutHolder[0]);
}
/**
* Subdirectories must aggregate into the top-level summary rather than
* emit their own "No PDF files found" / "Processed N..." lines — only the
* user-supplied folder path appears in the output.
*/
@Test
void testNestedSubdirectoriesAggregateIntoTopLevelSummary() throws IOException {
Path top = tempDir.resolve("top");
Path sub = top.resolve("sub");
Files.createDirectories(sub);
Files.write(sub.resolve("nested.pdf"), "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
runCapturingStdout(
() -> CLIMain.run(new String[]{top.toString()}),
stdoutHolder);
assertTrue(stdoutHolder[0].contains("Processed 1 PDF file in '" + top + "'"),
"stdout must summarize at the top-level folder including nested PDFs; got: "
+ stdoutHolder[0]);
assertFalse(stdoutHolder[0].contains("in '" + sub + "'"),
"stdout must not emit a separate summary for nested subdirectories; got: "
+ stdoutHolder[0]);
}
@Test
void testDirectoryWithMultiplePdfsEmitsProcessedSummary() throws IOException {
Path dir = tempDir.resolve("docs");
Files.createDirectory(dir);
Files.write(dir.resolve("a.pdf"), "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
Files.write(dir.resolve("b.pdf"), "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder);
assertTrue(stdoutHolder[0].contains("Processed 2 PDF files in '" + dir + "'"),
"stdout must summarize with plural 'files' when count > 1; got: "
+ stdoutHolder[0]);
}
/**
* When the command line mixes a valid PDF argument with a top-level non-PDF
* argument, the run must fail overall but only the non-PDF entry should
* produce the user-facing error message.
*/
@Test
void testMixedTopLevelArgumentsReportOnlyNonPdfAndFailOverall() throws IOException {
Path png = tempDir.resolve("note.png");
Files.write(png, new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
Path pdf = tempDir.resolve("doc.pdf");
Files.write(pdf, "%PDF-1.4 minimal".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{pdf.toString(), png.toString()}),
stdoutHolder);
assertNotEquals(0, exitCode,
"Exit code must be non-zero when any top-level argument is not a PDF");
assertTrue(stdoutHolder[0].contains("'note.png' is not a PDF file"),
"stdout must call out the non-PDF argument by name; got: " + stdoutHolder[0]);
}
/**
* Password-protected PDF with no password supplied must produce a single
* user-friendly error message (no verapdf stack trace) and exit non-zero.
*
* <p>Regression test for PDFDLOSP-9.
*/
@Test
void testPasswordProtectedWithoutPasswordEmitsFriendlyError() throws IOException {
Path pdf = createPasswordProtectedPdf(tempDir.resolve("locked.pdf"), "1234");
String[] holder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{pdf.toString()}),
holder);
assertNotEquals(0, exitCode, "exit code must be non-zero");
assertTrue(holder[0].contains("'locked.pdf' is password-protected"),
"stdout must call out the file by name and explain it is password-protected; got: " + holder[0]);
assertTrue(holder[0].contains("--password"),
"stdout must guide the user to the --password option; got: " + holder[0]);
assertFalse(holder[0].contains("InvalidPasswordException"),
"stdout must not expose internal exception type; got: " + holder[0]);
assertFalse(holder[0].contains("at org.verapdf"),
"stdout must not expose a stack trace; got: " + holder[0]);
}
/**
* Password-protected PDF with an incorrect password must produce a single
* user-friendly error message (no verapdf stack trace) and exit non-zero.
*
* <p>Regression test for PDFDLOSP-9.
*/
@Test
void testPasswordProtectedWithWrongPasswordEmitsFriendlyError() throws IOException {
Path pdf = createPasswordProtectedPdf(tempDir.resolve("locked.pdf"), "1234");
String[] holder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{pdf.toString(), "--password", "wrongpw"}),
holder);
assertNotEquals(0, exitCode, "exit code must be non-zero");
assertTrue(holder[0].contains("Incorrect password for 'locked.pdf'"),
"stdout must report incorrect password and the file name; got: " + holder[0]);
assertFalse(holder[0].contains("InvalidPasswordException"),
"stdout must not expose internal exception type; got: " + holder[0]);
assertFalse(holder[0].contains("at org.verapdf"),
"stdout must not expose a stack trace; got: " + holder[0]);
}
/**
* Correct password must keep the existing success path: exit code 0 and
* the JSON output produced next to the input. Regression guard for the
* happy case while fixing PDFDLOSP-9.
*/
@Test
void testPasswordProtectedWithCorrectPasswordSucceeds() throws IOException {
Path pdf = createPasswordProtectedPdf(tempDir.resolve("locked.pdf"), "1234");
int exitCode = CLIMain.run(new String[]{pdf.toString(), "--password", "1234"});
assertEquals(0, exitCode, "correct password must keep exit code 0");
assertTrue(Files.exists(tempDir.resolve("locked.json")),
"JSON output must be produced next to the input PDF");
}
private static Path createPasswordProtectedPdf(Path target, String password) throws IOException {
try (PDDocument document = new PDDocument()) {
document.addPage(new PDPage());
AccessPermission permissions = new AccessPermission();
StandardProtectionPolicy policy = new StandardProtectionPolicy(password, password, permissions);
policy.setEncryptionKeyLength(128);
document.protect(policy);
document.save(target.toFile());
}
return target;
}
// --- PDFDLOSP-21: hybrid backend unavailable, stack-trace guard -------
/**
* When the hybrid backend is unreachable and {@code --hybrid-fallback} is
* NOT supplied, CLIMain must surface the friendly fail-fast message
* (including the new {@code --hybrid-fallback} hint) via the SEVERE log
* record and must never leak a Java stack trace to stdout. Regression for
* PDFDLOSP-21.
*/
@Test
void testHybridUnavailableWithoutFallbackEmitsFriendlyMessageNoStackTrace() throws IOException {
Path testPdf = tempDir.resolve("ok.pdf");
// Minimal but valid-enough header so the file reaches HybridDocumentProcessor.
// PDF parsing happens after health-check on this branch.
try (PDDocument doc = new PDDocument()) {
doc.addPage(new PDPage());
doc.save(testPdf.toFile());
}
// Reserve an ephemeral port and release it so connect() is refused.
int closedPort;
try (java.net.ServerSocket socket = new java.net.ServerSocket(0)) {
closedPort = socket.getLocalPort();
}
String[] stdoutHolder = new String[1];
StringBuilder logCapture = new StringBuilder();
java.util.logging.Logger cliLogger = java.util.logging.Logger.getLogger(
"org.opendataloader.pdf.cli.CLIMain");
java.util.logging.Handler captureHandler = new java.util.logging.Handler() {
@Override public void publish(java.util.logging.LogRecord record) {
logCapture.append(record.getMessage()).append('\n');
Throwable thrown = record.getThrown();
while (thrown != null) {
logCapture.append(thrown).append('\n');
for (StackTraceElement frame : thrown.getStackTrace()) {
logCapture.append("\tat ").append(frame).append('\n');
}
thrown = thrown.getCause();
}
}
@Override public void flush() {}
@Override public void close() {}
};
cliLogger.addHandler(captureHandler);
int exitCode;
try {
exitCode = (int) runCapturingStdout(() -> CLIMain.run(new String[]{
"--hybrid", "docling-fast",
"--hybrid-mode", "full",
"--hybrid-url", "http://127.0.0.1:" + closedPort,
"--output", tempDir.toString(),
testPdf.toString()
}), stdoutHolder);
} finally {
cliLogger.removeHandler(captureHandler);
}
assertNotEquals(0, exitCode,
"Exit code must be non-zero when hybrid backend is unreachable and fallback is off");
String out = stdoutHolder[0];
assertFalse(out.contains("\tat "),
"stdout must not contain a Java stack trace ('\\tat '); got: " + out);
String logged = logCapture.toString();
assertTrue(logged.contains("Hybrid server is not available"),
"Log output should explain that the hybrid backend is unreachable; got: " + logged);
assertTrue(logged.contains("--hybrid-fallback"),
"Log output should include the --hybrid-fallback recovery hint; got: " + logged);
}
// --- PDFDLOSP-14: magic-number guard regressions ----------------------
private static final byte[] JPEG_PREFIX = new byte[]{
(byte) 0xFF, (byte) 0xD8, (byte) 0xFF, (byte) 0xE0,
0x00, 0x10, 'J', 'F', 'I', 'F', 0x00
};
/**
* A .pdf-named file whose content is not actually PDF must produce the
* user-facing magic-number error on stdout, fail with non-zero exit, AND
* must not leak any veraPDF internal stack trace. Regression for
* PDFDLOSP-14.
*/
@Test
void testPdfExtensionWithNonPdfContentEmitsMagicNumberError() throws IOException {
Path fakePdf = tempDir.resolve("fake.pdf");
Files.write(fakePdf, JPEG_PREFIX);
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{fakePdf.toString()}),
stdoutHolder);
assertNotEquals(0, exitCode,
"Exit code must be non-zero when input content is not PDF");
assertTrue(stdoutHolder[0].contains("'fake.pdf' is not a valid PDF file (missing %PDF- header)"),
"stdout must contain the magic-number error; got: " + stdoutHolder[0]);
}
/**
* Stack-trace leakage guard: the veraPDF internals must NEVER appear on
* stdout for the non-PDF-content case. This is the core regression we
* are preventing — if the catch branch is reverted or the guard moves
* back behind PDDocument construction, this test fails.
*/
@Test
void testPdfExtensionWithNonPdfContentDoesNotLeakStackTrace() throws IOException {
Path fakePdf = tempDir.resolve("fake.pdf");
Files.write(fakePdf, JPEG_PREFIX);
String[] stdoutHolder = new String[1];
runCapturingStdout(
() -> CLIMain.run(new String[]{fakePdf.toString()}),
stdoutHolder);
String out = stdoutHolder[0];
assertFalse(out.toLowerCase(java.util.Locale.ROOT).contains("verapdf"),
"Output must not contain 'verapdf'; got: " + out);
assertFalse(out.contains("java.io.IOException:"),
"Output must not contain 'java.io.IOException:'; got: " + out);
assertFalse(out.contains("\tat "),
"Output must not contain a Java stack trace ('\\tat '); got: " + out);
}
/**
* A directory containing a .pdf-named non-PDF file alongside another
* non-PDF file must (a) NOT print the top-level magic-number error to
* stdout, (b) still exit 0 (silent skip preserves PR #496's batch-folder
* semantics), and (c) emit a single WARNING log line so operators can
* investigate why a file was skipped. Captures CLIMain's JUL logger
* directly because java.util.logging routes WARNING to a ConsoleHandler
* on stderr by default and may be suppressed entirely under --quiet.
*/
@Test
void testPdfExtensionNonPdfInsideDirectoryIsSilentlySkipped() throws IOException {
Path dir = tempDir.resolve("mixed");
Files.createDirectory(dir);
Files.write(dir.resolve("fake.pdf"), JPEG_PREFIX);
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
List<LogRecord> records = new ArrayList<>();
String[] stdoutHolder = new String[1];
int exitCode = (int) captureLogsOf(records, () -> runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder)).intValue();
assertEquals(0, exitCode,
"Directory containing only invalid files must still exit 0 (silent skip)");
assertFalse(stdoutHolder[0].contains("Error:"),
"Directory traversal must not surface 'Error:' on stdout; got: " + stdoutHolder[0]);
assertFalse(stdoutHolder[0].contains("missing %PDF- header"),
"Directory traversal must not surface the magic-number error on stdout; got: "
+ stdoutHolder[0]);
long warningMatches = records.stream().filter(r ->
r.getLevel() == Level.WARNING
&& r.getMessage() != null
&& r.getMessage().contains("'fake.pdf'")
&& r.getMessage().contains("missing %PDF- header")).count();
assertEquals(1L, warningMatches,
"Exactly one WARNING log must name the offending file and the missing header; "
+ "got: " + records);
}
/**
* A PDF with a valid {@code %PDF-} header but a corrupt or truncated body
* must surface the friendly "corrupted or truncated content" error on
* stdout, exit non-zero, AND must not leak a veraPDF stack trace. The
* wording must also be distinct from the missing-header case so users
* can tell the two failure modes apart.
*/
@Test
void testCorruptPdfShowsCorruptedFileErrorWithoutStackTrace() throws IOException {
Path corrupt = tempDir.resolve("corrupt.pdf");
// Valid header, but nothing else — veraPDF will fail at parse time.
Files.write(corrupt, "%PDF-1.4\n<garbage>".getBytes(StandardCharsets.UTF_8));
String[] stdoutHolder = new String[1];
int exitCode = (int) runCapturingStdout(
() -> CLIMain.run(new String[]{corrupt.toString()}),
stdoutHolder);
String out = stdoutHolder[0];
assertNotEquals(0, exitCode,
"Corrupt PDF must still fail (non-zero exit); got stdout: " + out);
assertTrue(out.contains("'corrupt.pdf' is not a valid PDF file (corrupted or truncated content)"),
"stdout must surface the friendly corrupted-content error; got: " + out);
assertFalse(out.contains("missing %PDF- header"),
"Corrupt-but-headered PDFs must NOT be misreported as missing the header; got: "
+ out);
assertFalse(out.toLowerCase(java.util.Locale.ROOT).contains("verapdf"),
"Output must not contain 'verapdf'; got: " + out);
assertFalse(out.contains("java.io.IOException:"),
"Output must not contain 'java.io.IOException:'; got: " + out);
assertFalse(out.contains("\tat "),
"Output must not contain a Java stack trace ('\\tat '); got: " + out);
}
/**
* A corrupt-but-headered PDF inside a directory must follow the same
* batch-folder semantics as a non-PDF-content file: WARNING log, no
* {@code Error:} on stdout, exit 0. This mirrors
* {@link #testPdfExtensionNonPdfInsideDirectoryIsSilentlySkipped()} for
* the corrupted-body branch so both failure modes route through the
* same {@link InvalidPdfFileException} handler in CLIMain.
*/
@Test
void testCorruptPdfInsideDirectoryIsSilentlySkipped() throws IOException {
Path dir = tempDir.resolve("mixed-corrupt");
Files.createDirectory(dir);
Files.write(dir.resolve("corrupt.pdf"),
"%PDF-1.4\n<garbage>".getBytes(StandardCharsets.UTF_8));
Files.write(dir.resolve("note.png"), new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47});
List<LogRecord> records = new ArrayList<>();
String[] stdoutHolder = new String[1];
int exitCode = (int) captureLogsOf(records, () -> runCapturingStdout(
() -> CLIMain.run(new String[]{dir.toString()}),
stdoutHolder)).intValue();
assertEquals(0, exitCode,
"Directory traversal must exit 0 even when a child is a corrupt PDF "
+ "(silent-skip semantics); got stdout: " + stdoutHolder[0]);
assertFalse(stdoutHolder[0].contains("Error:"),
"Directory traversal must not surface 'Error:' on stdout; got: " + stdoutHolder[0]);
long warningMatches = records.stream().filter(r ->
r.getLevel() == Level.WARNING
&& r.getMessage() != null
&& r.getMessage().contains("'corrupt.pdf'")
&& r.getMessage().contains("corrupted or truncated")).count();
assertEquals(1L, warningMatches,
"Exactly one WARNING log must name the offending file and describe "
+ "the corrupted-content failure; got: " + records);
}
/**
* Runs {@code action} with a temporary {@link Handler} attached to
* {@link CLIMain}'s JUL logger, collecting every {@link LogRecord} into
* {@code sink}. Restores the prior handler list and level on exit.
*
* <p>JUL is global; this helper assumes sequential test execution
* (JUnit 5 + Surefire default). Returns whatever {@code action} returned.
*/
private static <T> T captureLogsOf(List<LogRecord> sink, java.util.concurrent.Callable<T> action) {
Logger logger = Logger.getLogger(CLIMain.class.getCanonicalName());
Level priorLevel = logger.getLevel();
boolean priorUseParent = logger.getUseParentHandlers();
Handler capture = new Handler() {
@Override public void publish(LogRecord record) { sink.add(record); }
@Override public void flush() { }
@Override public void close() { }
};
capture.setLevel(Level.ALL);
logger.addHandler(capture);
logger.setLevel(Level.ALL);
logger.setUseParentHandlers(false);
try {
return action.call();
} catch (Exception exception) {
throw new RuntimeException(exception);
} finally {
logger.removeHandler(capture);
logger.setLevel(priorLevel);
logger.setUseParentHandlers(priorUseParent);
}
}
private static String captureStdoutOf(Runnable action) {
String[] holder = new String[1];
runCapturingStdout(() -> {
action.run();
return 0;
}, holder);
return holder[0];
}
// System.setOut is JVM-global; this helper assumes sequential test
// execution (JUnit 5 + maven-surefire default). Revisit if parallel
// test execution is enabled — concurrent captures would interleave.
private static long runCapturingStdout(java.util.concurrent.Callable<Integer> action, String[] stdoutHolder) {
PrintStream originalOut = System.out;
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
System.setOut(new PrintStream(buffer, true, StandardCharsets.UTF_8));
try {
int exitCode = action.call();
stdoutHolder[0] = buffer.toString(StandardCharsets.UTF_8);
return exitCode;
} catch (Exception exception) {
throw new RuntimeException(exception);
} finally {
System.setOut(originalOut);
}
}
}
java/opendataloader-pdf-cli/src/test/java/org/opendataloader/pdf/cli/FormatLogRegressionTest.java
+124
View File
@@ -0,0 +1,124 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.cli;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDPage;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.logging.Handler;
import java.util.logging.Level;
import java.util.logging.LogRecord;
import java.util.logging.Logger;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
* Regression test for PDFDLOSP-26: every {@code --format} value must emit
* exactly one INFO-level "Created &lt;path&gt;" log line after writing its
* output. Tagged-PDF previously skipped this log because
* {@code AutoTaggingProcessor.createTaggedPDF()} had no {@code Logger}.
*/
class FormatLogRegressionTest {
@TempDir
Path tempDir;
@ParameterizedTest(name = "--format {0} emits exactly one Created log")
@ValueSource(strings = {"json", "text", "html", "pdf", "markdown", "tagged-pdf"})
void formatEmitsExactlyOneCreatedLog(String format) throws IOException {
Path pdf = createMinimalPdf(tempDir.resolve("in.pdf"));
Path outDir = tempDir.resolve(format);
Files.createDirectories(outDir);
List<LogRecord> records = new ArrayList<>();
int exitCode = captureAllLogsOf(records, () -> CLIMain.run(new String[]{
"--format", format,
"--output", outDir.toString(),
pdf.toString()
}));
assertEquals(0, exitCode, () ->
"Exit code must be 0 for --format " + format + "; records: " + summarize(records));
long createdCount = records.stream()
.filter(r -> r.getLevel() == Level.INFO)
.filter(r -> r.getMessage() != null && r.getMessage().contains("Created"))
.count();
assertEquals(1L, createdCount, () ->
"Exactly one INFO 'Created ...' log expected for --format " + format
+ "; got " + createdCount + ". Records: " + summarize(records));
}
private static Path createMinimalPdf(Path target) throws IOException {
try (PDDocument document = new PDDocument()) {
document.addPage(new PDPage());
document.save(target.toFile());
}
return target;
}
/**
* Captures every {@link LogRecord} emitted by any logger under the
* {@code org.opendataloader.pdf} hierarchy. Attaching to the root logger
* would also include third-party noise; attaching to the package root
* picks up CLIMain plus every generator/processor in core (JUL records
* propagate to parent handlers).
*
* <p>JUL is global; this assumes sequential test execution
* (JUnit 5 + Surefire default).
*/
private static <T> T captureAllLogsOf(List<LogRecord> sink, Callable<T> action) {
Logger logger = Logger.getLogger("org.opendataloader.pdf");
Level priorLevel = logger.getLevel();
boolean priorUseParent = logger.getUseParentHandlers();
Handler capture = new Handler() {
@Override public void publish(LogRecord record) { sink.add(record); }
@Override public void flush() { }
@Override public void close() { }
};
capture.setLevel(Level.ALL);
logger.addHandler(capture);
logger.setLevel(Level.ALL);
logger.setUseParentHandlers(false);
try {
return action.call();
} catch (Exception exception) {
throw new RuntimeException(exception);
} finally {
logger.removeHandler(capture);
logger.setLevel(priorLevel);
logger.setUseParentHandlers(priorUseParent);
}
}
private static String summarize(List<LogRecord> records) {
StringBuilder sb = new StringBuilder("[");
for (LogRecord r : records) {
sb.append('(').append(r.getLevel()).append(": ").append(r.getMessage()).append(") ");
}
return sb.append(']').toString();
}
}
java/opendataloader-pdf-core/pom.xml
+245
View File
@@ -0,0 +1,245 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Copyright 2025-2026 Hancom Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.opendataloader</groupId>
<artifactId>opendataloader-pdf-parent</artifactId>
<version>0.0.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>opendataloader-pdf-core</artifactId>
<packaging>jar</packaging>
<name>OpenDataLoader PDF Core</name>
<description>OpenDataLoader PDF Core</description>
<dependencies>
<dependency>
<groupId>org.verapdf</groupId>
<artifactId>validation-model</artifactId>
<version>${verapdf.version}</version>
<exclusions>
<exclusion>
<groupId>org.verapdf</groupId>
<artifactId>feature-reporting</artifactId>
</exclusion>
<exclusion>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.verapdf</groupId>
<artifactId>wcag-validation</artifactId>
<version>${verapdf.version}</version>
<exclusions>
<exclusion>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.verapdf</groupId>
<artifactId>wcag-algorithms</artifactId>
<version>${verapdf.wcag.algs.version}</version>
<exclusions>
<exclusion>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
</dependency>
<!--
okhttp 5's plain `okhttp` artifact is an empty Kotlin-Multiplatform
metadata jar; the JVM classes live in `okhttp-jvm`, and Maven cannot
resolve the JVM variant from Gradle module metadata — so depend on
`okhttp-jvm` explicitly. Version is managed by the okhttp-bom in the
parent pom (see java/pom.xml dependencyManagement).
-->
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp-jvm</artifactId>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver3</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>
<directory>${project.basedir}/../../</directory>
<includes>
<include>LICENSE</include>
<include>NOTICE</include>
</includes>
<targetPath>META-INF</targetPath>
<filtering>false</filtering>
</resource>
<resource>
<directory>${project.basedir}/../../THIRD_PARTY</directory>
<includes>
<include>**/*</include>
</includes>
<targetPath>META-INF/THIRD_PARTY</targetPath>
<filtering>false</filtering>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>flatten-maven-plugin</artifactId>
<configuration>
<updatePomFile>true</updatePomFile>
<flattenMode>oss</flattenMode>
<pomElements>
<name/>
<description/>
<url/>
<scm/>
<developers/>
</pomElements>
</configuration>
<executions>
<execution>
<id>flatten</id>
<phase>process-resources</phase>
<goals>
<goal>flatten</goal>
</goals>
</execution>
<execution>
<id>flatten.clean</id>
<phase>clean</phase>
<goals>
<goal>clean</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>prepare-agent</goal>
</goals>
</execution>
<execution>
<id>report</id>
<phase>test</phase>
<goals>
<goal>report</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>release</id>
<build>
<plugins>
<plugin>
<groupId>org.sonatype.central</groupId>
<artifactId>central-publishing-maven-plugin</artifactId>
<extensions>true</extensions>
<configuration>
<publishingServerId>central</publishingServerId>
<autoPublish>true</autoPublish>
<waitUntil>published</waitUntil>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project>
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/AutoTagger.java
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.processors.AutoTaggingProcessor;
import org.opendataloader.pdf.processors.DocumentProcessor;
import org.opendataloader.pdf.processors.ExtractionResult;
import org.verapdf.pd.PDDocument;
import org.verapdf.tools.StaticResources;
import java.io.IOException;
/**
* Public API for standalone PDF auto-tagging.
*
* <p>Returns an in-memory tagged {@link PDDocument} without writing intermediate files.
* Use {@link OpenDataLoaderPDF#processFile} for the full extraction + output pipeline.
*
* <p>Example usage:
* <pre>{@code
* Config config = new Config();
* config.setHybrid("docling-fast");
*
* try (TaggingResult result = AutoTagger.tag("input.pdf", config)) {
* PDDocument tagged = result.getDocument();
* // use tagged document...
* result.saveTo("output_tagged.pdf");
* }
* AutoTagger.shutdown();
* }</pre>
*/
public final class AutoTagger {
private AutoTagger() {
}
/**
* Extract content from a PDF and produce a tagged PDF document in-memory.
* Output format flags in config are ignored — this method only produces
* a tagged PDDocument.
*
* @param inputPdf path to the input PDF file
* @param config configuration (extraction + hybrid fields are used;
* output format flags are ignored)
* @return result containing the tagged PDDocument and timing metadata
* @throws IOException if unable to read or process the PDF
*/
public static TaggingResult tag(String inputPdf, Config config, Float pdfVersion) throws IOException {
ExtractionResult extraction = DocumentProcessor.extractContents(inputPdf, config);
return tag(extraction, pdfVersion);
}
/**
* Tag a PDF document from a previously computed {@link ExtractionResult}.
* Use this when you need both tagged PDF and other output formats from the
* same extraction — call {@link DocumentProcessor#extractContents} once,
* then pass the result here and to other output generators.
*
* @param extraction pre-computed extraction result
* @return result containing the tagged PDDocument and timing metadata
* @throws IOException if unable to tag the document
*/
public static TaggingResult tag(ExtractionResult extraction, Float pdfVersion) throws IOException {
long t0 = System.nanoTime();
PDDocument document = StaticResources.getDocument();
AutoTaggingProcessor.tagDocument(document, extraction.getContents(), pdfVersion);
long taggingNs = System.nanoTime() - t0;
return new TaggingResult(document, extraction.getExtractionNs(), taggingNs,
extraction.getHybridTimings(), extraction.getElementMetadata());
}
/**
* Release hybrid client resources. Call when processing is complete.
*/
public static void shutdown() {
OpenDataLoaderPDF.shutdown();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/Config.java
+922
View File
@@ -0,0 +1,922 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.opendataloader.pdf.hybrid.HybridConfig;
/**
* Configuration class for the PDF processing.
* Use this class to specify output formats, text processing options, and other settings.
*/
public class Config {
private static final Logger LOGGER = Logger.getLogger(Config.class.getCanonicalName());
/** Reading order option: no sorting, keeps PDF COS object order. */
public static final String READING_ORDER_OFF = "off";
/** Reading order option: XY-Cut++ algorithm for layout-aware sorting. */
public static final String READING_ORDER_XYCUT = "xycut";
private static Set<String> readingOrderOptions = new HashSet<>();
/** Hybrid mode: off (Java-only processing, no external dependency). */
public static final String HYBRID_OFF = "off";
/** Hybrid mode: docling backend (Docling FastAPI server). */
public static final String HYBRID_DOCLING = "docling";
/** Hybrid mode: docling-fast backend (deprecated alias for docling). */
public static final String HYBRID_DOCLING_FAST = "docling-fast";
/** Hybrid mode: hancom backend (Hancom Document AI). */
public static final String HYBRID_HANCOM = "hancom";
/** Hybrid mode: hancom-ai backend (Hancom AI HOCR SDK — individual modules). */
public static final String HYBRID_HANCOM_AI = "hancom-ai";
/** Hybrid mode: azure backend (Azure Document Intelligence). */
public static final String HYBRID_AZURE = "azure";
/** Hybrid mode: google backend (Google Document AI). */
public static final String HYBRID_GOOGLE = "google";
private static Set<String> hybridOptions = new HashSet<>();
/** Hybrid triage mode: auto (dynamic triage based on page content). */
public static final String HYBRID_MODE_AUTO = "auto";
/** Hybrid triage mode: full (skip triage, send all pages to backend). */
public static final String HYBRID_MODE_FULL = "full";
private static Set<String> hybridModeOptions = new HashSet<>();
/** Placeholder string for page number in separators. */
public static final String PAGE_NUMBER_STRING = "%page-number%";
private String password;
private boolean isGenerateMarkdown = false;
private boolean isGenerateHtml = false;
private boolean isGeneratePDF = false;
private boolean keepLineBreaks = false;
private boolean isGenerateJSON = true;
private boolean isGenerateText = false;
private boolean isGenerateTaggedPDF = false;
private boolean useStructTree = false;
private boolean useHTMLInMarkdown = false;
private boolean addImageToMarkdown = false;
private String replaceInvalidChars = " ";
private String outputFolder;
private String tableMethod = TABLE_METHOD_DEFAULT;
private String readingOrder = READING_ORDER_XYCUT;
private String markdownPageSeparator = "";
private String textPageSeparator = "";
private String htmlPageSeparator = "";
private String imageOutput = IMAGE_OUTPUT_EXTERNAL;
private String imageFormat = IMAGE_FORMAT_PNG;
private String imageDir;
private String pages;
private List<Integer> cachedPageNumbers;
private final FilterConfig filterConfig = new FilterConfig();
private String hybrid = HYBRID_OFF;
private final HybridConfig hybridConfig = new HybridConfig();
private boolean includeHeaderFooter = false;
private boolean detectStrikethrough = false;
/** Table detection method: default (border-based detection). */
public static final String TABLE_METHOD_DEFAULT = "default";
/** Table detection method: cluster-based detection (includes border-based). */
public static final String TABLE_METHOD_CLUSTER = "cluster";
private static Set<String> tableMethodOptions = new HashSet<>();
/** Image format: PNG. */
public static final String IMAGE_FORMAT_PNG = "png";
/** Image format: JPEG. */
public static final String IMAGE_FORMAT_JPEG = "jpeg";
private static Set<String> imageFormatOptions = new HashSet<>();
/** Image output mode: no image extraction. */
public static final String IMAGE_OUTPUT_OFF = "off";
/** Image output mode: embedded as Base64 data URIs. */
public static final String IMAGE_OUTPUT_EMBEDDED = "embedded";
/** Image output mode: external file references. */
public static final String IMAGE_OUTPUT_EXTERNAL = "external";
private static Set<String> imageOutputOptions = new HashSet<>();
static {
readingOrderOptions.add(READING_ORDER_OFF);
readingOrderOptions.add(READING_ORDER_XYCUT);
tableMethodOptions.add(TABLE_METHOD_DEFAULT);
tableMethodOptions.add(TABLE_METHOD_CLUSTER);
imageFormatOptions.add(IMAGE_FORMAT_PNG);
imageFormatOptions.add(IMAGE_FORMAT_JPEG);
imageOutputOptions.add(IMAGE_OUTPUT_OFF);
imageOutputOptions.add(IMAGE_OUTPUT_EMBEDDED);
imageOutputOptions.add(IMAGE_OUTPUT_EXTERNAL);
hybridOptions.add(HYBRID_OFF);
hybridOptions.add(HYBRID_DOCLING);
hybridOptions.add(HYBRID_DOCLING_FAST); // deprecated alias
hybridOptions.add(HYBRID_HANCOM);
hybridOptions.add(HYBRID_HANCOM_AI);
// azure, google added when implemented
hybridModeOptions.add(HYBRID_MODE_AUTO);
hybridModeOptions.add(HYBRID_MODE_FULL);
}
/**
* Gets the filter config.
*
* @return The FilterConfig.
*/
public FilterConfig getFilterConfig() {
return filterConfig;
}
/**
* Default constructor initializing the configuration with default values.
*/
public Config() {
}
/**
* Gets the password for opening encrypted PDF files.
*
* @return The password, or null if not set.
*/
public String getPassword() {
return password;
}
/**
* Sets the password for opening encrypted PDF files.
*
* @param password The password to use.
*/
public void setPassword(String password) {
this.password = password;
}
/**
* Checks if Markdown output generation is enabled.
* <p>
* Markdown generation is automatically enabled if {@link #isAddImageToMarkdown()} or
* {@link #isUseHTMLInMarkdown()} is true.
*
* @return true if Markdown output should be generated, false otherwise.
*/
public boolean isGenerateMarkdown() {
return isGenerateMarkdown || isAddImageToMarkdown() || isUseHTMLInMarkdown();
}
/**
* Enables or disables Markdown output generation.
*
* @param generateMarkdown true to enable, false to disable.
*/
public void setGenerateMarkdown(boolean generateMarkdown) {
isGenerateMarkdown = generateMarkdown;
}
/**
* Checks if HTML output generation is enabled.
*
* @return true if HTML output should be generated, false otherwise.
*/
public boolean isGenerateHtml() {
return isGenerateHtml;
}
/**
* Enables or disables HTML output generation.
*
* @param generateHtml true to enable, false to disable.
*/
public void setGenerateHtml(boolean generateHtml) {
isGenerateHtml = generateHtml;
}
/**
* Checks if a new PDF with tagged structure is generated.
*
* @return true if PDF generation is enabled, false otherwise.
*/
public boolean isGeneratePDF() {
return isGeneratePDF;
}
/**
* Enables or disables generation of a new, tagged PDF.
*
* @param generatePDF true to enable, false to disable.
*/
public void setGeneratePDF(boolean generatePDF) {
isGeneratePDF = generatePDF;
}
/**
* Checks if original line breaks within text blocks should be preserved.
*
* @return true if line breaks are preserved, false otherwise.
*/
public boolean isKeepLineBreaks() {
return keepLineBreaks;
}
/**
* Sets whether to preserve original line breaks within text blocks.
*
* @param keepLineBreaks true to preserve line breaks, false to merge lines into paragraphs.
*/
public void setKeepLineBreaks(boolean keepLineBreaks) {
this.keepLineBreaks = keepLineBreaks;
}
/**
* Checks if JSON output generation is enabled. Defaults to true.
*
* @return true if JSON output should be generated, false otherwise.
*/
public boolean isGenerateJSON() {
return isGenerateJSON;
}
/**
* Enables or disables JSON output generation.
*
* @param generateJSON true to enable, false to disable.
*/
public void setGenerateJSON(boolean generateJSON) {
isGenerateJSON = generateJSON;
}
/**
* Checks if plain text output generation is enabled.
*
* @return true if plain text output should be generated, false otherwise.
*/
public boolean isGenerateText() {
return isGenerateText;
}
/**
* Enables or disables plain text output generation.
*
* @param generateText true to enable, false to disable.
*/
public void setGenerateText(boolean generateText) {
isGenerateText = generateText;
}
public boolean isGenerateTaggedPDF() {
return isGenerateTaggedPDF;
}
public void setGenerateTaggedPDF(boolean generateTaggedPDF) {
isGenerateTaggedPDF = generateTaggedPDF;
}
/**
* Checks if HTML tags should be used within the Markdown output for complex structures like tables.
*
* @return true if HTML is used in Markdown, false otherwise.
*/
public boolean isUseHTMLInMarkdown() {
return useHTMLInMarkdown;
}
/**
* Enables or disables the use of HTML tags in Markdown output.
* Enabling this will also enable {@link #isGenerateMarkdown()}.
*
* @param useHTMLInMarkdown true to use HTML, false for pure Markdown.
*/
public void setUseHTMLInMarkdown(boolean useHTMLInMarkdown) {
this.useHTMLInMarkdown = useHTMLInMarkdown;
}
/**
* Checks if images should be extracted and included in the Markdown output.
*
* @return true if images are included in Markdown, false otherwise.
*/
public boolean isAddImageToMarkdown() {
return addImageToMarkdown;
}
/**
* Enables or disables the inclusion of extracted images in Markdown output.
* Enabling this will also enable {@link #isGenerateMarkdown()}.
*
* @param addImageToMarkdown true to include images, false otherwise.
*/
public void setAddImageToMarkdown(boolean addImageToMarkdown) {
this.addImageToMarkdown = addImageToMarkdown;
}
/**
* Gets the path to the output folder where generated files will be saved.
*
* @return The output folder path.
*/
public String getOutputFolder() {
return outputFolder;
}
/**
* Sets the path to the output folder where generated files will be saved.
* The directory will be created if it does not exist.
*
* @param outputFolder The path to the output folder.
*/
public void setOutputFolder(String outputFolder) {
this.outputFolder = outputFolder;
}
/**
* Gets the character, that replaces invalid or unrecognized characters (e.g., �, \u0000).
*
* @return The specified replacement character.
*/
public String getReplaceInvalidChars() {
return replaceInvalidChars;
}
/**
* Sets the character, that replaces invalid or unrecognized characters (e.g., �, \u0000).
*
* @param replaceInvalidChars The specified replacement character.
*/
public void setReplaceInvalidChars(String replaceInvalidChars) {
this.replaceInvalidChars = replaceInvalidChars;
}
/**
* Checks if the PDF structure tree should be used for document parsing.
*
* @return true if structure tree should be used, false otherwise.
*/
public boolean isUseStructTree() {
return useStructTree;
}
/**
* Enables or disables use of PDF structure tree for document parsing.
*
* @param useStructTree true to use structure tree, false otherwise.
*/
public void setUseStructTree(boolean useStructTree) {
this.useStructTree = useStructTree;
}
/**
* Checks if cluster-based table detection is enabled.
*
* @return true if cluster table detection is enabled, false otherwise.
*/
public boolean isClusterTableMethod() {
return TABLE_METHOD_CLUSTER.equals(tableMethod);
}
/**
* Gets the table detection method.
*
* @return The table detection method (default or cluster).
*/
public String getTableMethod() {
return tableMethod;
}
/**
* Sets the table detection method.
*
* @param tableMethod The table detection method (default or cluster).
* @throws IllegalArgumentException if the method is not supported.
*/
public void setTableMethod(String tableMethod) {
if (tableMethod != null && !isValidTableMethod(tableMethod)) {
throw new IllegalArgumentException(
String.format("Unsupported table method '%s'. Supported values: %s",
tableMethod, getTableMethodOptions(", ")));
}
this.tableMethod = tableMethod != null ? tableMethod.toLowerCase(Locale.ROOT) : TABLE_METHOD_DEFAULT;
}
/**
* Gets the list of methods of table detection.
*
* @param delimiter the delimiter to use between options
* @return the string with methods separated by the delimiter
*/
public static String getTableMethodOptions(CharSequence delimiter) {
return String.join(delimiter, tableMethodOptions);
}
/**
* Checks if the given table method is valid.
*
* @param method The table method to check.
* @return true if the method is valid, false otherwise.
*/
public static boolean isValidTableMethod(String method) {
return method != null && tableMethodOptions.contains(method.toLowerCase(Locale.ROOT));
}
/**
* Gets the reading order, that states in which order content should be processed.
*
* @return The specified order.
*/
public String getReadingOrder() {
return readingOrder;
}
/**
* Sets the reading order, that states in which order content should be processed.
*
* @param readingOrder The specified order (off or xycut).
* @throws IllegalArgumentException if the order is not supported.
*/
public void setReadingOrder(String readingOrder) {
if (readingOrder != null && !isValidReadingOrder(readingOrder)) {
throw new IllegalArgumentException(
String.format("Unsupported reading order '%s'. Supported values: %s",
readingOrder, getReadingOrderOptions(", ")));
}
this.readingOrder = readingOrder != null ? readingOrder.toLowerCase(Locale.ROOT) : READING_ORDER_XYCUT;
}
/**
* Gets the list of reading order options.
*
* @param delimiter The delimiter to use between options.
* @return The string with reading orders separated by the delimiter.
*/
public static String getReadingOrderOptions(CharSequence delimiter) {
return String.join(delimiter, readingOrderOptions);
}
/**
* Checks if the given reading order is valid.
*
* @param order The reading order to check.
* @return true if the order is valid, false otherwise.
*/
public static boolean isValidReadingOrder(String order) {
return order != null && readingOrderOptions.contains(order.toLowerCase(Locale.ROOT));
}
/**
* Gets the string, that separates content from different pages in markdown.
*
* @return The specified string.
*/
public String getMarkdownPageSeparator() {
return markdownPageSeparator;
}
/**
* Sets the string, that separates content from different pages in markdown.
*
* @param markdownPageSeparator The specified string.
*/
public void setMarkdownPageSeparator(String markdownPageSeparator) {
this.markdownPageSeparator = markdownPageSeparator;
}
/**
* Gets the string, that separates content from different pages in text.
*
* @return The specified string.
*/
public String getTextPageSeparator() {
return textPageSeparator;
}
/**
* Sets the string, that separates content from different pages in text.
*
* @param textPageSeparator The specified string.
*/
public void setTextPageSeparator(String textPageSeparator) {
this.textPageSeparator = textPageSeparator;
}
/**
* Gets the string, that separates content from different pages in html.
*
* @return The specified string.
*/
public String getHtmlPageSeparator() {
return htmlPageSeparator;
}
/**
* Sets the string, that separates content from different pages in html.
*
* @param htmlPageSeparator The specified string.
*/
public void setHtmlPageSeparator(String htmlPageSeparator) {
this.htmlPageSeparator = htmlPageSeparator;
}
/**
* Checks if images should be embedded as Base64 data URIs in the output.
*
* @return true if images should be embedded as Base64, false for file path references.
*/
public boolean isEmbedImages() {
return IMAGE_OUTPUT_EMBEDDED.equals(imageOutput);
}
/**
* Checks if image extraction is disabled.
*
* @return true if image output is off, false otherwise.
*/
public boolean isImageOutputOff() {
return IMAGE_OUTPUT_OFF.equals(imageOutput);
}
/**
* Gets the image output mode.
*
* @return The image output mode (off, embedded, or external).
*/
public String getImageOutput() {
return imageOutput;
}
/**
* Sets the image output mode.
*
* @param imageOutput The image output mode (off, embedded, or external).
* @throws IllegalArgumentException if the mode is not supported.
*/
public void setImageOutput(String imageOutput) {
if (imageOutput != null && !isValidImageOutput(imageOutput)) {
throw new IllegalArgumentException(
String.format("Unsupported image output mode '%s'. Supported values: %s",
imageOutput, getImageOutputOptions(", ")));
}
this.imageOutput = imageOutput != null ? imageOutput.toLowerCase(Locale.ROOT) : IMAGE_OUTPUT_EXTERNAL;
}
/**
* Gets the list of supported image output options.
*
* @param delimiter The delimiter to use between options.
* @return The string with image output modes separated by the delimiter.
*/
public static String getImageOutputOptions(CharSequence delimiter) {
return String.join(delimiter, imageOutputOptions);
}
/**
* Checks if the given image output mode is valid.
*
* @param mode The image output mode to check.
* @return true if the mode is valid, false otherwise.
*/
public static boolean isValidImageOutput(String mode) {
return mode != null && imageOutputOptions.contains(mode.toLowerCase(Locale.ROOT));
}
/**
* Gets the image format for extracted images.
*
* @return The image format (png or jpeg).
*/
public String getImageFormat() {
return imageFormat;
}
/**
* Sets the image format for extracted images.
*
* @param imageFormat The image format (png or jpeg).
* @throws IllegalArgumentException if the format is not supported.
*/
public void setImageFormat(String imageFormat) {
if (imageFormat != null && !isValidImageFormat(imageFormat)) {
throw new IllegalArgumentException(
String.format("Unsupported image format '%s'. Supported values: %s",
imageFormat, getImageFormatOptions(", ")));
}
this.imageFormat = imageFormat != null ? imageFormat.toLowerCase(Locale.ROOT) : IMAGE_FORMAT_PNG;
}
/**
* Gets the list of supported image format options.
*
* @param delimiter The delimiter to use between options.
* @return The string with image formats separated by the delimiter.
*/
public static String getImageFormatOptions(CharSequence delimiter) {
return String.join(delimiter, imageFormatOptions);
}
/**
* Checks if the given image format is valid.
*
* @param format The image format to check.
* @return true if the format is valid, false otherwise.
*/
public static boolean isValidImageFormat(String format) {
return format != null && imageFormatOptions.contains(format.toLowerCase(Locale.ROOT));
}
/**
* Gets the directory for extracted images.
*
* @return The image directory path, or null for default.
*/
public String getImageDir() {
return imageDir;
}
/**
* Sets the directory for extracted images.
* Empty or whitespace-only strings are treated as null (use default).
*
* @param imageDir The directory path for extracted images.
*/
public void setImageDir(String imageDir) {
if (imageDir != null && imageDir.trim().isEmpty()) {
this.imageDir = null;
} else {
this.imageDir = imageDir;
}
}
private static final String INVALID_PAGE_RANGE_FORMAT = "Invalid page range format: '%s'. Expected format: 1,3,5-7";
/** Split limit to preserve trailing empty strings (e.g., "5-" splits to ["5", ""]). */
private static final int SPLIT_KEEP_EMPTY_TRAILING = -1;
/**
* Gets the pages to extract from the PDF.
*
* @return The page specification string (e.g., "1,3,5-7"), or null for all pages.
*/
public String getPages() {
return pages;
}
/**
* Sets the pages to extract from the PDF.
*
* @param pages The page specification (e.g., "1,3,5-7"). Use null or empty for all pages.
* @throws IllegalArgumentException if the format is invalid.
*/
public void setPages(String pages) {
if (pages != null && !pages.trim().isEmpty()) {
this.cachedPageNumbers = parsePageRanges(pages);
} else {
this.cachedPageNumbers = null;
}
this.pages = pages;
}
/**
* Gets the list of page numbers to extract.
*
* @return List of 1-based page numbers, or empty list if all pages should be extracted.
*/
public List<Integer> getPageNumbers() {
if (cachedPageNumbers == null) {
return new ArrayList<>();
}
return new ArrayList<>(cachedPageNumbers);
}
/**
* Parses a page range specification into a list of page numbers.
*
* @param pages The page specification (e.g., "1,3,5-7").
* @return List of 1-based page numbers.
* @throws IllegalArgumentException if the format is invalid.
*/
private static List<Integer> parsePageRanges(String pages) {
List<Integer> result = new ArrayList<>();
String[] parts = pages.split(",");
for (String part : parts) {
String trimmed = part.trim();
if (trimmed.isEmpty()) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, pages));
}
if (trimmed.contains("-")) {
parseRange(trimmed, pages, result);
} else {
parseSinglePage(trimmed, pages, result);
}
}
return result;
}
private static void parseRange(String range, String fullInput, List<Integer> result) {
String[] parts = range.split("-", SPLIT_KEEP_EMPTY_TRAILING);
if (parts.length != 2 || parts[0].isEmpty() || parts[1].isEmpty()) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, fullInput));
}
try {
int start = Integer.parseInt(parts[0].trim());
int end = Integer.parseInt(parts[1].trim());
if (start < 1 || end < 1) {
throw new IllegalArgumentException(
String.format("Page numbers must be positive: '%s'", fullInput));
}
if (start > end) {
throw new IllegalArgumentException(
String.format("Invalid page range '%s': start page cannot be greater than end page", range));
}
for (int i = start; i <= end; i++) {
result.add(i);
}
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, fullInput));
}
}
private static void parseSinglePage(String page, String fullInput, List<Integer> result) {
try {
int pageNum = Integer.parseInt(page);
if (pageNum < 1) {
throw new IllegalArgumentException(
String.format("Page numbers must be positive: '%s'", fullInput));
}
result.add(pageNum);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(String.format(INVALID_PAGE_RANGE_FORMAT, fullInput));
}
}
/**
* Gets the hybrid backend name.
*
* @return The hybrid backend (off, docling, hancom, azure, google).
*/
public String getHybrid() {
return hybrid;
}
/**
* Sets the hybrid backend.
*
* @param hybrid The hybrid backend (off, docling, hancom, azure, google).
* @throws IllegalArgumentException if the backend is not supported.
*/
public void setHybrid(String hybrid) {
if (hybrid != null && !isValidHybrid(hybrid)) {
throw new IllegalArgumentException(
String.format("Unsupported hybrid backend '%s'. Supported values: %s",
hybrid, getHybridOptions(", ")));
}
this.hybrid = hybrid != null ? hybrid.toLowerCase(Locale.ROOT) : HYBRID_OFF;
}
/**
* Gets the list of supported hybrid backend options.
*
* @param delimiter The delimiter to use between options.
* @return The string with hybrid backends separated by the delimiter.
*/
public static String getHybridOptions(CharSequence delimiter) {
return String.join(delimiter, hybridOptions);
}
/**
* Checks if the given hybrid backend is valid.
*
* @param hybrid The hybrid backend to check.
* @return true if the backend is valid, false otherwise.
*/
public static boolean isValidHybrid(String hybrid) {
return hybrid != null && hybridOptions.contains(hybrid.toLowerCase(Locale.ROOT));
}
/**
* Checks if hybrid processing is enabled.
*
* @return true if hybrid mode is not off, false otherwise.
*/
public boolean isHybridEnabled() {
return !HYBRID_OFF.equals(hybrid);
}
/**
* Gets the hybrid configuration.
*
* @return The HybridConfig instance.
*/
public HybridConfig getHybridConfig() {
return hybridConfig;
}
/**
* Gets the list of supported hybrid mode options.
*
* @param delimiter The delimiter to use between options.
* @return The string with hybrid modes separated by the delimiter.
*/
public static String getHybridModeOptions(CharSequence delimiter) {
return String.join(delimiter, hybridModeOptions);
}
/**
* Checks if the given hybrid mode is valid.
*
* @param mode The hybrid mode to check.
* @return true if the mode is valid, false otherwise.
*/
public static boolean isValidHybridMode(String mode) {
return mode != null && hybridModeOptions.contains(mode.toLowerCase(Locale.ROOT));
}
/**
* Checks if page headers and footers should be included in output.
*
* @return true if headers and footers should be included, false otherwise.
*/
public boolean isIncludeHeaderFooter() {
return includeHeaderFooter;
}
/**
* Enables or disables inclusion of page headers and footers in output.
*
* @param includeHeaderFooter true to include headers and footers, false to exclude.
*/
public void setIncludeHeaderFooter(boolean includeHeaderFooter) {
this.includeHeaderFooter = includeHeaderFooter;
}
public boolean isDetectStrikethrough() {
return detectStrikethrough;
}
public void setDetectStrikethrough(boolean detectStrikethrough) {
this.detectStrikethrough = detectStrikethrough;
}
private boolean outputStdout = false;
public boolean isOutputStdout() {
return outputStdout;
}
public void setOutputStdout(boolean outputStdout) {
this.outputStdout = outputStdout;
}
private int threads = 1;
public int getThreads() {
return threads;
}
public void setThreads(int threads) {
if (threads < 1) {
throw new IllegalArgumentException("threads must be >= 1, got " + threads);
}
this.threads = Math.min(threads, Runtime.getRuntime().availableProcessors());
}
/**
* Returns true if any output format requires structured content
* (reading order, heading levels, list detection, etc.).
* Text-only output does not need these expensive processing steps.
*/
public boolean needsStructuredProcessing() {
return isGenerateMarkdown() || isGenerateHtml() || isGenerateJSON() || isGeneratePDF();
}
/**
* Resolves conflicts between individually valid option values.
* Call once after all setters, before passing the Config to a processor.
* Currently: in hybrid mode, forces {@code threads} to 1 because the hybrid
* pipeline runs sequentially regardless of this value.
*/
public void normalize() {
if (isHybridEnabled() && threads > 1) {
LOGGER.log(Level.WARNING,
"--threads={0} ignored in hybrid mode (forcing threads=1); "
+ "the hybrid pipeline processes pages sequentially",
threads);
threads = 1;
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/FilterConfig.java
+205
View File
@@ -0,0 +1,205 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.utils.SanitizationRule;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;
/**
* Configuration class for content filtering options.
* Controls filtering of hidden text, out-of-page content, tiny text, and hidden OCGs.
*/
public class FilterConfig {
private boolean filterHiddenText = false;
private boolean filterOutOfPage = true;
private boolean filterTinyText = true;
private boolean filterHiddenOCG = true;
private boolean filterSensitiveData = false;
private final List<SanitizationRule> filterRules;
/** Default rules */
private void initializeDefaultRules() {
filterRules.add(new SanitizationRule(
Pattern.compile("[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}"),
"email@example.com"
));
filterRules.add(new SanitizationRule(
Pattern.compile("[+]\\d+(?:-\\d+)+"),
"+00-0000-0000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("[A-Z]{1,2}\\d{6,9}"),
"AA0000000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b\\d{4}-?\\d{4}-?\\d{4}-?\\d{4}\\b"),
"0000-0000-0000-0000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b\\d{10,18}\\b"),
"0000000000000000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b(?:\\d{1,3}\\.){3}\\d{1,3}\\b"),
"0.0.0.0"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}\\b"),
"0.0.0.0::1"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b(?:[0-9A-Fa-f]{2}:){5}[0-9A-Fa-f]{2}\\b"),
"00:00:00:00:00:00"
));
filterRules.add(new SanitizationRule(
Pattern.compile("\\b\\d{15}\\b"),
"000000000000000"
));
filterRules.add(new SanitizationRule(
Pattern.compile("https?://[A-Za-z0-9.-]+(:\\d+)?(/\\S*)?"),
"https://example.com"
));
}
/**
* Constructor initializing the configuration of filter.
*/
public FilterConfig() {
this.filterRules = new ArrayList<>();
initializeDefaultRules();
}
/**
* Enables or disables filter of hidden text.
*
* @param filterHiddenText true to enable filter, false to disable.
*/
public void setFilterHiddenText(boolean filterHiddenText) {
this.filterHiddenText = filterHiddenText;
}
/**
* Checks if the processor should attempt to find and extract hidden text.
*
* @return true if hidden text is filtered, false otherwise.
*/
public boolean isFilterHiddenText() {
return filterHiddenText;
}
/**
* Enables or disables checking content that exceeds MediaBox or CropBox.
*
* @param filterOutOfPage true to enable, false to disable.
*/
public void setFilterOutOfPage(boolean filterOutOfPage) {
this.filterOutOfPage = filterOutOfPage;
}
/**
* Checks if the processor should filter out of page content.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterOutOfPage() {
return filterOutOfPage;
}
/**
* Checks if the processor should filter out tiny text.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterTinyText() {
return filterTinyText;
}
/**
* Enables or disables filter of tiny text.
*
* @param filterTinyText true to enable filter, false to disable.
*/
public void setFilterTinyText(boolean filterTinyText) {
this.filterTinyText = filterTinyText;
}
/**
* Checks if the processor should filter out hidden OCGs.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterHiddenOCG() {
return filterHiddenOCG;
}
/**
* Enables or disables filter of hidden OCGs.
*
* @param filterHiddenOCG true to enable filter, false to disable.
*/
public void setFilterHiddenOCG(boolean filterHiddenOCG) {
this.filterHiddenOCG = filterHiddenOCG;
}
/**
* Checks if the processor should filter out sensitive data.
*
* @return true if filter is enabled, false otherwise.
*/
public boolean isFilterSensitiveData() {
return filterSensitiveData;
}
/**
* Enables or disables filter of sensitive data.
*
* @param filterSensitiveData true to enable filter, false to disable.
*/
public void setFilterSensitiveData(boolean filterSensitiveData) {
this.filterSensitiveData = filterSensitiveData;
}
/**
* Gets custom filter sanitization rules.
*
* @return List of sanitization rules.
*/
public List<SanitizationRule> getFilterRules() {
return filterRules;
}
/**
* Add custom filter sanitization rule.
*
* @param pattern pattern string.
* @param replacement pattern replacement string.
*/
public void addFilterRule(String pattern, String replacement) {
filterRules.add(new SanitizationRule(Pattern.compile(pattern), replacement));
}
/**
* Remove filter sanitization rule.
*
* @param pattern pattern string.
*/
public void removeFilterRule(String pattern) {
filterRules.removeIf(rule -> rule.getPattern().pattern().equals(pattern));
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/OpenDataLoaderPDF.java
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.hybrid.HybridClientFactory;
import org.opendataloader.pdf.processors.DocumentProcessor;
import java.io.IOException;
/**
* The main entry point for the opendataloader-pdf library.
* Use the static method {@link #processFile(String, Config)} to process a PDF.
*/
public final class OpenDataLoaderPDF {
private OpenDataLoaderPDF() {
}
/**
* Processes a PDF file to extract its content and structure based on the provided configuration.
*
* @param inputPdfName The path to the input PDF file.
* @param config The configuration object specifying output formats and other options.
* @throws IOException If an error occurs during file reading or processing.
*/
public static void processFile(String inputPdfName, Config config) throws IOException {
DocumentProcessor.processFile(inputPdfName, config);
}
/**
* Shuts down any cached resources used by the library.
*
* <p>This method should be called when processing is complete, typically at CLI exit.
* It releases resources such as HTTP client thread pools used for hybrid mode backends.
*/
public static void shutdown() {
HybridClientFactory.shutdown();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/OutputWriter.java
+83
View File
@@ -0,0 +1,83 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api;
import org.opendataloader.pdf.processors.DocumentProcessor;
import org.opendataloader.pdf.processors.ExtractionResult;
import java.io.IOException;
/**
* Writes configured output files (JSON, Markdown, HTML, PDF, text, images,
* tagged PDF) from a pre-computed {@link ExtractionResult}.
*
* <p>Use this when you have already run extraction once (e.g. via
* {@link AutoTagger#tag(ExtractionResult, Float)}) and want to emit file
* outputs from that same result without re-extracting.
*
* <p>Typical two-phase usage:
* <pre>{@code
* Config config = new Config();
* config.setOutputFolder("/out");
* config.setGenerateJSON(true);
* config.setGenerateMarkdown(true);
*
* // Phase 1: extract once
* ExtractionResult extraction =
* org.opendataloader.pdf.processors.DocumentProcessor.extractContents(
* "input.pdf", config);
*
* // Phase 2a: write output files
* OutputWriter.writeOutputs("input.pdf", extraction, config);
*
* // Phase 2b: tag in-memory and reuse the same extraction
* try (TaggingResult tagged = AutoTagger.tag("input.pdf", extraction)) {
* // ... use tagged.getDocument()
* }
* }</pre>
*
* <p>For the single-call extraction-and-output pipeline, use
* {@link OpenDataLoaderPDF#processFile} instead.
*/
public final class OutputWriter {
private OutputWriter() {
}
/**
* Writes the output files configured on {@code config} (e.g.
* {@code generateJSON}, {@code generateMarkdown}, {@code generateHtml},
* {@code generatePDF}, {@code generateTaggedPDF}, {@code generateText})
* using the supplied pre-computed extraction.
*
* <p>This method does <em>not</em> re-run extraction. Output behaviour is
* identical to {@link OpenDataLoaderPDF#processFile} for the same
* {@link Config}, including stdout mode, image directory resolution, and
* tagged-PDF generation.
*
* @param inputPdfName path to the input PDF file (used for filename derivation
* and tagged-PDF / annotated-PDF re-saves; not re-parsed)
* @param extraction pre-computed extraction result (from
* {@code DocumentProcessor.extractContents})
* @param config configuration controlling which output formats to emit
* @throws IOException if writing any output file fails
*/
public static void writeOutputs(String inputPdfName, ExtractionResult extraction, Config config)
throws IOException {
DocumentProcessor.generateOutputs(inputPdfName, extraction.getContents(), config,
extraction.getElementMetadata());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/TaggingResult.java
+83
View File
@@ -0,0 +1,83 @@
package org.opendataloader.pdf.api;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.hybrid.ElementMetadata;
import org.verapdf.pd.PDDocument;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
/**
* Result of {@link AutoTagger#tag}. Contains the tagged PDF document in-memory
* and processing timing metadata.
*
* <p>Implements {@link AutoCloseable} for use with try-with-resources.
* The caller is responsible for closing this result, which releases the
* underlying PDDocument resources.
*/
public class TaggingResult implements AutoCloseable {
private final PDDocument document;
private final long extractionNs;
private final long taggingNs;
private final JsonNode hybridTimings;
private final Map<Long, ElementMetadata> elementMetadata;
public TaggingResult(PDDocument document, long extractionNs, long taggingNs,
JsonNode hybridTimings, Map<Long, ElementMetadata> elementMetadata) {
if (document == null) {
throw new IllegalArgumentException("document must not be null");
}
this.document = document;
this.extractionNs = extractionNs;
this.taggingNs = taggingNs;
this.hybridTimings = hybridTimings;
this.elementMetadata = elementMetadata != null ? elementMetadata : Collections.emptyMap();
}
public TaggingResult(PDDocument document, long extractionNs, long taggingNs, JsonNode hybridTimings) {
this(document, extractionNs, taggingNs, hybridTimings, Collections.emptyMap());
}
/** The tagged PDF document. Do not close this directly — close the TaggingResult instead. */
public PDDocument getDocument() {
return document;
}
/** Time spent on extraction (parsing + layout + content extraction) in nanoseconds. */
public long getExtractionNs() {
return extractionNs;
}
/** Time spent on auto-tagging (structure tree creation) in nanoseconds. */
public long getTaggingNs() {
return taggingNs;
}
/** Per-step hybrid server timings, or null if hybrid mode was not used. */
public JsonNode getHybridTimings() {
return hybridTimings;
}
/** Element metadata from hybrid backend, or empty map if not available. */
public Map<Long, ElementMetadata> getElementMetadata() {
return elementMetadata;
}
/**
* Save the tagged PDF to a file.
*
* @param outputPath the output file path
*/
public void saveTo(String outputPath) throws IOException {
document.saveAs(outputPath);
}
@Override
public void close() {
if (document != null) {
document.close();
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/api/cli/CLIOptions.java
+819
View File
@@ -0,0 +1,819 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.api.cli;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.hybrid.HybridConfig;
import java.io.File;
import java.io.PrintStream;
import java.util.Arrays;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Adapter that maps Apache Commons CLI options to {@link Config} / {@link HybridConfig}.
*
* <p><b>Stable API for downstream tools</b> (e.g. opendataloader-pdfua) — these four
* members are the supported integration surface and will not break compatibly:
* <ul>
* <li>{@link #defineOptions()} — get a fully populated {@code Options} instance</li>
* <li>{@link #addAllTo(Options)} — add core options into an externally-built {@code Options}</li>
* <li>{@link #applyAllTo(Config, CommandLine)} — populate a {@code Config} from a parsed line</li>
* <li>{@link #FOLDER_OPTION} — short option name for {@code --output-dir}</li>
* </ul>
*
* <p><b>Everything else is internal.</b> The numerous other public {@code static} members
* (option-name constants, helpers like {@code createConfigFromCommandLine},
* {@code exportOptionsAsJson}) exist for the CLI module ({@code CLIMain}) and the
* options-export tooling that drives Node/Python binding generation. Their visibility
* is {@code public} only for cross-package access within this codebase; they are
* <i>not</i> part of the supported API and may be renamed, moved, or removed in any
* release. Downstream consumers depending on them do so at their own risk.
*
* <p>Pdfua's usage pattern (build your own {@code Options}, add core's, parse, then
* populate {@code Config}):
* <pre>{@code
* Options options = new Options();
* options.addOption(...); // your tool-specific options
* CLIOptions.addAllTo(options); // add core's options
* CommandLine cmd = parser.parse(options, args);
* Config core = new Config();
* CLIOptions.applyAllTo(core, cmd);
* }</pre>
*/
public class CLIOptions {
// ===== Output Directory =====
public static final String FOLDER_OPTION = "o";
private static final String FOLDER_LONG_OPTION = "output-dir";
private static final String FOLDER_DESC = "Directory where output files are written. Default: input file directory";
// ===== Password =====
public static final String PASSWORD_OPTION = "p";
private static final String PASSWORD_LONG_OPTION = "password";
private static final String PASSWORD_DESC = "Password for encrypted PDF files";
// ===== Format =====
public static final String FORMAT_OPTION = "f";
public static final String FORMAT_LONG_OPTION = "format";
private static final String FORMAT_DESC = "Output formats (comma-separated). "
+ "Values: json, text, html, pdf, markdown, tagged-pdf. Default: json. "
+ "For HTML inside Markdown use --markdown-with-html. "
+ "For image extraction control use --image-output.";
// ===== Quiet =====
public static final String QUIET_OPTION = "q";
private static final String QUIET_LONG_OPTION = "quiet";
private static final String QUIET_DESC = "Suppress console logging output";
// ===== Content Safety =====
private static final String CONTENT_SAFETY_OFF_LONG_OPTION = "content-safety-off";
private static final String CONTENT_SAFETY_OFF_DESC = "Disable content safety filters. "
+ "Values: all, hidden-text, off-page, tiny, hidden-ocg";
// ===== Sanitize =====
private static final String SANITIZE_LONG_OPTION = "sanitize";
private static final String SANITIZE_DESC = "Enable sensitive data sanitization. "
+ "Replaces emails, phone numbers, IPs, credit cards, and URLs with placeholders";
// ===== Keep Line Breaks =====
private static final String KEEP_LINE_BREAKS_LONG_OPTION = "keep-line-breaks";
private static final String KEEP_LINE_BREAKS_DESC = "Preserve original line breaks in extracted text";
// ===== Replace Invalid Chars =====
private static final String REPLACE_INVALID_CHARS_LONG_OPTION = "replace-invalid-chars";
private static final String REPLACE_INVALID_CHARS_DESC = "Replacement character for invalid/unrecognized characters. Default: space";
// ===== Use Struct Tree =====
private static final String USE_STRUCT_TREE_LONG_OPTION = "use-struct-tree";
private static final String USE_STRUCT_TREE_DESC = "Use PDF structure tree (tagged PDF) for reading order and semantic structure. Output quality depends on tag quality";
// ===== Table Method =====
private static final String TABLE_METHOD_LONG_OPTION = "table-method";
private static final String TABLE_METHOD_DESC = "Table detection method. Values: default (border-based), cluster (border + cluster). Default: default";
// ===== Reading Order =====
private static final String READING_ORDER_LONG_OPTION = "reading-order";
private static final String READING_ORDER_DESC = "Reading order algorithm. Values: off, xycut. Default: xycut";
// ===== Page Separators =====
private static final String MARKDOWN_PAGE_SEPARATOR_LONG_OPTION = "markdown-page-separator";
private static final String MARKDOWN_PAGE_SEPARATOR_DESC = "Separator between pages in Markdown output. Use %page-number% for page numbers. Default: none";
private static final String TEXT_PAGE_SEPARATOR_LONG_OPTION = "text-page-separator";
private static final String TEXT_PAGE_SEPARATOR_DESC = "Separator between pages in text output. Use %page-number% for page numbers. Default: none";
private static final String HTML_PAGE_SEPARATOR_LONG_OPTION = "html-page-separator";
private static final String HTML_PAGE_SEPARATOR_DESC = "Separator between pages in HTML output. Use %page-number% for page numbers. Default: none";
// ===== Image Options =====
private static final String IMAGE_OUTPUT_LONG_OPTION = "image-output";
private static final String IMAGE_OUTPUT_DESC = "Image output mode. Values: off (no images), embedded (Base64 data URIs), external (file references). Default: external";
private static final String IMAGE_FORMAT_LONG_OPTION = "image-format";
private static final String IMAGE_FORMAT_DESC = "Output format for extracted images. Values: png, jpeg. Default: png";
private static final String IMAGE_DIR_LONG_OPTION = "image-dir";
private static final String IMAGE_DIR_DESC = "Directory for extracted images (applies only with --image-output external)";
// ===== Pages =====
private static final String PAGES_LONG_OPTION = "pages";
private static final String PAGES_DESC = "Pages to extract (e.g., \"1,3,5-7\"). Default: all pages";
// ===== Include Header Footer =====
private static final String INCLUDE_HEADER_FOOTER_LONG_OPTION = "include-header-footer";
private static final String INCLUDE_HEADER_FOOTER_DESC = "Include page headers and footers in output";
// ===== Detect Strikethrough =====
private static final String DETECT_STRIKETHROUGH_LONG_OPTION = "detect-strikethrough";
private static final String DETECT_STRIKETHROUGH_DESC = "Detect strikethrough text and wrap with ~~ in Markdown output or <del></del> tag in HTML output (experimental)";
// ===== Hybrid Mode =====
private static final String HYBRID_LONG_OPTION = "hybrid";
private static final String HYBRID_DESC = "Hybrid backend (requires a running server). "
+ "Quick start: pip install \"opendataloader-pdf[hybrid]\" && opendataloader-pdf-hybrid --port 5002. "
+ "For remote servers use --hybrid-url. Values: off (default), docling-fast, hancom-ai";
private static final String HYBRID_MODE_LONG_OPTION = "hybrid-mode";
private static final String HYBRID_MODE_DESC = "Hybrid triage mode. Values: auto (default, dynamic triage), full (skip triage, all pages to backend)";
// Deprecated: OCR settings are now configured on the hybrid server
private static final String HYBRID_OCR_LONG_OPTION = "hybrid-ocr";
private static final String HYBRID_OCR_DESC = "[Deprecated] OCR settings are now configured on the hybrid server (--ocr-lang, --force-ocr)";
private static final String HYBRID_URL_LONG_OPTION = "hybrid-url";
private static final String HYBRID_URL_DESC = "Hybrid backend server URL (overrides default)";
private static final String HYBRID_TIMEOUT_LONG_OPTION = "hybrid-timeout";
private static final String HYBRID_TIMEOUT_DESC = "Hybrid backend request timeout in milliseconds (0 = no timeout). Default: 0";
private static final String HYBRID_FALLBACK_LONG_OPTION = "hybrid-fallback";
private static final String HYBRID_FALLBACK_DESC = "Opt in to Java fallback on hybrid backend error (default: disabled)";
// ===== Hybrid hancom-ai backend-specific =====
private static final String HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION =
"hybrid-hancom-ai-regionlist-strategy";
private static final String HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_DESC =
"DLA label 7 (regionlist) handling. Requires --hybrid=hancom-ai. "
+ "Values: table-first (default; check TSR overlap), list-only (skip TSR, always treat as list)";
private static final String HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION =
"hybrid-hancom-ai-ocr-strategy";
private static final String HYBRID_HANCOM_AI_OCR_STRATEGY_DESC =
"OCR strategy. Requires --hybrid=hancom-ai. "
+ "Values: off (stream-only), auto (default; stream first, OCR fallback), force (OCR-only)";
private static final String HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION =
"hybrid-hancom-ai-image-cache";
private static final String HYBRID_HANCOM_AI_IMAGE_CACHE_DESC =
"Page image cache backing. Requires --hybrid=hancom-ai. "
+ "Values: memory (default), disk";
private static final String HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION =
"hybrid-hancom-ai-save-crops";
private static final String HYBRID_HANCOM_AI_SAVE_CROPS_DESC =
"Persist cropped figure images to disk for debugging. Requires --hybrid=hancom-ai";
private static final String HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION =
"hybrid-hancom-ai-crop-output-dir";
private static final String HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_DESC =
"Output directory for --hybrid-hancom-ai-save-crops. Requires --hybrid=hancom-ai";
// ===== Stdout Output =====
private static final String TO_STDOUT_LONG_OPTION = "to-stdout";
private static final String TO_STDOUT_DESC = "Write output to stdout instead of file (single format only)";
// ===== Threads =====
private static final String THREADS_LONG_OPTION = "threads";
private static final String THREADS_DESC = "Number of worker threads for per-page processing. "
+ "Default: 1 (sequential, stable). Values >1 (experimental) run pages in parallel for faster throughput; "
+ "output may vary slightly on some PDFs. Capped at the number of available CPU cores. "
+ "Applies to the native Java pipeline only; ignored in --hybrid mode";
// ===== Markdown modifiers =====
public static final String HTML_IN_MARKDOWN_LONG_OPTION = "markdown-with-html";
private static final String HTML_IN_MARKDOWN_DESC =
"Allow HTML tags inside Markdown output for complex structures such as multi-row-span tables. "
+ "Implies --format markdown.";
// ===== Export Options (internal) =====
public static final String EXPORT_OPTIONS_LONG_OPTION = "export-options";
// ===== Legacy Options (hidden, backward compatibility) =====
public static final String PDF_REPORT_LONG_OPTION = "pdf";
public static final String MARKDOWN_REPORT_LONG_OPTION = "markdown";
public static final String HTML_REPORT_LONG_OPTION = "html";
private static final String MARKDOWN_IMAGE_LONG_OPTION = "markdown-with-images";
public static final String NO_JSON_REPORT_LONG_OPTION = "no-json";
/**
* Single source of truth for all CLI option definitions.
* Add new options here - they will automatically be available in both CLI and
* JSON export.
*/
private static final List<OptionDefinition> OPTION_DEFINITIONS = Arrays.asList(
// Primary options (exported to JSON)
new OptionDefinition(FOLDER_LONG_OPTION, FOLDER_OPTION, "string", null, FOLDER_DESC, true),
new OptionDefinition(PASSWORD_LONG_OPTION, PASSWORD_OPTION, "string", null, PASSWORD_DESC, true),
new OptionDefinition(FORMAT_LONG_OPTION, FORMAT_OPTION, "string", null, FORMAT_DESC, true),
new OptionDefinition(QUIET_LONG_OPTION, QUIET_OPTION, "boolean", false, QUIET_DESC, true),
new OptionDefinition(CONTENT_SAFETY_OFF_LONG_OPTION, null, "string", null, CONTENT_SAFETY_OFF_DESC, true),
new OptionDefinition(SANITIZE_LONG_OPTION, null, "boolean", false, SANITIZE_DESC, true),
new OptionDefinition(KEEP_LINE_BREAKS_LONG_OPTION, null, "boolean", false, KEEP_LINE_BREAKS_DESC, true),
new OptionDefinition(REPLACE_INVALID_CHARS_LONG_OPTION, null, "string", " ", REPLACE_INVALID_CHARS_DESC,
true),
new OptionDefinition(USE_STRUCT_TREE_LONG_OPTION, null, "boolean", false, USE_STRUCT_TREE_DESC, true),
new OptionDefinition(TABLE_METHOD_LONG_OPTION, null, "string", "default", TABLE_METHOD_DESC, true),
new OptionDefinition(READING_ORDER_LONG_OPTION, null, "string", "xycut", READING_ORDER_DESC, true),
new OptionDefinition(MARKDOWN_PAGE_SEPARATOR_LONG_OPTION, null, "string", null,
MARKDOWN_PAGE_SEPARATOR_DESC, true),
new OptionDefinition(HTML_IN_MARKDOWN_LONG_OPTION, null, "boolean", false,
HTML_IN_MARKDOWN_DESC, true),
new OptionDefinition(TEXT_PAGE_SEPARATOR_LONG_OPTION, null, "string", null, TEXT_PAGE_SEPARATOR_DESC, true),
new OptionDefinition(HTML_PAGE_SEPARATOR_LONG_OPTION, null, "string", null, HTML_PAGE_SEPARATOR_DESC, true),
new OptionDefinition(IMAGE_OUTPUT_LONG_OPTION, null, "string", "external", IMAGE_OUTPUT_DESC, true),
new OptionDefinition(IMAGE_FORMAT_LONG_OPTION, null, "string", "png", IMAGE_FORMAT_DESC, true),
new OptionDefinition(IMAGE_DIR_LONG_OPTION, null, "string", null, IMAGE_DIR_DESC, true),
new OptionDefinition(PAGES_LONG_OPTION, null, "string", null, PAGES_DESC, true),
new OptionDefinition(INCLUDE_HEADER_FOOTER_LONG_OPTION, null, "boolean", false,
INCLUDE_HEADER_FOOTER_DESC, true),
new OptionDefinition(DETECT_STRIKETHROUGH_LONG_OPTION, null, "boolean", false,
DETECT_STRIKETHROUGH_DESC, true),
new OptionDefinition(HYBRID_LONG_OPTION, null, "string", "off", HYBRID_DESC, true),
new OptionDefinition(HYBRID_MODE_LONG_OPTION, null, "string", "auto", HYBRID_MODE_DESC, true),
new OptionDefinition(HYBRID_URL_LONG_OPTION, null, "string", null, HYBRID_URL_DESC, true),
new OptionDefinition(HYBRID_TIMEOUT_LONG_OPTION, null, "string", "0", HYBRID_TIMEOUT_DESC, true),
new OptionDefinition(HYBRID_FALLBACK_LONG_OPTION, null, "boolean", false, HYBRID_FALLBACK_DESC, true),
new OptionDefinition(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION, null, "string",
"table-first", HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_DESC, true),
new OptionDefinition(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION, null, "string",
"auto", HYBRID_HANCOM_AI_OCR_STRATEGY_DESC, true),
new OptionDefinition(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION, null, "string",
"memory", HYBRID_HANCOM_AI_IMAGE_CACHE_DESC, true),
new OptionDefinition(TO_STDOUT_LONG_OPTION, null, "boolean", false, TO_STDOUT_DESC, true),
new OptionDefinition(THREADS_LONG_OPTION, null, "string", "1", THREADS_DESC, true),
new OptionDefinition(EXPORT_OPTIONS_LONG_OPTION, null, "boolean", null, null, false),
// Legacy options (not exported, for backward compatibility)
new OptionDefinition(HYBRID_OCR_LONG_OPTION, null, "string", null, HYBRID_OCR_DESC, false),
new OptionDefinition(PDF_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(MARKDOWN_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(HTML_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(MARKDOWN_IMAGE_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(NO_JSON_REPORT_LONG_OPTION, null, "boolean", null, null, false),
new OptionDefinition(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION, null, "boolean",
false, HYBRID_HANCOM_AI_SAVE_CROPS_DESC, false),
new OptionDefinition(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION, null, "string",
null, HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_DESC, false));
public static Options defineOptions() {
Options options = new Options();
addAllTo(options);
return options;
}
/**
* Registers every core CLI option onto an external {@link Options} instance.
* Used by downstream CLIs (e.g. opendataloader-pdfua) that want to inherit
* the entire core option set and add their own options on top.
*
* @param options the Options instance to populate
*/
public static void addAllTo(Options options) {
for (OptionDefinition def : OPTION_DEFINITIONS) {
options.addOption(def.toOption());
}
}
public static Config createConfigFromCommandLine(CommandLine commandLine) {
Config config = new Config();
if (commandLine.hasOption(CLIOptions.FOLDER_OPTION)) {
config.setOutputFolder(commandLine.getOptionValue(CLIOptions.FOLDER_OPTION));
} else {
String argument = commandLine.getArgs()[0];
File file = new File(argument);
file = new File(file.getAbsolutePath());
config.setOutputFolder(file.isDirectory() ? file.getAbsolutePath() : file.getParent());
}
applyAllTo(config, commandLine);
return config;
}
/**
* Applies every core CLI option from the parsed command line onto the given Config.
* Caller is responsible for setting required Config state that is not represented
* by a CLI option (e.g. output folder when no positional input file is provided).
*
* Used by downstream CLIs that build their own Options + Config and want core
* options applied without paying for the positional-arg-based output-folder
* fallback that {@link #createConfigFromCommandLine} performs.
*
* @param config Config to populate
* @param commandLine parsed CommandLine
*/
public static void applyAllTo(Config config, CommandLine commandLine) {
if (commandLine.hasOption(CLIOptions.PASSWORD_OPTION)) {
config.setPassword(commandLine.getOptionValue(CLIOptions.PASSWORD_OPTION));
}
if (commandLine.hasOption(CLIOptions.KEEP_LINE_BREAKS_LONG_OPTION)) {
config.setKeepLineBreaks(true);
}
if (commandLine.hasOption(CLIOptions.PDF_REPORT_LONG_OPTION)) {
config.setGeneratePDF(true);
}
if (commandLine.hasOption(CLIOptions.MARKDOWN_REPORT_LONG_OPTION)) {
config.setGenerateMarkdown(true);
}
if (commandLine.hasOption(CLIOptions.HTML_REPORT_LONG_OPTION)) {
config.setGenerateHtml(true);
}
if (commandLine.hasOption(CLIOptions.HTML_IN_MARKDOWN_LONG_OPTION)) {
config.setUseHTMLInMarkdown(true);
}
if (commandLine.hasOption(CLIOptions.MARKDOWN_IMAGE_LONG_OPTION)) {
config.setGenerateMarkdown(true);
}
if (commandLine.hasOption(CLIOptions.NO_JSON_REPORT_LONG_OPTION)) {
config.setGenerateJSON(false);
}
if (commandLine.hasOption(CLIOptions.REPLACE_INVALID_CHARS_LONG_OPTION)) {
config.setReplaceInvalidChars(commandLine.getOptionValue(CLIOptions.REPLACE_INVALID_CHARS_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.USE_STRUCT_TREE_LONG_OPTION)) {
config.setUseStructTree(true);
}
if (commandLine.hasOption(INCLUDE_HEADER_FOOTER_LONG_OPTION)) {
config.setIncludeHeaderFooter(true);
}
if (commandLine.hasOption(DETECT_STRIKETHROUGH_LONG_OPTION)) {
config.setDetectStrikethrough(true);
}
if (commandLine.hasOption(CLIOptions.READING_ORDER_LONG_OPTION)) {
config.setReadingOrder(commandLine.getOptionValue(CLIOptions.READING_ORDER_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.MARKDOWN_PAGE_SEPARATOR_LONG_OPTION)) {
config.setMarkdownPageSeparator(commandLine.getOptionValue(CLIOptions.MARKDOWN_PAGE_SEPARATOR_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.TEXT_PAGE_SEPARATOR_LONG_OPTION)) {
config.setTextPageSeparator(commandLine.getOptionValue(CLIOptions.TEXT_PAGE_SEPARATOR_LONG_OPTION));
}
if (commandLine.hasOption(CLIOptions.HTML_PAGE_SEPARATOR_LONG_OPTION)) {
config.setHtmlPageSeparator(commandLine.getOptionValue(CLIOptions.HTML_PAGE_SEPARATOR_LONG_OPTION));
}
applyContentSafetyOption(config, commandLine);
applySanitizeOption(config, commandLine);
applyFormatOption(config, commandLine);
applyTableMethodOption(config, commandLine);
applyImageOptions(config, commandLine);
applyPagesOption(config, commandLine);
applyHybridOptions(config, commandLine);
applyThreadsOption(config, commandLine);
config.normalize();
}
private static void applyThreadsOption(Config config, CommandLine commandLine) {
if (!commandLine.hasOption(THREADS_LONG_OPTION)) {
return;
}
String value = commandLine.getOptionValue(THREADS_LONG_OPTION);
int requested;
try {
requested = Integer.parseInt(value.trim());
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
String.format("Option --threads requires an integer >= 1, got '%s'", value));
}
if (requested < 1) {
throw new IllegalArgumentException(
String.format("Option --threads requires an integer >= 1, got %d", requested));
}
config.setThreads(requested);
int applied = config.getThreads();
if (applied < requested) {
System.err.println(String.format(
"Warning: --threads=%d exceeds available CPU cores; capped to %d.",
requested, applied));
}
}
private static void applyImageOptions(Config config, CommandLine commandLine) {
if (commandLine.hasOption(IMAGE_OUTPUT_LONG_OPTION)) {
String outputValue = commandLine.getOptionValue(IMAGE_OUTPUT_LONG_OPTION);
if (outputValue == null || outputValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --image-output requires a value. Supported values: %s",
Config.getImageOutputOptions(", ")));
}
String output = outputValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidImageOutput(output)) {
throw new IllegalArgumentException(
String.format("Unsupported image output mode '%s'. Supported values: %s",
output, Config.getImageOutputOptions(", ")));
}
config.setImageOutput(output);
}
if (commandLine.hasOption(IMAGE_FORMAT_LONG_OPTION)) {
String formatValue = commandLine.getOptionValue(IMAGE_FORMAT_LONG_OPTION);
if (formatValue == null || formatValue.trim().isEmpty()) {
throw new IllegalArgumentException(
"Option --image-format requires a value. Supported values: png, jpeg");
}
String format = formatValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidImageFormat(format)) {
throw new IllegalArgumentException(
String.format("Unsupported image format '%s'. Supported values: png, jpeg", format));
}
config.setImageFormat(format);
}
if (commandLine.hasOption(IMAGE_DIR_LONG_OPTION)) {
config.setImageDir(commandLine.getOptionValue(IMAGE_DIR_LONG_OPTION));
}
}
private static void applyPagesOption(Config config, CommandLine commandLine) {
if (commandLine.hasOption(PAGES_LONG_OPTION)) {
config.setPages(commandLine.getOptionValue(PAGES_LONG_OPTION));
}
}
private static void applyTableMethodOption(Config config, CommandLine commandLine) {
if (commandLine.hasOption(TABLE_METHOD_LONG_OPTION)) {
String methodValue = commandLine.getOptionValue(TABLE_METHOD_LONG_OPTION);
if (methodValue == null || methodValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --table-method requires a value. Supported values: %s",
Config.getTableMethodOptions(", ")));
}
String method = methodValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidTableMethod(method)) {
throw new IllegalArgumentException(
String.format("Unsupported table method '%s'. Supported values: %s",
method, Config.getTableMethodOptions(", ")));
}
config.setTableMethod(method);
}
}
private static void applyContentSafetyOption(Config config, CommandLine commandLine) {
if (!commandLine.hasOption(CONTENT_SAFETY_OFF_LONG_OPTION)) {
return;
}
String[] optionValues = commandLine.getOptionValues(CONTENT_SAFETY_OFF_LONG_OPTION);
if (optionValues == null || optionValues.length == 0) {
throw new IllegalArgumentException(
"Option --content-safety-off requires at least one value. Supported values: all, hidden-text, off-page, tiny, hidden-ocg");
}
Set<String> values = parseOptionValues(optionValues);
if (values.isEmpty()) {
throw new IllegalArgumentException(
"Option --content-safety-off requires at least one value. Supported values: all, hidden-text, off-page, tiny, hidden-ocg");
}
for (String value : values) {
switch (value) {
case "hidden-text":
config.getFilterConfig().setFilterHiddenText(false);
break;
case "off-page":
config.getFilterConfig().setFilterOutOfPage(false);
break;
case "tiny":
config.getFilterConfig().setFilterTinyText(false);
break;
case "hidden-ocg":
config.getFilterConfig().setFilterHiddenOCG(false);
break;
case "sensitive-data":
System.err.println("Warning: '--content-safety-off sensitive-data' is deprecated and has no effect. "
+ "Sensitive data sanitization is now opt-in. "
+ "Use '--sanitize' to enable masking.");
break;
case "all":
config.getFilterConfig().setFilterHiddenText(false);
config.getFilterConfig().setFilterOutOfPage(false);
config.getFilterConfig().setFilterTinyText(false);
config.getFilterConfig().setFilterHiddenOCG(false);
break;
default:
throw new IllegalArgumentException(String.format(
"Unsupported value '%s'. Supported values: all, hidden-text, off-page, tiny, hidden-ocg",
value));
}
}
}
private static void applySanitizeOption(Config config, CommandLine commandLine) {
if (commandLine.hasOption(SANITIZE_LONG_OPTION)) {
config.getFilterConfig().setFilterSensitiveData(true);
}
}
private static void applyFormatOption(Config config, CommandLine commandLine) {
if (!commandLine.hasOption(FORMAT_OPTION)) {
return;
}
String[] optionValues = commandLine.getOptionValues(FORMAT_OPTION);
if (optionValues == null || optionValues.length == 0) {
throw new IllegalArgumentException(
"Option --format requires at least one value. Supported values: json, text, html, pdf, markdown, tagged-pdf");
}
Set<String> values = parseOptionValues(optionValues);
if (values.isEmpty()) {
throw new IllegalArgumentException(
"Option --format requires at least one value. Supported values: json, text, html, pdf, markdown, tagged-pdf");
}
config.setGenerateJSON(false);
for (String value : values) {
switch (value) {
case "json":
config.setGenerateJSON(true);
break;
case "html":
config.setGenerateHtml(true);
break;
case "text":
config.setGenerateText(true);
break;
case "pdf":
config.setGeneratePDF(true);
break;
case "markdown":
config.setGenerateMarkdown(true);
break;
case "markdown-with-html":
System.err.println("[WARN] --format markdown-with-html is deprecated and will be removed "
+ "in the next major release. Use --format markdown --markdown-with-html instead.");
config.setUseHTMLInMarkdown(true);
break;
case "markdown-with-images":
System.err.println("[WARN] --format markdown-with-images is deprecated and will be removed "
+ "in the next major release. Use --format markdown with --image-output "
+ "(off|embedded|external) instead.");
config.setGenerateMarkdown(true);
break;
case "tagged-pdf":
config.setGenerateTaggedPDF(true);
break;
default:
throw new IllegalArgumentException(String.format(
"Unsupported format '%s'. Supported values: json, text, html, pdf, markdown, tagged-pdf",
value));
}
}
}
private static Set<String> parseOptionValues(String[] optionValues) {
Set<String> values = new LinkedHashSet<>();
for (String rawValue : optionValues) {
if (rawValue == null) {
continue;
}
String[] splitValues = rawValue.split(",");
for (String candidate : splitValues) {
String format = candidate.trim().toLowerCase(Locale.ROOT);
if (!format.isEmpty()) {
values.add(format);
}
}
}
return values;
}
private static void applyHybridOptions(Config config, CommandLine commandLine) {
if (commandLine.hasOption(HYBRID_LONG_OPTION)) {
String hybridValue = commandLine.getOptionValue(HYBRID_LONG_OPTION);
if (hybridValue == null || hybridValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --hybrid requires a value. Supported values: %s",
Config.getHybridOptions(", ")));
}
String hybrid = hybridValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidHybrid(hybrid)) {
throw new IllegalArgumentException(
String.format("Unsupported hybrid backend '%s'. Supported values: %s",
hybrid, Config.getHybridOptions(", ")));
}
config.setHybrid(hybrid);
}
if (commandLine.hasOption(HYBRID_MODE_LONG_OPTION)) {
String modeValue = commandLine.getOptionValue(HYBRID_MODE_LONG_OPTION);
if (modeValue == null || modeValue.trim().isEmpty()) {
throw new IllegalArgumentException(
String.format("Option --hybrid-mode requires a value. Supported values: %s",
Config.getHybridModeOptions(", ")));
}
String mode = modeValue.trim().toLowerCase(Locale.ROOT);
if (!Config.isValidHybridMode(mode)) {
throw new IllegalArgumentException(
String.format("Unsupported hybrid mode '%s'. Supported values: %s",
mode, Config.getHybridModeOptions(", ")));
}
config.getHybridConfig().setMode(mode);
}
if (commandLine.hasOption(HYBRID_OCR_LONG_OPTION)) {
// Deprecated: OCR settings are now configured on the hybrid server
System.err.println("Warning: --hybrid-ocr is deprecated. "
+ "Configure OCR settings on the hybrid server instead (--ocr-lang, --force-ocr).");
}
if (commandLine.hasOption(HYBRID_URL_LONG_OPTION)) {
String url = commandLine.getOptionValue(HYBRID_URL_LONG_OPTION);
if (url != null && !url.trim().isEmpty()) {
config.getHybridConfig().setUrl(url.trim());
}
}
if (commandLine.hasOption(HYBRID_TIMEOUT_LONG_OPTION)) {
String timeoutValue = commandLine.getOptionValue(HYBRID_TIMEOUT_LONG_OPTION);
if (timeoutValue != null && !timeoutValue.trim().isEmpty()) {
try {
int timeout = Integer.parseInt(timeoutValue.trim());
config.getHybridConfig().setTimeoutMs(timeout);
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
String.format("Invalid timeout value '%s'. Must be a non-negative integer.", timeoutValue));
}
}
}
if (commandLine.hasOption(HYBRID_FALLBACK_LONG_OPTION)) {
config.getHybridConfig().setFallbackToJava(true);
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
String normalized = value.trim().toLowerCase(Locale.ROOT);
if (!HybridConfig.REGIONLIST_TABLE_FIRST.equals(normalized)
&& !HybridConfig.REGIONLIST_LIST_ONLY.equals(normalized)) {
throw new IllegalArgumentException(String.format(
"Option --%s: unsupported value '%s'. Supported values: %s, %s",
HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION, normalized,
HybridConfig.REGIONLIST_TABLE_FIRST, HybridConfig.REGIONLIST_LIST_ONLY));
}
config.getHybridConfig().setRegionlistStrategy(normalized);
}
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
String normalized = value.trim().toLowerCase(Locale.ROOT);
if (!HybridConfig.OCR_OFF.equals(normalized)
&& !HybridConfig.OCR_AUTO.equals(normalized)
&& !HybridConfig.OCR_FORCE.equals(normalized)) {
throw new IllegalArgumentException(String.format(
"Option --%s: unsupported value '%s'. Supported values: %s, %s, %s",
HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION, normalized,
HybridConfig.OCR_OFF, HybridConfig.OCR_AUTO, HybridConfig.OCR_FORCE));
}
config.getHybridConfig().setOcrStrategy(normalized);
}
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
String normalized = value.trim().toLowerCase(Locale.ROOT);
if (!"memory".equals(normalized) && !"disk".equals(normalized)) {
throw new IllegalArgumentException(String.format(
"Option --%s: unsupported value '%s'. Supported values: memory, disk",
HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION, normalized));
}
config.getHybridConfig().setImageCache(normalized);
}
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION)) {
config.getHybridConfig().setSaveCrops(true);
}
if (commandLine.hasOption(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION)) {
String value = commandLine.getOptionValue(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION);
if (value != null && !value.trim().isEmpty()) {
config.getHybridConfig().setCropOutputDir(value.trim());
}
}
if (commandLine.hasOption(TO_STDOUT_LONG_OPTION)) {
config.setOutputStdout(true);
}
// Keep in sync with all HYBRID_HANCOM_AI_*_LONG_OPTION constants above.
boolean usesHancomAiOnly =
commandLine.hasOption(HYBRID_HANCOM_AI_REGIONLIST_STRATEGY_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_OCR_STRATEGY_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_IMAGE_CACHE_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_SAVE_CROPS_LONG_OPTION) ||
commandLine.hasOption(HYBRID_HANCOM_AI_CROP_OUTPUT_DIR_LONG_OPTION);
if (usesHancomAiOnly && !Config.HYBRID_HANCOM_AI.equals(config.getHybrid())) {
throw new IllegalArgumentException(
"Options --hybrid-hancom-ai-* require --hybrid=hancom-ai (got --hybrid="
+ config.getHybrid() + ")");
}
}
/**
* Exports CLI option definitions as JSON for code generation.
* This is used to generate Node.js, Python, and documentation from a single
* source of truth.
*
* @param out The output stream to write JSON to
*/
public static void exportOptionsAsJson(PrintStream out) {
List<OptionDefinition> exportable = OPTION_DEFINITIONS.stream()
.filter(d -> d.exported)
.collect(Collectors.toList());
// Build JSON manually to avoid external dependencies
StringBuilder json = new StringBuilder();
json.append("{\n");
json.append(" \"options\": [\n");
for (int i = 0; i < exportable.size(); i++) {
OptionDefinition opt = exportable.get(i);
json.append(" {\n");
json.append(" \"name\": \"").append(opt.longName).append("\",\n");
json.append(" \"shortName\": ").append(opt.shortName == null ? "null" : "\"" + opt.shortName + "\"")
.append(",\n");
json.append(" \"type\": \"").append(opt.type).append("\",\n");
json.append(" \"required\": false,\n");
if (opt.defaultValue == null) {
json.append(" \"default\": null,\n");
} else if (opt.defaultValue instanceof Boolean) {
json.append(" \"default\": ").append(opt.defaultValue).append(",\n");
} else {
json.append(" \"default\": \"").append(escapeJson(opt.defaultValue.toString())).append("\",\n");
}
json.append(" \"description\": \"").append(escapeJson(opt.description)).append("\"\n");
json.append(" }");
if (i < exportable.size() - 1) {
json.append(",");
}
json.append("\n");
}
json.append(" ]\n");
json.append("}\n");
out.print(json.toString());
}
private static String escapeJson(String value) {
if (value == null) {
return "";
}
return value
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\n", "\\n")
.replace("\r", "\\r")
.replace("\t", "\\t");
}
/**
* Internal class to hold option definition for both CLI and JSON export.
* Single source of truth for all option metadata.
*/
private static class OptionDefinition {
final String longName;
final String shortName;
final String type; // "string" | "boolean"
final Object defaultValue;
final String description;
final boolean exported; // Whether to include in JSON export
OptionDefinition(String longName, String shortName, String type, Object defaultValue, String description,
boolean exported) {
this.longName = longName;
this.shortName = shortName;
this.type = type;
this.defaultValue = defaultValue;
this.description = description;
this.exported = exported;
}
/** Creates an Apache Commons CLI Option from this definition. */
Option toOption() {
boolean hasArg = "string".equals(type);
return new Option(shortName, longName, hasArg, description);
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/autotagging/ChunksWriter.java
+323
View File
@@ -0,0 +1,323 @@
package org.opendataloader.pdf.autotagging;
import org.opendataloader.pdf.processors.AutoTaggingProcessor;
import org.verapdf.as.ASAtom;
import org.verapdf.as.io.ASInputStream;
import org.verapdf.cos.*;
import org.verapdf.gf.model.factory.chunks.ChunkParser;
import org.verapdf.gf.model.factory.chunks.GraphicsState;
import org.verapdf.gf.model.impl.sa.util.ResourceHandler;
import org.verapdf.operator.Operator;
import org.verapdf.parser.Operators;
import org.verapdf.parser.PDFStreamParser;
import org.verapdf.pd.PDContentStream;
import org.verapdf.pd.PDExtGState;
import org.verapdf.pd.images.PDXForm;
import org.verapdf.pd.images.PDXObject;
import org.verapdf.tools.StaticResources;
import org.verapdf.tools.TaggedPDFConstants;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.StreamInfo;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
public class ChunksWriter {
private static final java.util.logging.Logger CHUNKS_LOGGER = java.util.logging.Logger.getLogger(ChunksWriter.class.getName());
private final ResourceHandler resourceHandler;
private final GraphicsState graphicsState;
public ChunksWriter(GraphicsState inheritedGraphicState, ResourceHandler resourceHandler) {
this.graphicsState = inheritedGraphicState.clone();
this.resourceHandler = resourceHandler;
}
public static List<Object> getTokens(PDContentStream pdContentStream) {
if (pdContentStream != null) {
try {
COSObject contentStream = pdContentStream.getContents();
if (contentStream.getType() == COSObjType.COS_STREAM || contentStream.getType() == COSObjType.COS_ARRAY) {
try (ASInputStream opStream = contentStream.getDirectBase().getData(COSStream.FilterFlags.DECODE)) {
try (PDFStreamParser streamParser = new PDFStreamParser(opStream)) {
streamParser.parseTokens();
return streamParser.getTokens();
}
}
}
} catch (IOException e) {
CHUNKS_LOGGER.warning("Failed to read content stream tokens: " + e.getMessage());
}
}
return Collections.emptyList();
}
public List<Object> processTokens(List<Object> processTokens, OperatorStreamKey operatorStreamKey) throws IOException {
Map<Integer, Set<StreamInfo>> operatorIndexesToStreamInfosMap = AutoTaggingProcessor.getOperatorIndexesToStreamInfosMap().get(operatorStreamKey);
if (operatorIndexesToStreamInfosMap == null) {
operatorIndexesToStreamInfosMap = Collections.emptyMap();
}
List<Object> result = new ArrayList<>();
List<COSBase> arguments = new ArrayList<>();
for (int index = 0; index < processTokens.size(); index++) {
Object token = processTokens.get(index);
if (token instanceof COSBase) {
arguments.add((COSBase) token);
} else if (token instanceof Operator) {
processOperator(result, (Operator)token, arguments, index, operatorIndexesToStreamInfosMap, operatorStreamKey);
arguments.clear();
}
}
return result;
}
private void processOperator(List<Object> result, Operator rawOperator, List<COSBase> arguments, int operatorIndex,
Map<Integer, Set<StreamInfo>> operatorIndexesToStreamInfosMap,
OperatorStreamKey operatorStreamKey) throws IOException {
String operatorName = rawOperator.getOperator();
switch (operatorName) {
case Operators.BMC:
case Operators.EMC:
case Operators.BDC:
break;
case Operators.DO:
OperatorStreamKey xObjectOperatorStreamKey = new OperatorStreamKey(operatorStreamKey.getPageNumber(), arguments.get(0).getString());
Integer xObjectStructParent = AutoTaggingProcessor.getStructParentsIntegers().get(xObjectOperatorStreamKey);
if (AutoTaggingProcessor.getOperatorIndexesToStreamInfosMap().containsKey(xObjectOperatorStreamKey)
&& xObjectStructParent != null) {
COSName xObjectName = getLastCOSName(arguments);
PDXObject pdxObject = resourceHandler.getXObject(xObjectName);
if (pdxObject == null) {
processContentOperator(result, rawOperator, arguments, operatorIndex, operatorIndexesToStreamInfosMap, operatorName, operatorStreamKey);
break;
}
pdxObject.setKey(ASAtom.STRUCT_PARENTS,
COSInteger.construct(xObjectStructParent));
StaticResources.getDocument().getDocument().addChangedObject(pdxObject.getObject());
PDXForm pdxForm = (PDXForm)pdxObject;
GraphicsState xFormGraphicsState = graphicsState.clone();
AutoTaggingProcessor.setUpContents(pdxForm.getObject(), new ChunksWriter(xFormGraphicsState,
resourceHandler.getExtendedResources(pdxForm.getResources())).processTokens(
ChunksWriter.getTokens(pdxForm), xObjectOperatorStreamKey));
// Preserve the Do operator in the parent stream so the XObject is still invoked
result.addAll(arguments);
result.add(rawOperator);
} else {
processContentOperator(result, rawOperator, arguments, operatorIndex, operatorIndexesToStreamInfosMap, operatorName, operatorStreamKey);
}
break;
case Operators.TJ_SHOW:
case Operators.TJ_SHOW_POS:
case Operators.QUOTE:
case Operators.DOUBLE_QUOTE:
case Operators.BI:
case Operators.F_FILL:
case Operators.F_FILL_OBSOLETE:
case Operators.F_STAR_FILL:
case Operators.B_CLOSEPATH_FILL_STROKE:
case Operators.B_STAR_CLOSEPATH_EOFILL_STROKE:
case Operators.S_CLOSE_STROKE:
case Operators.S_STROKE:
processContentOperator(result, rawOperator, arguments, operatorIndex, operatorIndexesToStreamInfosMap, operatorName, operatorStreamKey);
break;
case org.verapdf.model.tools.constants.Operators.GS:
PDExtGState extGState = this.resourceHandler.getExtGState(getLastCOSName(arguments));
this.graphicsState.copyPropertiesFromExtGState(extGState);
result.addAll(arguments);
result.add(rawOperator);
break;
case org.verapdf.model.tools.constants.Operators.TF:
this.graphicsState.getTextState().setTextFont(resourceHandler.getFont(getFirstCOSName(arguments)));
result.addAll(arguments);
result.add(rawOperator);
break;
default:
// if (!Operators.D_SET_DASH.equals(operatorName)) {
result.addAll(arguments);
result.add(rawOperator);
// }
break;
}
}
private void processContentOperator(List<Object> result, Operator rawOperator, List<COSBase> arguments, int operatorIndex,
Map<Integer, Set<StreamInfo>> operatorIndexesToStreamInfosMap, String operatorName,
OperatorStreamKey operatorStreamKey) {
Set<StreamInfo> streamInfos = operatorIndexesToStreamInfosMap.get(operatorIndex);
if (streamInfos == null || streamInfos.isEmpty()) {
writeMarkedContent(result, arguments, rawOperator, operatorName, null, null);
} else {
if (streamInfos.size() == 1) {
Integer mcid = streamInfos.iterator().next().getMcid();
writeMarkedContent(result, arguments, rawOperator, operatorName, mcid, operatorStreamKey);
} else {
List<StreamInfo> streamInfosList = updateStreamInfos(streamInfos);
Map<StreamInfo, COSObject> newArguments = getArguments(arguments.get(arguments.size() - 1), streamInfosList);
for (Map.Entry<StreamInfo, COSObject> entry : newArguments.entrySet()) {
arguments.set(arguments.size() - 1, entry.getValue().get());
writeMarkedContent(result, arguments, rawOperator, operatorName, entry.getKey().getMcid(), operatorStreamKey);
}
}
}
}
private static String getStructureType(Integer mcid, OperatorStreamKey operatorStreamKey) {
if (mcid == null || operatorStreamKey == null) return null;
List<COSObject> parents = AutoTaggingProcessor.getStructParents().get(operatorStreamKey);
if (parents == null) {
CHUNKS_LOGGER.warning("structParents: no entry for key page=" + operatorStreamKey.getPageNumber() + " xobj=" + operatorStreamKey.getXObjectName() + " (available keys: " + AutoTaggingProcessor.getStructParents().keySet().stream().map(k -> "p"+k.getPageNumber()+"x"+k.getXObjectName()).collect(java.util.stream.Collectors.joining(",")) + ")");
return null;
}
if (mcid >= parents.size()) {
CHUNKS_LOGGER.warning("structParents: mcid=" + mcid + " out of range (size=" + parents.size() + ") for key page=" + operatorStreamKey.getPageNumber() + " xobj=" + operatorStreamKey.getXObjectName());
return null;
}
COSObject structElem = parents.get(mcid);
if (structElem == null) return null;
COSObject typeObj = structElem.getKey(ASAtom.S);
if (typeObj == null || typeObj.empty()) return null;
return typeObj.getString();
}
private static void writeMarkedContent(List<Object> result, List<COSBase> arguments, Operator token,
String operatorName, Integer mcid, OperatorStreamKey operatorStreamKey) {
if (mcid == null) {
result.add(COSName.construct(TaggedPDFConstants.ARTIFACT).getDirectBase());
result.add(Operator.getOperator(Operators.BMC));
} else {
String tagName;
if (Operators.BI.equals(operatorName) || Operators.DO.equals(operatorName)) {
tagName = TaggedPDFConstants.FIGURE;
} else {
String structType = getStructureType(mcid, operatorStreamKey);
tagName = (structType != null) ? structType : TaggedPDFConstants.SPAN;
}
result.add(COSName.construct(tagName).getDirectBase());
COSObject dictionary = COSDictionary.construct();
dictionary.setKey(ASAtom.MCID, COSInteger.construct(mcid));
result.add(dictionary.getDirectBase());
result.add(Operator.getOperator(Operators.BDC));
}
result.addAll(arguments);
result.add(token);
result.add(Operator.getOperator(Operators.EMC));
}
private Map<StreamInfo, COSObject> getArguments(COSBase object, List<StreamInfo> streamInfos) {
Map<StreamInfo, COSObject> map = new TreeMap<>();
if (object.getType() == COSObjType.COS_STRING) {
Queue<StreamInfo> streamInfoQueue = new LinkedList<>(streamInfos);
processString((COSString)object, map, streamInfoQueue, null, 0);
} else if (object.getType() == COSObjType.COS_ARRAY) {
int stringIndex = 0;
List<COSObject> array = new ArrayList<>();
Queue<StreamInfo> streamInfoQueue = new LinkedList<>(streamInfos);
for (COSObject element : (COSArray)object.getDirectBase()) {
if (element.getType() == COSObjType.COS_STRING) {
stringIndex = processString((COSString)element.get(), map, streamInfoQueue, array, stringIndex);
} else if (element.getType().isNumber()) {
array.add(element);
}
}
if (!array.isEmpty()) {
StreamInfo peekInfo = streamInfoQueue.peek();
COSObject target = peekInfo != null ? map.get(peekInfo) : null;
if (target != null && target.getType() == COSObjType.COS_ARRAY) {
COSArray currentArray = (COSArray) target.get();
for (COSObject element : array) {
currentArray.add(element);
}
} else {
CHUNKS_LOGGER.warning("Issue during text operator processing");
}
}
}
return map;
}
public int processString(COSString string, Map<StreamInfo, COSObject> map, Queue<StreamInfo> streamInfoQueue,
List<COSObject> array, int stringIndex) {
int currentStringIndex = stringIndex;
int currentBytesIndex = 0;
int dif = 0;
byte[] bytes = string.get();
try (InputStream inputStream = new ByteArrayInputStream(bytes)) {
int available = inputStream.available();
while (inputStream.available() > 0) {
int code = graphicsState.getTextState().getTextFont().readCode(inputStream);
String value = graphicsState.getTextState().getTextFont().toUnicode(code);
if (value == null) {
value = StaticContainers.getIsIgnoreCharactersWithoutUnicode() ? "" : ChunkParser.REPLACEMENT_CHARACTER_STRING;
}
int newAvailable = inputStream.available();
dif += available - newAvailable;
available = newAvailable;
int length = streamInfoQueue.peek().getEndIndex() - currentStringIndex;
currentStringIndex += value.length();
if (length <= value.length()) {
COSObject cosString = COSString.construct(Arrays.copyOfRange(bytes, currentBytesIndex, currentBytesIndex + dif), string.isHexadecimal());
if (array != null) {
array.add(cosString);
map.put(streamInfoQueue.peek(), new COSObject(new COSArray(array)));
} else {
map.put(streamInfoQueue.peek(), cosString);
}
currentBytesIndex += dif;
dif = 0;
if (array != null) {
array.clear();
}
if (streamInfoQueue.size() > 1) {
streamInfoQueue.poll();
}
}
}
} catch (IOException e) {
CHUNKS_LOGGER.warning("Failed to process font string: " + e.getMessage());
}
if (array != null && dif > 0) {
array.add(COSString.construct(Arrays.copyOfRange(bytes, currentBytesIndex, currentBytesIndex + dif), string.isHexadecimal()));
}
return currentStringIndex;
}
private static List<StreamInfo> updateStreamInfos(Set<StreamInfo> streamInfos) {
Iterator<StreamInfo> streamInfoIterator = streamInfos.iterator();
List<StreamInfo> newStreamInfos = new ArrayList<>();
StreamInfo streamInfo = null;
int currentIndex = 0;
while (streamInfoIterator.hasNext()) {
streamInfo = streamInfoIterator.next();
if (currentIndex < streamInfo.getStartIndex()) {
newStreamInfos.add(new StreamInfo(streamInfo.getOperatorIndex(), streamInfo.getXObjectName(),
streamInfo.getXImageObjectKey(), currentIndex, streamInfo.getStartIndex(), streamInfo.getLength(), null));
}
currentIndex = streamInfo.getEndIndex();
newStreamInfos.add(streamInfo);
}
if (currentIndex < streamInfo.getLength()) {
newStreamInfos.add(new StreamInfo(streamInfo.getOperatorIndex(), streamInfo.getXObjectName(),
streamInfo.getXImageObjectKey(), streamInfo.getEndIndex(), streamInfo.getLength(), streamInfo.getLength(), null));
}
return newStreamInfos;
}
private static COSName getFirstCOSName(List<COSBase> arguments) {
COSBase lastElement = arguments.isEmpty() ? null : arguments.get(0);
if (lastElement instanceof COSName) {
return (COSName) lastElement;
}
return null;
}
private static COSName getLastCOSName(List<COSBase> arguments) {
COSBase lastElement = arguments.isEmpty() ? null : arguments.get(arguments.size() - 1);
if (lastElement instanceof COSName) {
return (COSName) lastElement;
}
return null;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/autotagging/OperatorStreamKey.java
+39
View File
@@ -0,0 +1,39 @@
package org.opendataloader.pdf.autotagging;
import java.util.Objects;
public class OperatorStreamKey {
private final int pageNumber;
private final String xObjectName;
public OperatorStreamKey(int pageNumber, String xObjectName) {
this.pageNumber = pageNumber;
this.xObjectName = xObjectName;
}
public int getPageNumber() {
return pageNumber;
}
public String getXObjectName() {
return xObjectName;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
OperatorStreamKey that = (OperatorStreamKey) o;
return pageNumber == that.pageNumber &&
Objects.equals(xObjectName, that.xObjectName);
}
@Override
public int hashCode() {
return Objects.hash(pageNumber, xObjectName);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/containers/StaticLayoutContainers.java
+164
View File
@@ -0,0 +1,164 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.containers;
import org.opendataloader.pdf.api.Config;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import java.io.File;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.logging.Logger;
public class StaticLayoutContainers {
protected static final Logger LOGGER = Logger.getLogger(StaticLayoutContainers.class.getCanonicalName());
private static final ThreadLocal<Long> currentContentId = new ThreadLocal<>();
private static final ThreadLocal<List<SemanticHeading>> headings = new ThreadLocal<>();
private static final ThreadLocal<Integer> imageIndex = new ThreadLocal<>();
private static final ThreadLocal<Boolean> isUseStructTree = new ThreadLocal<>();
private static final ThreadLocal<String> imagesDirectory = new ThreadLocal<>();
private static final ThreadLocal<Boolean> embedImages = new ThreadLocal<>();
private static final ThreadLocal<String> imageFormat = new ThreadLocal<>();
private static final ThreadLocal<Map<Integer, Double>> replacementCharRatios = ThreadLocal.withInitial(ConcurrentHashMap::new);
private static final ThreadLocal<Map<String, byte[]>> embeddedImageBytes = ThreadLocal.withInitial(ConcurrentHashMap::new);
public static void clearContainers() {
currentContentId.set(1L);
headings.set(Collections.synchronizedList(new LinkedList<>()));
imageIndex.set(1);
isUseStructTree.set(false);
imagesDirectory.set("");
embedImages.set(false);
imageFormat.set(Config.IMAGE_FORMAT_PNG);
replacementCharRatios.get().clear();
embeddedImageBytes.get().clear();
}
public static long getCurrentContentId() {
return currentContentId.get();
}
public static long incrementContentId() {
long id = getCurrentContentId();
StaticLayoutContainers.setCurrentContentId(id + 1);
return id;
}
public static void setCurrentContentId(long currentContentId) {
StaticLayoutContainers.currentContentId.set(currentContentId);
}
public static String getImagesDirectory() {
return imagesDirectory.get();
}
public static String getImagesDirectoryName() {
String dir = imagesDirectory.get();
return dir != null && !dir.isEmpty() ? new File(dir).getName() : "";
}
public static void setImagesDirectory(String imagesDirectory) {
StaticLayoutContainers.imagesDirectory.set(imagesDirectory);
}
public static List<SemanticHeading> getHeadings() {
return headings.get();
}
public static void setHeadings(List<SemanticHeading> headings) {
StaticLayoutContainers.headings.set(headings);
}
public static Boolean isUseStructTree() {
return isUseStructTree.get();
}
public static void setIsUseStructTree(Boolean isUseStructTree) {
StaticLayoutContainers.isUseStructTree.set(isUseStructTree);
}
public static int incrementImageIndex() {
int imageIndex = StaticLayoutContainers.imageIndex.get();
StaticLayoutContainers.imageIndex.set(imageIndex + 1);
return imageIndex;
}
public static void resetImageIndex() {
StaticLayoutContainers.imageIndex.set(1);
}
public static boolean isEmbedImages() {
return Boolean.TRUE.equals(embedImages.get());
}
public static void setEmbedImages(boolean embedImages) {
StaticLayoutContainers.embedImages.set(embedImages);
}
public static String getImageFormat() {
String format = imageFormat.get();
return format != null ? format : Config.IMAGE_FORMAT_PNG;
}
public static void setImageFormat(String format) {
StaticLayoutContainers.imageFormat.set(format);
}
public static void setReplacementCharRatio(int pageNumber, double ratio) {
replacementCharRatios.get().put(pageNumber, ratio);
}
public static double getReplacementCharRatio(int pageNumber) {
return replacementCharRatios.get().getOrDefault(pageNumber, 0.0);
}
public static void cacheEmbeddedImageBytes(String absolutePath, byte[] bytes) {
embeddedImageBytes.get().put(normalizeImageKey(absolutePath), bytes);
}
public static byte[] getEmbeddedImageBytes(String absolutePath) {
return embeddedImageBytes.get().get(normalizeImageKey(absolutePath));
}
public static boolean hasEmbeddedImageBytes(String absolutePath) {
return embeddedImageBytes.get().containsKey(normalizeImageKey(absolutePath));
}
// Map-level accessors are used by DocumentProcessor.propagateState so worker threads
// share the main thread's cache instance. Today the cache is only touched on the main
// thread, but propagating it eliminates a silent-data-loss trap if generators ever
// run on workers — matches the CLAUDE.md ThreadLocal-propagation gotcha.
public static Map<String, byte[]> getEmbeddedImageBytesMap() {
return embeddedImageBytes.get();
}
public static void setEmbeddedImageBytesMap(Map<String, byte[]> map) {
embeddedImageBytes.set(map);
}
// Image paths flow through String.format with File.separator (cache write side) and
// through File.getPath() (cache read side via Base64ImageUtils.toDataUri). The two
// forms can differ on Windows when the user passes forward-slash output paths
// (e.g. /tmp/x). Collapse both forms to a single canonical key so writers and readers
// agree on cache identity.
private static String normalizeImageKey(String path) {
return path == null ? null : new File(path).getPath();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/EnrichedImageChunk.java
+106
View File
@@ -0,0 +1,106 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
/**
* An ImageChunk enriched with an AI-generated description (alt text).
*
* <p>Created when the hybrid backend returns a SemanticPicture whose bounding
* box overlaps a Java-extracted ImageChunk. The description is matched by
* bounding-box IoU in HybridDocumentProcessor and propagated to:
* <ul>
* <li>AutoTaggingProcessor — inserts /Alt into the Figure struct element</li>
* <li>ImageSerializer — writes "alt" field to JSON output</li>
* <li>MarkdownGenerator / HtmlGenerator — uses description as alt text</li>
* </ul>
*/
public class EnrichedImageChunk extends ImageChunk {
/**
* Where the alt text came from. Drives the {@code alt_source} JSON field
* and downstream evidence-report badging — a reviewer treats a human-authored
* /Alt very differently from an AI-generated caption.
*/
public enum AltSource {
/** Pulled from the source PDF's struct-tree /Alt entry. */
ORIGINAL,
/** Generated by the hybrid backend (Hancom AI / docling SmolVLM). */
AI_GENERATED
}
private final String description;
private final AltSource altSource;
/**
* Convenience constructor for author-authored /Alt sourced from the input
* PDF's struct tree (used by {@link org.opendataloader.pdf.processors.TaggedDocumentProcessor}).
* AI / backend-generated captions must use the 3-arg form with
* {@link AltSource#AI_GENERATED} so downstream consumers can distinguish
* human alt text from synthesised descriptions.
*/
public EnrichedImageChunk(ImageChunk source, String description) {
this(source, description, AltSource.ORIGINAL);
}
public EnrichedImageChunk(ImageChunk source, String description, AltSource altSource) {
super(source.getBoundingBox());
// Copy index so serializers can reference the image file
setIndex(source.getIndex());
// Copy stream infos so MCID / struct-tree linkage is preserved
getStreamInfos().addAll(source.getStreamInfos());
this.description = description;
this.altSource = altSource;
}
public String getDescription() {
return description != null ? description : "";
}
public boolean hasDescription() {
return description != null && !description.isEmpty();
}
public AltSource getAltSource() {
return altSource;
}
/**
* Sanitized description safe for use as PDF /Alt, Markdown alt text,
* HTML alt attribute, and JSON value.
*/
public String sanitizeDescription() {
if (!hasDescription()) return "";
return description
.replace("\r\n", " ")
.replace("\n", " ")
.replace("\r", " ")
.replace("\"", "")
.replace("[", "")
.replace("]", "")
.replace("<", "")
.replace(">", "")
.replace("&", "")
.replace("\u0000", "")
// PDF/UA-2 clause 8.4.3.3 forbids Private Use Area code points in /Alt.
// Strip BMP PUA (U+E000–U+F8FF) and supplementary PUA (encoded as surrogate pairs
// U+DB80–U+DBFF paired with U+DC00–U+DFFF).
.replaceAll("[\\uE000-\\uF8FF]|[\\uDB80-\\uDBFF][\\uDC00-\\uDFFF]", "")
.replaceAll("\\s{2,}", " ")
.trim();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/SemanticFootnote.java
+34
View File
@@ -0,0 +1,34 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
/**
* Represents a footnote/endnote element.
* Maps to PDF 2.0 FENote structure element in AutoTaggingProcessor.
*/
public class SemanticFootnote extends SemanticParagraph {
public SemanticFootnote() {
super();
}
public SemanticFootnote(SemanticParagraph node) {
super(node);
this.setSemanticType(SemanticType.NOTE);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/SemanticFormula.java
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.BaseObject;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
/**
* Represents a mathematical formula element with LaTeX content.
*
* <p>This class stores formula content in LaTeX format, which can be rendered
* using MathJax, KaTeX, or similar libraries in the output formats.
*
* <p>Extends BaseObject to leverage the standard IObject implementation.
*/
public class SemanticFormula extends BaseObject {
private final String latex;
/**
* Creates a SemanticFormula with the given bounding box and LaTeX content.
*
* @param boundingBox The bounding box of the formula
* @param latex The LaTeX representation of the formula
*/
public SemanticFormula(BoundingBox boundingBox, String latex) {
super(boundingBox);
this.latex = latex;
}
/**
* Gets the LaTeX representation of the formula.
*
* @return The LaTeX string, or empty string if null
*/
public String getLatex() {
return latex != null ? latex : "";
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/entities/SemanticPicture.java
+120
View File
@@ -0,0 +1,120 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.entities;
import org.verapdf.wcag.algorithms.entities.BaseObject;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
/**
* Represents a picture element with optional description (alt text).
*
* <p>This class stores picture metadata including AI-generated descriptions
* for accessibility purposes. Descriptions are generated using vision-language
* models when the hybrid server is configured with --enrich-picture-description.
*
* <p>Extends BaseObject to leverage the standard IObject implementation.
*/
public class SemanticPicture extends BaseObject {
private final int index;
private final String description;
/**
* Creates a SemanticPicture with the given bounding box and index.
*
* @param boundingBox The bounding box of the picture
* @param index The sequential index of the picture
*/
public SemanticPicture(BoundingBox boundingBox, int index) {
this(boundingBox, index, null);
}
/**
* Creates a SemanticPicture with the given bounding box, index, and description.
*
* @param boundingBox The bounding box of the picture
* @param index The sequential index of the picture
* @param description The AI-generated description (alt text) for accessibility
*/
public SemanticPicture(BoundingBox boundingBox, int index, String description) {
super(boundingBox);
this.index = index;
this.description = description;
}
/**
* Gets the sequential index of this picture.
*
* @return The picture index
*/
public int getPictureIndex() {
return index;
}
/**
* Gets the description (alt text) of this picture.
*
* @return The description string, or empty string if null
*/
public String getDescription() {
return description != null ? description : "";
}
/**
* Checks if this picture has a description.
*
* @return true if description is non-null and non-empty
*/
public boolean hasDescription() {
return description != null && !description.isEmpty();
}
/**
* Returns a sanitized version of the description safe for use as alt text
* across all output formats (Markdown, HTML, JSON) without format-specific escaping.
*
* <p>Removes characters that are structurally significant in at least one output format:
* <ul>
* <li>{@code "} — HTML attribute delimiter</li>
* <li>{@code [}, {@code ]} — Markdown alt text delimiters</li>
* <li>{@code <}, {@code >} — HTML tag delimiters</li>
* <li>{@code &} — HTML entity prefix</li>
* <li>{@code \u0000} — null character</li>
* <li>Newlines ({@code \n}, {@code \r}) — replaced with a space</li>
* </ul>
* Consecutive whitespace is collapsed to a single space and the result is trimmed.
*
* @return sanitized description string, or empty string if no description
*/
public String sanitizeDescription() {
if (!hasDescription()) {
return "";
}
return description
.replace("\r\n", " ")
.replace("\n", " ")
.replace("\r", " ")
.replace("\"", "")
.replace("[", "")
.replace("]", "")
.replace("<", "")
.replace(">", "")
.replace("&", "")
.replace("\u0000", "")
.replaceAll("\\s{2,}", " ")
.trim();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/exceptions/EncryptedTaggedPdfNotSupportedException.java
+36
View File
@@ -0,0 +1,36 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.exceptions;
import java.io.IOException;
/**
* Thrown when tagged-pdf generation is requested for an encrypted document.
*
* <p>Encrypted PDFs (those with an {@code /Encrypt} dictionary in the trailer)
* are not supported for tagged-pdf output. Writing a tagged copy requires
* re-serializing the document, which interacts poorly with the document's
* encryption state and is also not permitted by the PDF specification for
* documents whose permissions deny modification.
*/
public class EncryptedTaggedPdfNotSupportedException extends IOException {
private static final long serialVersionUID = 1L;
public EncryptedTaggedPdfNotSupportedException(String message) {
super(message);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/exceptions/InvalidPdfFileException.java
+60
View File
@@ -0,0 +1,60 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.exceptions;
import java.io.IOException;
/**
* Thrown when an input file cannot be processed as a PDF. Two failure modes
* share this exception type, distinguished by message wording:
* <ul>
* <li><em>Missing header</em> — the {@code %PDF-} magic number is absent
* from the first 1024 bytes (JPEG renamed to {@code .pdf}, empty file,
* arbitrary text). Detected before veraPDF is invoked.</li>
* <li><em>Corrupted or truncated content</em> — the magic number is
* present but the body fails to parse (interrupted download, missing
* trailing xref, garbage payload). Detected when {@code new PDDocument}
* throws {@link IOException}; the original veraPDF exception is
* preserved via {@link #getCause()} for diagnostics.</li>
* </ul>
*
* <p>This is a checked subtype of {@link IOException} so callers that already
* handle {@code IOException} keep compiling, while callers that want to
* distinguish "not a usable PDF" from other I/O failures can catch this type
* specifically.
*
* <p>Public entry points that may surface this exception:
* <ul>
* <li>{@code OpenDataLoaderPDF.processFile(String, Config)}</li>
* <li>{@code DocumentProcessor.processFile(String, Config)}</li>
* <li>{@code DocumentProcessor.processFileWithResult(String, Config)}</li>
* <li>{@code DocumentProcessor.extractContents(String, Config)}</li>
* <li>{@code DocumentProcessor.preprocessing(String, Config)}</li>
* <li>{@code AutoTagger.tag(String, Config, Float)}</li>
* </ul>
*/
public class InvalidPdfFileException extends IOException {
private static final long serialVersionUID = 1L;
public InvalidPdfFileException(String message) {
super(message);
}
public InvalidPdfFileException(String message, Throwable cause) {
super(message, cause);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/html/HtmlGenerator.java
+550
View File
@@ -0,0 +1,550 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.html;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.EnrichedImageChunk;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.markdown.MarkdownSyntax;
import org.opendataloader.pdf.utils.Base64ImageUtils;
import org.opendataloader.pdf.utils.GeneratorUtils;
import org.opendataloader.pdf.utils.ImagesUtils;
import org.opendataloader.pdf.utils.OutputType;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.*;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.awt.Color;
import java.io.Closeable;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Generates HTML output from PDF document content.
* Converts semantic elements like paragraphs, headings, tables, and images into HTML format.
*/
public class HtmlGenerator implements Closeable {
/** Logger for this class. */
protected static final Logger LOGGER = Logger.getLogger(HtmlGenerator.class.getCanonicalName());
/** Writer for the HTML output file. */
protected final FileWriter htmlWriter;
/** Name of the input PDF file. */
protected final String pdfFileName;
/** Absolute path to the input PDF file. */
protected final Path pdfFilePath;
/** Name of the output HTML file. */
protected final String htmlFileName;
/** Absolute path to the output HTML file. */
protected final Path htmlFilePath;
/** Current table nesting level for tracking nested tables. */
protected int tableNesting = 0;
/** String to insert between pages in HTML output. */
protected String htmlPageSeparator = "";
/**
* Page numbers (1-based) selected by --pages; an empty set means all pages.
* Sourced from the raw {@link Config#getPageNumbers()} list (not the
* validated set built by {@code DocumentProcessor.getValidPageNumbers}).
* Safe to compare against {@code pageNumber + 1} because the surrounding
* loop is bounded by the document's actual page count, so out-of-range
* values from the raw list are never tested for membership.
*/
protected final Set<Integer> selectedPageNumbers;
/** Whether to embed images as Base64 data URIs. */
protected boolean embedImages = false;
/** Format for extracted images (png or jpeg). */
protected String imageFormat = Config.IMAGE_FORMAT_PNG;
/** Whether to include page headers and footers in output. */
protected boolean includeHeaderFooter = false;
/** Opening tag for strikethrough text*/
protected static final String strikethroughTextHtmlOpeningTag = "<del>";
/** Closing tag for strikethrough text*/
protected static final String strikethroughTextHtmlClosingTag = "</del>";;
/**
* Creates a new HtmlGenerator for the specified PDF file.
*
* @param inputPdf the input PDF file
* @param config the configuration settings
* @throws IOException if unable to create the output file
*/
public HtmlGenerator(File inputPdf, Config config) throws IOException {
this.pdfFileName = inputPdf.getName();
this.pdfFilePath = inputPdf.toPath().toAbsolutePath();
this.htmlFileName = pdfFileName.substring(0, pdfFileName.length() - 3) + "html";
this.htmlFilePath = Path.of(config.getOutputFolder(), htmlFileName);
this.htmlWriter = new FileWriter(htmlFilePath.toFile(), StandardCharsets.UTF_8);
this.htmlPageSeparator = escapeHtmlAttribute(config.getHtmlPageSeparator());
this.selectedPageNumbers = new HashSet<>(config.getPageNumbers());
this.embedImages = config.isEmbedImages();
this.imageFormat = config.getImageFormat();
this.includeHeaderFooter = config.isIncludeHeaderFooter();
}
/**
* Writes the document contents to HTML format.
*
* @param contents the document contents organized by page
*/
public void writeToHtml(List<List<IObject>> contents) {
try {
htmlWriter.write("<!DOCTYPE html>\n");
htmlWriter.write("<html lang=\"und\">\n<head>\n<meta charset=\"utf-8\">\n");
htmlWriter.write("<title>" + escapeHtmlText(pdfFileName) + "</title>\n");
htmlWriter.write("</head>\n<body>\n");
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
if (selectedPageNumbers.isEmpty() || selectedPageNumbers.contains(pageNumber + 1)) {
writePageSeparator(pageNumber);
}
for (IObject content : contents.get(pageNumber)) {
this.write(content);
}
}
htmlWriter.write("\n</body>\n</html>");
LOGGER.log(Level.INFO, "Created {0}", htmlFilePath);
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Unable to create html output: " + e.getMessage());
}
}
/**
* Writes a page separator to the HTML output if configured.
*
* @param pageNumber the current page number (0-indexed)
* @throws IOException if unable to write to the output
*/
protected void writePageSeparator(int pageNumber) throws IOException {
if (!htmlPageSeparator.isEmpty()) {
htmlWriter.write(htmlPageSeparator.contains(Config.PAGE_NUMBER_STRING)
? htmlPageSeparator.replace(Config.PAGE_NUMBER_STRING, String.valueOf(pageNumber + 1))
: htmlPageSeparator);
htmlWriter.write("\n");
}
}
/**
* Writes a single content object to the HTML output.
*
* @param object the content object to write
* @throws IOException if unable to write to the output
*/
protected void write(IObject object) throws IOException {
if (object instanceof SemanticHeaderOrFooter) {
if (includeHeaderFooter) {
writeHeaderOrFooter((SemanticHeaderOrFooter) object);
}
return;
} else if (object instanceof SemanticPicture) {
writePicture((SemanticPicture) object);
} else if (object instanceof ImageChunk) {
writeImage((ImageChunk) object);
} else if (object instanceof SemanticFormula) {
writeFormula((SemanticFormula) object);
} else if (object instanceof SemanticHeading) {
writeHeading((SemanticHeading) object);
} else if (object instanceof SemanticParagraph) {
writeParagraph((SemanticParagraph) object);
} else if (object instanceof SemanticTextNode) {
writeSemanticTextNode((SemanticTextNode) object);
} else if (object instanceof TableBorder) {
writeTable((TableBorder) object);
} else if (object instanceof PDFList) {
writeList((PDFList) object);
} else if (object instanceof SemanticTOC) {
writeTOC((SemanticTOC) object);
} else {
return;
}
if (!isInsideTable()) {
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
/**
* Writes a header or footer element to the HTML output.
*
* @param headerOrFooter the header or footer to write
* @throws IOException if unable to write to the output
*/
protected void writeHeaderOrFooter(SemanticHeaderOrFooter headerOrFooter) throws IOException {
for (IObject content : headerOrFooter.getContents()) {
write(content);
}
}
/**
* Writes a formula element to the HTML output using MathJax-compatible markup.
*
* @param formula the formula to write
* @throws IOException if unable to write to the output
*/
protected void writeFormula(SemanticFormula formula) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_MATH_DISPLAY_TAG);
htmlWriter.write("\\[");
htmlWriter.write(escapeHtmlText(formula.getLatex()));
htmlWriter.write("\\]");
htmlWriter.write(HtmlSyntax.HTML_MATH_DISPLAY_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes an image element to the HTML output.
*
* @param image the image chunk to write
*/
protected void writeImage(ImageChunk image) {
try {
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, image.getIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", image.getIndex(), imageFormat);
if (ImagesUtils.isImageFileExists(absolutePath)) {
String imageSource;
if (embedImages) {
File imageFile = new File(absolutePath);
imageSource = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (imageSource == null) {
LOGGER.log(Level.WARNING, "Failed to convert image to Base64: {0}", absolutePath);
}
} else {
imageSource = relativePath;
}
if (imageSource != null) {
String escapedSource = escapeHtmlAttribute(imageSource);
// Empty alt is correct HTML for "missing description": screen
// readers skip it, and our evidence-report flags it as
// alt_source="missing". Never synthesize "figureN".
String altText = (image instanceof EnrichedImageChunk && ((EnrichedImageChunk) image).hasDescription())
? ((EnrichedImageChunk) image).sanitizeDescription()
: "";
String imageString = String.format("<img src=\"%s\" alt=\"%s\">", escapedSource, escapeHtmlAttribute(altText));
htmlWriter.write(imageString);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to write image for html output: " + e.getMessage());
}
}
/**
* Writes a SemanticPicture element with figure/figcaption for description.
*
* @param picture the picture to write
*/
protected void writePicture(SemanticPicture picture) {
try {
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, picture.getPictureIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", picture.getPictureIndex(), imageFormat);
if (ImagesUtils.isImageFileExists(absolutePath)) {
String imageSource;
if (embedImages) {
File imageFile = new File(absolutePath);
imageSource = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (imageSource == null) {
LOGGER.log(Level.WARNING, "Failed to convert image to Base64: {0}", absolutePath);
}
} else {
imageSource = relativePath;
}
if (imageSource != null) {
String altText = picture.hasDescription()
? picture.sanitizeDescription()
: "";
String escapedSource = escapeHtmlAttribute(imageSource);
htmlWriter.write(HtmlSyntax.HTML_FIGURE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
String imageString = String.format("<img src=\"%s\" alt=\"%s\">", escapedSource, escapeHtmlAttribute(altText));
htmlWriter.write(imageString);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
htmlWriter.write(HtmlSyntax.HTML_FIGURE_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to write picture for html output: " + e.getMessage());
}
}
/**
* Writes a list element to the HTML output.
*
* @param list the PDF list to write
* @throws IOException if unable to write to the output
*/
protected void writeList(PDFList list) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (ListItem item : list.getListItems()) {
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_TAG);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_TAG);
String value = GeneratorUtils.getTextFromLines(item.getLines(), OutputType.HTML);
htmlWriter.write(value);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_CLOSE_TAG);
for (IObject object : item.getContents()) {
write(object);
}
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
protected void writeTOC(SemanticTOC toc) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (IObject item : toc.getTOCItems()) {
if (item instanceof SemanticTOC) {
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
writeTOC((SemanticTOC) item);
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_CLOSE_TAG);
} else if (item instanceof SemanticTOCI) {
SemanticTOCI tocItem = (SemanticTOCI) item;
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_TAG);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_TAG);
String value = GeneratorUtils.getTextFromLines(tocItem.getLines(), OutputType.HTML);
htmlWriter.write(value);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_CLOSE_TAG);
for (IObject object : tocItem.getContents()) {
write(object);
}
htmlWriter.write(HtmlSyntax.HTML_LIST_ITEM_CLOSE_TAG);
}
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
htmlWriter.write(HtmlSyntax.HTML_UNORDERED_LIST_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes a semantic text node as a figure caption to the HTML output.
*
* @param textNode the text node to write
* @throws IOException if unable to write to the output
*/
protected void writeSemanticTextNode(SemanticTextNode textNode) throws IOException {
htmlWriter.write(HtmlSyntax.HTML_FIGURE_CAPTION_TAG);
htmlWriter.write(GeneratorUtils.getTextFromTextNode(textNode, OutputType.HTML));
htmlWriter.write(HtmlSyntax.HTML_FIGURE_CAPTION_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes a table element to the HTML output.
*
* @param table the table border to write
* @throws IOException if unable to write to the output
*/
protected void writeTable(TableBorder table) throws IOException {
enterTable();
htmlWriter.write(HtmlSyntax.HTML_TABLE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (int rowNumber = 0; rowNumber < table.getNumberOfRows(); rowNumber++) {
TableBorderRow row = table.getRow(rowNumber);
htmlWriter.write(HtmlSyntax.HTML_TABLE_ROW_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
for (int colNumber = 0; colNumber < table.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = row.getCell(colNumber);
if (cell.getRowNumber() == rowNumber && cell.getColNumber() == colNumber) {
writeCellTag(cell);
List<IObject> contents = cell.getContents();
if (!contents.isEmpty()) {
for (IObject contentItem : contents) {
this.write(contentItem);
}
}
if (cell.isHeaderCell()) {
htmlWriter.write(HtmlSyntax.HTML_TABLE_HEADER_CLOSE_TAG);
} else {
htmlWriter.write(HtmlSyntax.HTML_TABLE_CELL_CLOSE_TAG);
}
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
}
htmlWriter.write(HtmlSyntax.HTML_TABLE_ROW_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
htmlWriter.write(HtmlSyntax.HTML_TABLE_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
leaveTable();
}
/**
* Writes a paragraph element to the HTML output.
*
* @param paragraph the semantic paragraph to write
* @throws IOException if unable to write to the output
*/
protected void writeParagraph(SemanticParagraph paragraph) throws IOException {
double paragraphIndent = paragraph.getColumns().get(0).getBlocks().get(0).getFirstLineIndent();
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_TAG);
if (paragraphIndent > 0) {
htmlWriter.write(HtmlSyntax.HTML_INDENT);
}
String paragraphValue = GeneratorUtils.getTextFromTextNode(paragraph, OutputType.HTML);
if (isInsideTable() && StaticContainers.isKeepLineBreaks()) {
paragraphValue = paragraphValue.replace(HtmlSyntax.HTML_LINE_BREAK, HtmlSyntax.HTML_LINE_BREAK_TAG);
}
htmlWriter.write(paragraphValue);
htmlWriter.write(HtmlSyntax.HTML_PARAGRAPH_CLOSE_TAG);
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
/**
* Writes a heading element to the HTML output.
*
* @param heading the semantic heading to write
* @throws IOException if unable to write to the output
*/
protected void writeHeading(SemanticHeading heading) throws IOException {
int headingLevel = Math.min(6, Math.max(1, heading.getHeadingLevel()));
htmlWriter.write("<h" + headingLevel + ">");
htmlWriter.write(GeneratorUtils.getTextFromTextNode(heading, OutputType.HTML));
htmlWriter.write("</h" + headingLevel + ">");
htmlWriter.write(HtmlSyntax.HTML_LINE_BREAK);
}
private void writeCellTag(TableBorderCell cell) throws IOException {
String tag = cell.isHeaderCell() ? "<th" : "<td";
StringBuilder cellTag = new StringBuilder(tag);
int colSpan = cell.getColSpan();
if (colSpan != 1) {
cellTag.append(" colspan=\"").append(colSpan).append("\"");
}
int rowSpan = cell.getRowSpan();
if (rowSpan != 1) {
cellTag.append(" rowspan=\"").append(rowSpan).append("\"");
}
cellTag.append(">");
htmlWriter.write(cellTag.toString());
}
/**
* Increments the table nesting level when entering a table.
*/
protected void enterTable() {
tableNesting++;
}
/**
* Decrements the table nesting level when leaving a table.
*/
protected void leaveTable() {
if (tableNesting > 0) {
tableNesting--;
}
}
/**
* Checks whether currently writing inside a table.
*
* @return true if inside a table, false otherwise
*/
protected boolean isInsideTable() {
return tableNesting > 0;
}
/**
* Escapes special characters for use in HTML attributes.
* Handles quotes, ampersands, less-than, greater-than, and newlines.
*
* @param value the string to escape
* @return the escaped string safe for HTML attribute values
*/
protected static String escapeHtmlAttribute(String value) {
if (value == null) {
return "";
}
value = escapeHtmlText(value);
return value
.replace("\"", "&quot;")
.replace("\n", " ")
.replace("\r", "");
}
protected static String escapeHtmlText(String value) {
if (value == null) {
return "";
}
return value
.replace("\u0000", "")
.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;");
}
public static void getTextFromLineForHTML(TextLine line, StringBuilder stringBuilder) {
for (TextChunk chunk : line.getTextChunks()) {
String style = getTextStyle(chunk);
if (!style.isEmpty()) {
String styleAttribute = String.format(HtmlSyntax.HTML_STYLE_ATTRIBUTE, style.trim());
stringBuilder.append(String.format(HtmlSyntax.HTML_SPAN_START_TAG, styleAttribute));
stringBuilder.append(escapeHtmlText(chunk.getValue()));
stringBuilder.append(HtmlSyntax.HTML_SPAN_CLOSE_TAG);
} else {
stringBuilder.append(escapeHtmlText(chunk.getValue()));
}
}
}
private static String getTextStyle(TextChunk chunk) {
StringBuilder style = new StringBuilder();
if (chunk.getIsStrikethroughText() || chunk.getIsUnderlinedText()) {
style.append(String.format(HtmlSyntax.HTML_TEXT_DECORATION_STYLE_PROPERTY,
(chunk.getIsStrikethroughText() ? HtmlSyntax.HTML_STRIKETHROUGH_VALUE : "") +
(chunk.getIsUnderlinedText() ? HtmlSyntax.HTML_UNDERLINE_VALUE : "")));
}
return style.toString();
}
@Override
public void close() throws IOException {
if (htmlWriter != null) {
htmlWriter.close();
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/html/HtmlGeneratorFactory.java
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.html;
import org.opendataloader.pdf.api.Config;
import java.io.File;
import java.io.IOException;
/**
* Factory class for creating HtmlGenerator instances.
*/
public class HtmlGeneratorFactory {
/**
* Creates a new HtmlGenerator for the specified PDF file.
*
* @param inputPdf the input PDF file
* @param config the configuration settings
* @return a new HtmlGenerator instance
* @throws IOException if unable to create the generator
*/
public static HtmlGenerator getHtmlGenerator(File inputPdf, Config config) throws IOException {
return new HtmlGenerator(inputPdf, config);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/html/HtmlSyntax.java
+86
View File
@@ -0,0 +1,86 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.html;
/**
* Constants for HTML syntax elements used in HTML output generation.
*/
public class HtmlSyntax {
/** Format string for image file names. */
public static final String IMAGE_FILE_NAME_FORMAT = "figure%d.png";
/** Line break character for HTML output. */
public static final String HTML_LINE_BREAK = "\n";
/** Opening table tag with border. */
public static final String HTML_TABLE_TAG = "<table border=\"1\">";
/** Closing table tag. */
public static final String HTML_TABLE_CLOSE_TAG = "</table>";
/** Opening table row tag. */
public static final String HTML_TABLE_ROW_TAG = "<tr>";
/** Closing table row tag. */
public static final String HTML_TABLE_ROW_CLOSE_TAG = "</tr>";
/** Opening table cell tag. */
public static final String HTML_TABLE_CELL_TAG = "<td>";
/** Closing table cell tag. */
public static final String HTML_TABLE_CELL_CLOSE_TAG = "</td>";
/** Opening table header cell tag. */
public static final String HTML_TABLE_HEADER_TAG = "<th>";
/** Closing table header cell tag. */
public static final String HTML_TABLE_HEADER_CLOSE_TAG = "</th>";
/** Opening ordered list tag. */
public static final String HTML_ORDERED_LIST_TAG = "<ol>";
/** Closing ordered list tag. */
public static final String HTML_ORDERED_LIST_CLOSE_TAG = "</ol>";
/** Opening unordered list tag. */
public static final String HTML_UNORDERED_LIST_TAG = "<ul>";
/** Closing unordered list tag. */
public static final String HTML_UNORDERED_LIST_CLOSE_TAG = "</ul>";
/** Opening list item tag. */
public static final String HTML_LIST_ITEM_TAG = "<li>";
/** Closing list item tag. */
public static final String HTML_LIST_ITEM_CLOSE_TAG = "</li>";
/** HTML line break tag. */
public static final String HTML_LINE_BREAK_TAG = "<br>";
/** Indentation string for paragraphs. */
public static final String HTML_INDENT = "";
/** Opening paragraph tag. */
public static final String HTML_PARAGRAPH_TAG = "<p>";
/** Closing paragraph tag. */
public static final String HTML_PARAGRAPH_CLOSE_TAG = "</p>";
/** Opening figure tag. */
public static final String HTML_FIGURE_TAG = "<figure>";
/** Closing figure tag. */
public static final String HTML_FIGURE_CLOSE_TAG = "</figure>";
/** Opening figure caption tag. */
public static final String HTML_FIGURE_CAPTION_TAG = "<figcaption>";
/** Closing figure caption tag. */
public static final String HTML_FIGURE_CAPTION_CLOSE_TAG = "</figcaption>";
/** Opening math display block tag for MathJax/KaTeX rendering. */
public static final String HTML_MATH_DISPLAY_TAG = "<div class=\"math-display\">";
/** Closing math display block tag. */
public static final String HTML_MATH_DISPLAY_CLOSE_TAG = "</div>";
/** Span opening tag. */
public static final String HTML_SPAN_START_TAG = "<span%s>";
/** Style attribute */
public static final String HTML_STYLE_ATTRIBUTE = " style=\"%s\"";
/** Span closing tag. */
public static final String HTML_SPAN_CLOSE_TAG = "</span>";
/** Text decoration property. */
public static final String HTML_TEXT_DECORATION_STYLE_PROPERTY = "text-decoration:%s; ";
/** Strikethrough text property. */
public static final String HTML_STRIKETHROUGH_VALUE = " line-through";
/** Underline text property. */
public static final String HTML_UNDERLINE_VALUE = " underline";
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/DiskPageImageCache.java
+97
View File
@@ -0,0 +1,97 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Disk-backed page image cache. Stores page images as PNG files in a
* temporary directory. evict() is a no-op (keeps files for potential re-read).
* close() deletes the temp directory and all files.
*/
public class DiskPageImageCache implements PageImageCache {
private static final Logger LOGGER = Logger.getLogger(DiskPageImageCache.class.getCanonicalName());
private final Path tempDir;
public DiskPageImageCache() throws IOException {
this.tempDir = Files.createTempDirectory("odl-pages-");
}
// Visible for testing
DiskPageImageCache(Path tempDir) {
this.tempDir = tempDir;
}
@Override
public BufferedImage getOrFetch(int pageIndex, PageImageFetcher fetcher) throws IOException {
Path file = tempDir.resolve("page-" + pageIndex + ".png");
if (Files.exists(file)) {
BufferedImage cached = ImageIO.read(file.toFile());
if (cached != null) {
return cached;
}
// Cached file is unreadable (corrupt or no ImageReader) — re-fetch.
LOGGER.log(Level.WARNING, "Cached page image is unreadable, re-fetching: {0}", file);
Files.deleteIfExists(file);
}
BufferedImage image = fetcher.fetch(pageIndex);
if (image == null) {
throw new IOException("Page image fetcher returned null for page " + pageIndex);
}
if (!ImageIO.write(image, "png", file.toFile())) {
throw new IOException("No ImageIO writer accepted PNG output for page " + pageIndex);
}
return image;
}
@Override
public void evict(int pageIndex) {
// no-op: keep on disk for potential re-read
}
@Override
public void close() throws IOException {
if (!Files.exists(tempDir)) {
return;
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(tempDir)) {
for (Path entry : stream) {
try {
Files.deleteIfExists(entry);
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to delete temp file: {0}", entry);
}
}
}
Files.deleteIfExists(tempDir);
}
/**
* Returns the temp directory path (for testing).
*/
Path getTempDir() {
return tempDir;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/DoclingFastServerClient.java
+315
View File
@@ -0,0 +1,315 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.MediaType;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* HTTP client for docling-fast-server API.
*
* <p>This client communicates with the optimized FastAPI server (opendataloader-pdf-hybrid)
* which provides 3.3x faster performance than docling-serve by using a DocumentConverter
* singleton pattern.
*
* <p>The API is compatible with docling-serve, using the same /v1/convert/file endpoint
* and response format.
*
* @see HybridClient
* @see HybridConfig
*/
public class DoclingFastServerClient implements HybridClient {
private static final Logger LOGGER = Logger.getLogger(DoclingFastServerClient.class.getCanonicalName());
/** Default URL for docling-fast-server. */
public static final String DEFAULT_URL = "http://localhost:5002";
private static final String CONVERT_ENDPOINT = "/v1/convert/file";
private static final String HEALTH_ENDPOINT = "/health";
private static final int HEALTH_CHECK_TIMEOUT_MS = 3000;
private static final String DEFAULT_FILENAME = "document.pdf";
private static final MediaType MEDIA_TYPE_PDF = MediaType.parse("application/pdf");
private final String baseUrl;
private final OkHttpClient httpClient;
private final ObjectMapper objectMapper;
/**
* Creates a new DoclingFastServerClient with the specified configuration.
*
* @param config The hybrid configuration containing URL and timeout settings.
*/
public DoclingFastServerClient(HybridConfig config) {
this.baseUrl = config.getEffectiveUrl("docling-fast");
this.objectMapper = new ObjectMapper();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.readTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.writeTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.build();
}
/**
* Creates a new DoclingFastServerClient with a custom OkHttpClient (for testing).
*
* @param baseUrl The base URL of the docling-fast-server instance.
* @param httpClient The OkHttp client to use for requests.
* @param objectMapper The Jackson ObjectMapper for JSON parsing.
*/
DoclingFastServerClient(String baseUrl, OkHttpClient httpClient, ObjectMapper objectMapper) {
this.baseUrl = baseUrl;
this.httpClient = httpClient;
this.objectMapper = objectMapper;
}
@Override
public void checkAvailability() throws IOException {
OkHttpClient healthClient = httpClient.newBuilder()
.connectTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.readTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.build();
Request healthRequest = new Request.Builder()
.url(baseUrl + HEALTH_ENDPOINT)
.get()
.build();
Response response;
try {
response = healthClient.newCall(healthRequest).execute();
} catch (IOException e) {
throw new IOException(
"Hybrid server is not available at " + baseUrl + "\n"
+ "To start the local hybrid server:\n"
+ " 1. Install: pip install \"opendataloader-pdf[hybrid]\"\n"
+ " 2. Start: opendataloader-pdf-hybrid --port 5002\n"
+ "To use a remote server or custom port: --hybrid-url http://host:port\n"
+ "Or pass --hybrid-fallback to fall back to Java-only output for this run.\n"
+ "Or run without --hybrid flag for Java-only processing.", e);
}
try (response) {
if (!response.isSuccessful()) {
throw new IOException(
"Hybrid server at " + baseUrl + " returned HTTP " + response.code()
+ " during health check.\n"
+ "The server is reachable but may be starting up or unhealthy.");
}
}
}
@Override
public HybridResponse convert(HybridRequest request) throws IOException {
Request httpRequest = buildConvertRequest(request);
LOGGER.log(Level.FINE, "Sending request to {0}", baseUrl + CONVERT_ENDPOINT);
try (Response response = httpClient.newCall(httpRequest).execute()) {
return parseResponse(response);
}
}
@Override
public CompletableFuture<HybridResponse> convertAsync(HybridRequest request) {
return CompletableFuture.supplyAsync(() -> {
try {
return convert(request);
} catch (IOException e) {
throw new IllegalStateException("Failed to convert", e);
}
});
}
/**
* Gets the base URL of this client.
*
* @return The base URL.
*/
public String getBaseUrl() {
return baseUrl;
}
/**
* Builds a multipart/form-data HTTP request for the convert endpoint.
*/
private Request buildConvertRequest(HybridRequest request) {
MultipartBody.Builder bodyBuilder = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("files", DEFAULT_FILENAME,
RequestBody.create(request.getPdfBytes(), MEDIA_TYPE_PDF));
// Add page range if specified
if (request.getPageNumbers() != null && !request.getPageNumbers().isEmpty()) {
int minPage = request.getPageNumbers().stream().min(Integer::compareTo).orElse(1);
int maxPage = request.getPageNumbers().stream().max(Integer::compareTo).orElse(Integer.MAX_VALUE);
bodyBuilder.addFormDataPart("page_ranges", minPage + "-" + maxPage);
}
return new Request.Builder()
.url(baseUrl + CONVERT_ENDPOINT)
.post(bodyBuilder.build())
.build();
}
/**
* Parses the HTTP response into a HybridResponse.
*/
private HybridResponse parseResponse(Response response) throws IOException {
if (!response.isSuccessful()) {
ResponseBody body = response.body();
String bodyStr = body != null ? body.string() : "";
throw new IOException("Docling Fast Server request failed with status " + response.code() +
": " + bodyStr);
}
ResponseBody body = response.body();
if (body == null) {
throw new IOException("Empty response body");
}
String responseStr = body.string();
JsonNode root = objectMapper.readTree(responseStr);
// Check for API error status
JsonNode statusNode = root.get("status");
String status = statusNode != null ? statusNode.asText() : "";
if ("failure".equals(status)) {
JsonNode errorsNode = root.get("errors");
String errorMessage = errorsNode != null ? errorsNode.toString() : "Unknown error";
throw new IOException("Docling Fast Server processing failed: " + errorMessage);
}
// Log partial_success status
if ("partial_success".equals(status)) {
JsonNode errorsNode = root.get("errors");
LOGGER.log(Level.WARNING, "Backend returned partial_success: {0}",
errorsNode != null ? errorsNode.toString() : "no error details");
}
// Extract document content
JsonNode documentNode = root.get("document");
if (documentNode == null) {
throw new IOException("Invalid response: missing 'document' field");
}
JsonNode jsonContent = documentNode.get("json_content");
// Extract per-page content from json_content if available
Map<Integer, JsonNode> pageContents = extractPageContents(jsonContent);
// Extract failed pages (1-indexed) from partial_success responses
List<Integer> failedPages = extractFailedPages(root);
// Extract per-step pipeline timings (layout, ocr, table_structure, etc.)
JsonNode timingsNode = root.get("timings");
return new HybridResponse(null, null, jsonContent, pageContents, failedPages, timingsNode);
}
/**
* Extracts per-page content from the DoclingDocument JSON structure.
*
* <p>The DoclingDocument stores page information in the "pages" object,
* keyed by page number (as string). This method extracts the content
* elements for each page based on the "prov" (provenance) information.
*/
private Map<Integer, JsonNode> extractPageContents(JsonNode jsonContent) {
Map<Integer, JsonNode> pageContents = new HashMap<>();
if (jsonContent == null) {
return pageContents;
}
// The pages node contains page metadata keyed by page number
JsonNode pagesNode = jsonContent.get("pages");
if (pagesNode != null && pagesNode.isObject()) {
Iterator<String> fieldNames = pagesNode.fieldNames();
while (fieldNames.hasNext()) {
String pageNumStr = fieldNames.next();
try {
int pageNum = Integer.parseInt(pageNumStr);
pageContents.put(pageNum, pagesNode.get(pageNumStr));
} catch (NumberFormatException ignored) {
// Skip non-numeric page keys
}
}
}
return pageContents;
}
/**
* Extracts the list of failed page numbers from the response.
*
* <p>When the backend returns partial_success, the failed_pages array contains
* 1-indexed page numbers that failed during processing (e.g., due to Invalid code point
* errors in PDF font encoding).
*/
private List<Integer> extractFailedPages(JsonNode root) {
JsonNode failedPagesNode = root.get("failed_pages");
if (failedPagesNode == null || !failedPagesNode.isArray() || failedPagesNode.isEmpty()) {
return Collections.emptyList();
}
List<Integer> failedPages = new ArrayList<>();
for (JsonNode pageNode : failedPagesNode) {
if (pageNode.isNumber() && pageNode.canConvertToInt()) {
failedPages.add(pageNode.asInt());
}
}
return failedPages;
}
/**
* Shuts down the HTTP client and releases all resources.
*
* <p>This gracefully shuts down the dispatcher's executor service,
* allowing the JVM to exit cleanly. Idle connections are evicted
* from the connection pool.
*/
public void shutdown() {
// Gracefully shutdown the dispatcher - allows pending requests to complete
httpClient.dispatcher().executorService().shutdown();
// Evict idle connections from pool (does not affect the server)
httpClient.connectionPool().evictAll();
// Close the cache if present
if (httpClient.cache() != null) {
try {
httpClient.cache().close();
} catch (Exception ignored) {
// Ignore cache close errors
}
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/DoclingSchemaTransformer.java
+621
View File
@@ -0,0 +1,621 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.hybrid.HybridClient.HybridResponse;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Transforms Docling JSON output to OpenDataLoader IObject hierarchy.
*
* <p>This transformer handles the DoclingDocument JSON format and converts
* its elements (texts, tables, pictures) to the equivalent IObject types
* used by OpenDataLoader's downstream processors and generators.
*
* <h2>Schema Mapping</h2>
* <ul>
* <li>texts (label: text) → SemanticParagraph</li>
* <li>texts (label: section_header) → SemanticHeading</li>
* <li>texts (label: caption, footnote) → SemanticParagraph</li>
* <li>texts (label: page_header, page_footer) → Filtered out (furniture)</li>
* <li>tables → TableBorder with rows and cells</li>
* <li>pictures → SemanticPicture (with optional description)</li>
* </ul>
*
* <h2>Coordinate System</h2>
* <p>Both Docling and OpenDataLoader use BOTTOMLEFT origin. Docling provides
* bbox as {l, t, r, b} while OpenDataLoader uses [left, bottom, right, top].
* When Docling uses TOPLEFT origin, coordinates are converted appropriately.
*
* <h2>Thread Safety</h2>
* <p>This class is NOT thread-safe. The {@code transform()} method updates
* internal state (pictureIndex) during each call. Concurrent calls
* to transform() on the same instance may produce incorrect results.
* Use separate instances for concurrent transformations.
*/
public class DoclingSchemaTransformer implements HybridSchemaTransformer {
private static final Logger LOGGER = Logger.getLogger(DoclingSchemaTransformer.class.getCanonicalName());
private static final String BACKEND_TYPE = "docling";
// Picture index counter — accumulates across transform() calls on the same instance
// to ensure document-unique indices when processing chunked responses (#352).
private int pictureIndex;
// Docling text labels
private static final String LABEL_TEXT = "text";
private static final String LABEL_SECTION_HEADER = "section_header";
private static final String LABEL_CAPTION = "caption";
private static final String LABEL_FOOTNOTE = "footnote";
private static final String LABEL_PAGE_HEADER = "page_header";
private static final String LABEL_PAGE_FOOTER = "page_footer";
private static final String LABEL_LIST_ITEM = "list_item";
private static final String LABEL_FORMULA = "formula";
// Docling coordinate origins
private static final String COORD_ORIGIN_BOTTOMLEFT = "BOTTOMLEFT";
private static final String COORD_ORIGIN_TOPLEFT = "TOPLEFT";
@Override
public String getBackendType() {
return BACKEND_TYPE;
}
@Override
public List<List<IObject>> transform(HybridResponse response, Map<Integer, Double> pageHeights) {
JsonNode json = response.getJson();
if (json == null) {
LOGGER.log(Level.WARNING, "HybridResponse JSON is null, returning empty result");
return Collections.emptyList();
}
// Note: pictureIndex is NOT reset here — it must accumulate across
// multiple transform() calls when processing chunked responses (#352).
// Each transformer instance starts with pictureIndex=0 (field default),
// so single-call usage is unaffected.
// Determine number of pages from page info or content
int numPages = determinePageCount(json, pageHeights);
// Initialize result list
List<List<IObject>> result = new ArrayList<>(numPages);
for (int i = 0; i < numPages; i++) {
result.add(new ArrayList<>());
}
// Transform texts
JsonNode texts = json.get("texts");
if (texts != null && texts.isArray()) {
for (JsonNode textNode : texts) {
transformText(textNode, result, pageHeights);
}
}
// Transform tables
JsonNode tables = json.get("tables");
if (tables != null && tables.isArray()) {
for (JsonNode tableNode : tables) {
transformTable(tableNode, result, pageHeights);
}
}
// Transform pictures
JsonNode pictures = json.get("pictures");
if (pictures != null && pictures.isArray()) {
for (JsonNode pictureNode : pictures) {
transformPicture(pictureNode, result, pageHeights);
}
}
// Sort each page's contents by reading order (top to bottom, left to right)
for (List<IObject> pageContents : result) {
sortByReadingOrder(pageContents);
}
return result;
}
@Override
public List<IObject> transformPage(int pageNumber, JsonNode pageContent, double pageHeight) {
Map<Integer, Double> pageHeights = new HashMap<>();
pageHeights.put(pageNumber, pageHeight);
// Create a wrapper response with just this page's content
HybridResponse singlePageResponse = new HybridResponse("", pageContent, Collections.emptyMap());
List<List<IObject>> result = transform(singlePageResponse, pageHeights);
if (result.isEmpty()) {
return Collections.emptyList();
}
// Find the page in the result
int pageIndex = pageNumber - 1;
if (pageIndex >= 0 && pageIndex < result.size()) {
return result.get(pageIndex);
}
return Collections.emptyList();
}
/**
* Determines the number of pages from the JSON response.
*/
private int determinePageCount(JsonNode json, Map<Integer, Double> pageHeights) {
// First check pageHeights if provided
if (pageHeights != null && !pageHeights.isEmpty()) {
return pageHeights.keySet().stream().mapToInt(Integer::intValue).max().orElse(0);
}
// Check pages array in JSON
JsonNode pages = json.get("pages");
if (pages != null && pages.isArray()) {
return pages.size();
}
// Check page_dimensions or similar
JsonNode pageDimensions = json.get("page_dimensions");
if (pageDimensions != null && pageDimensions.isObject()) {
int maxPage = 0;
Iterator<String> fieldNames = pageDimensions.fieldNames();
while (fieldNames.hasNext()) {
try {
int pageNum = Integer.parseInt(fieldNames.next());
maxPage = Math.max(maxPage, pageNum);
} catch (NumberFormatException e) {
// ignore
}
}
return maxPage;
}
// Default to scanning content
return scanContentForPageCount(json);
}
/**
* Scans content elements to determine page count.
*/
private int scanContentForPageCount(JsonNode json) {
int maxPage = 0;
JsonNode texts = json.get("texts");
if (texts != null && texts.isArray()) {
for (JsonNode text : texts) {
maxPage = Math.max(maxPage, getPageNumberFromProv(text));
}
}
JsonNode tables = json.get("tables");
if (tables != null && tables.isArray()) {
for (JsonNode table : tables) {
maxPage = Math.max(maxPage, getPageNumberFromProv(table));
}
}
return maxPage;
}
/**
* Extracts page number from provenance info.
*/
private int getPageNumberFromProv(JsonNode node) {
JsonNode prov = node.get("prov");
if (prov != null && prov.isArray() && prov.size() > 0) {
JsonNode firstProv = prov.get(0);
JsonNode pageNo = firstProv.get("page_no");
if (pageNo != null && pageNo.isInt()) {
return pageNo.asInt();
}
}
return 0;
}
/**
* Transforms a Docling text element to an IObject.
*/
private void transformText(JsonNode textNode, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
String label = getTextValue(textNode, "label");
// Skip furniture elements (page headers/footers)
if (LABEL_PAGE_HEADER.equals(label) || LABEL_PAGE_FOOTER.equals(label)) {
return;
}
// Get provenance for position info
JsonNode prov = textNode.get("prov");
if (prov == null || !prov.isArray() || prov.size() == 0) {
LOGGER.log(Level.FINE, "Text element missing provenance, skipping");
return;
}
JsonNode firstProv = prov.get(0);
int pageNo = firstProv.has("page_no") ? firstProv.get("page_no").asInt() : 1;
int pageIndex = pageNo - 1;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get bounding box
BoundingBox bbox = extractBoundingBox(firstProv.get("bbox"), pageIndex, pageHeights.get(pageNo));
// Get text content
String text = getTextValue(textNode, "text");
if (text == null || text.isEmpty()) {
text = getTextValue(textNode, "orig");
}
// Create appropriate IObject based on label
IObject object;
if (LABEL_SECTION_HEADER.equals(label)) {
object = createHeading(text, bbox, textNode);
} else if (LABEL_FORMULA.equals(label)) {
object = createFormula(text, bbox);
} else {
object = createParagraph(text, bbox);
}
if (object != null) {
result.get(pageIndex).add(object);
}
}
/**
* Creates a SemanticHeading from Docling section_header.
*/
private SemanticHeading createHeading(String text, BoundingBox bbox, JsonNode textNode) {
int level = 1; // Default level
// Try to extract level from node metadata
JsonNode meta = textNode.get("meta");
if (meta != null && meta.has("level")) {
level = meta.get("level").asInt(1);
}
// Create a text chunk and wrap in TextLine
TextChunk textChunk = new TextChunk(bbox, text, 12.0, 12.0);
textChunk.adjustSymbolEndsToBoundingBox(null);
TextLine textLine = new TextLine(textChunk);
// Create heading using default constructor and add content
SemanticHeading heading = new SemanticHeading();
heading.add(textLine);
heading.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
heading.setHeadingLevel(level);
return heading;
}
/**
* Creates a SemanticParagraph from Docling text element.
*/
private SemanticParagraph createParagraph(String text, BoundingBox bbox) {
// Create a text chunk and wrap in TextLine
TextChunk textChunk = new TextChunk(bbox, text, 12.0, 12.0);
textChunk.adjustSymbolEndsToBoundingBox(null);
TextLine textLine = new TextLine(textChunk);
// Create paragraph using default constructor and add content
SemanticParagraph paragraph = new SemanticParagraph();
paragraph.add(textLine);
paragraph.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return paragraph;
}
/**
* Creates a SemanticFormula from Docling formula element.
*
* @param latex The LaTeX representation of the formula
* @param bbox The bounding box
* @return A SemanticFormula object
*/
private SemanticFormula createFormula(String latex, BoundingBox bbox) {
SemanticFormula formula = new SemanticFormula(bbox, latex);
formula.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return formula;
}
/**
* Transforms a Docling picture element to a SemanticPicture.
*/
private void transformPicture(JsonNode pictureNode, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
// Get provenance for position info
JsonNode prov = pictureNode.get("prov");
if (prov == null || !prov.isArray() || prov.size() == 0) {
LOGGER.log(Level.FINE, "Picture element missing provenance, skipping");
return;
}
JsonNode firstProv = prov.get(0);
int pageNo = firstProv.has("page_no") ? firstProv.get("page_no").asInt() : 1;
int pageIndex = pageNo - 1;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get bounding box
BoundingBox bbox = extractBoundingBox(firstProv.get("bbox"), pageIndex, pageHeights.get(pageNo));
// Extract description from annotations (if available)
String description = extractPictureDescription(pictureNode);
// Create SemanticPicture with description
SemanticPicture picture = new SemanticPicture(bbox, ++pictureIndex, description);
picture.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
result.get(pageIndex).add(picture);
}
/**
* Extracts picture description from annotations array.
*
* <p>Docling stores picture descriptions in the annotations array with kind="description".
*
* @param pictureNode The picture JSON node
* @return The description text, or null if not available
*/
private String extractPictureDescription(JsonNode pictureNode) {
JsonNode annotations = pictureNode.get("annotations");
if (annotations != null && annotations.isArray()) {
for (JsonNode annotation : annotations) {
String kind = getTextValue(annotation, "kind");
if ("description".equals(kind)) {
return getTextValue(annotation, "text");
}
}
}
return null;
}
/**
* Transforms a Docling table element to a TableBorder.
*/
private void transformTable(JsonNode tableNode, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
// Get provenance for position info
JsonNode prov = tableNode.get("prov");
if (prov == null || !prov.isArray() || prov.size() == 0) {
LOGGER.log(Level.FINE, "Table element missing provenance, skipping");
return;
}
JsonNode firstProv = prov.get(0);
int pageNo = firstProv.has("page_no") ? firstProv.get("page_no").asInt() : 1;
int pageIndex = pageNo - 1;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get table data
JsonNode data = tableNode.get("data");
if (data == null) {
LOGGER.log(Level.FINE, "Table element missing data, skipping");
return;
}
// Get grid dimensions
JsonNode gridNode = data.get("grid");
if (gridNode == null || !gridNode.isArray()) {
LOGGER.log(Level.FINE, "Table missing grid data, skipping");
return;
}
int numRows = gridNode.size();
int numCols = 0;
if (numRows > 0 && gridNode.get(0).isArray()) {
numCols = gridNode.get(0).size();
}
if (numRows == 0 || numCols == 0) {
return;
}
// Get table bounding box
BoundingBox tableBbox = extractBoundingBox(firstProv.get("bbox"), pageIndex, pageHeights.get(pageNo));
// Create TableBorder
TableBorder table = new TableBorder(numRows, numCols);
table.setBoundingBox(tableBbox);
table.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
// Get table cells from data
JsonNode tableCells = data.get("table_cells");
Map<String, JsonNode> cellMap = new HashMap<>();
if (tableCells != null && tableCells.isArray()) {
for (JsonNode cell : tableCells) {
int startRow = cell.has("start_row_offset_idx") ? cell.get("start_row_offset_idx").asInt() : 0;
int startCol = cell.has("start_col_offset_idx") ? cell.get("start_col_offset_idx").asInt() : 0;
String key = startRow + "," + startCol;
cellMap.put(key, cell);
}
}
// Build table structure
double rowHeight = (tableBbox.getTopY() - tableBbox.getBottomY()) / numRows;
double colWidth = (tableBbox.getRightX() - tableBbox.getLeftX()) / numCols;
for (int row = 0; row < numRows; row++) {
TableBorderRow borderRow = new TableBorderRow(row, numCols, 0L);
double rowTop = tableBbox.getTopY() - (row * rowHeight);
double rowBottom = rowTop - rowHeight;
borderRow.setBoundingBox(new BoundingBox(pageIndex, tableBbox.getLeftX(), rowBottom, tableBbox.getRightX(), rowTop));
table.getRows()[row] = borderRow;
}
for (int row = 0; row < numRows; row++) {
for (int col = 0; col < numCols; col++) {
// Slot already covered by a spanning cell from an earlier
// iteration — do not overwrite with a fresh placeholder.
if (table.getRows()[row].getCells()[col] != null) continue;
String key = row + "," + col;
JsonNode cellNode = cellMap.get(key);
int rowSpan = 1;
int colSpan = 1;
String cellText = "";
if (cellNode != null) {
rowSpan = cellNode.has("row_span") ? cellNode.get("row_span").asInt(1) : 1;
colSpan = cellNode.has("col_span") ? cellNode.get("col_span").asInt(1) : 1;
cellText = getTextValue(cellNode, "text");
if (cellText == null) {
cellText = "";
}
}
// Clamp spans to the declared table bounds. Docling occasionally
// emits col_span/row_span values that, combined with the start
// index, run past the table; let the spanning area be exactly
// what stays inside the grid. Math.max(1, ...) defends against
// malformed zero or negative span values that would otherwise
// leave the slot unfilled.
int effectiveColSpan = Math.max(1, Math.min(colSpan, numCols - col));
int effectiveRowSpan = Math.max(1, Math.min(rowSpan, numRows - row));
TableBorderCell cell = new TableBorderCell(row, col, effectiveRowSpan, effectiveColSpan, 0L);
cell.setSemanticType(row == 0 ? SemanticType.TABLE_HEADER : SemanticType.TABLE_CELL);
double cellLeft = tableBbox.getLeftX() + (col * colWidth);
double cellRight = cellLeft + (effectiveColSpan * colWidth);
double cellTop = tableBbox.getTopY() - (row * rowHeight);
double cellBottom = cellTop - (effectiveRowSpan * rowHeight);
cell.setBoundingBox(new BoundingBox(pageIndex, cellLeft, cellBottom, cellRight, cellTop));
if (!cellText.isEmpty()) {
SemanticParagraph content = createParagraph(cellText, cell.getBoundingBox());
cell.addContentObject(content);
}
// Mark every slot covered by this cell's span with the same
// instance so AutoTaggingProcessor.addTableRow skips the
// duplicates. Without this Docling produces redundant
// 1x1 placeholders next to spanning cells, breaking
// PDF/UA-1 §7.2 (test 43) / PDF/UA-2 §8.2.5.26 "Table rows
// shall have the same number of columns (taking into account
// column spans)".
for (int r = row; r < row + effectiveRowSpan; r++) {
for (int c = col; c < col + effectiveColSpan; c++) {
table.getRows()[r].getCells()[c] = cell;
}
}
}
}
result.get(pageIndex).add(table);
}
/**
* Extracts a BoundingBox from Docling bbox JSON.
*
* @param bboxNode The bbox JSON node with l, t, r, b, coord_origin fields
* @param pageIndex The 0-indexed page number
* @param pageHeight The page height for coordinate transformation
* @return A BoundingBox in OpenDataLoader format
*/
private BoundingBox extractBoundingBox(JsonNode bboxNode, int pageIndex, Double pageHeight) {
if (bboxNode == null) {
return new BoundingBox(pageIndex, 0, 0, 0, 0);
}
double l = bboxNode.has("l") ? bboxNode.get("l").asDouble() : 0;
double t = bboxNode.has("t") ? bboxNode.get("t").asDouble() : 0;
double r = bboxNode.has("r") ? bboxNode.get("r").asDouble() : 0;
double b = bboxNode.has("b") ? bboxNode.get("b").asDouble() : 0;
String coordOrigin = bboxNode.has("coord_origin") ?
bboxNode.get("coord_origin").asText() : COORD_ORIGIN_BOTTOMLEFT;
double left, bottom, right, top;
if (COORD_ORIGIN_TOPLEFT.equals(coordOrigin) && pageHeight != null) {
// Convert from TOPLEFT to BOTTOMLEFT
// In TOPLEFT: t is distance from top, b is distance from top (t < b since t is higher)
// In BOTTOMLEFT: bottom is distance from bottom, top is distance from bottom
left = l;
right = r;
top = pageHeight - t; // t was distance from top
bottom = pageHeight - b; // b was distance from top
} else {
// BOTTOMLEFT origin - Docling uses {l, t, r, b} where t=top, b=bottom
left = l;
bottom = b;
right = r;
top = t;
}
return new BoundingBox(pageIndex, left, bottom, right, top);
}
/**
* Gets a text value from a JSON node.
*/
private String getTextValue(JsonNode node, String fieldName) {
if (node != null && node.has(fieldName)) {
JsonNode field = node.get(fieldName);
if (field.isTextual()) {
return field.asText();
}
}
return null;
}
/**
* Sorts page contents by reading order (top to bottom, left to right).
*/
private void sortByReadingOrder(List<IObject> contents) {
contents.sort(new Comparator<IObject>() {
@Override
public int compare(IObject o1, IObject o2) {
// Sort by top Y (descending - higher on page first)
double topDiff = o2.getTopY() - o1.getTopY();
if (Math.abs(topDiff) > 5.0) { // Use tolerance for same-line detection
return topDiff > 0 ? 1 : -1;
}
// Same line, sort by left X (ascending)
return Double.compare(o1.getLeftX(), o2.getLeftX());
}
});
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/ElementMetadata.java
+298
View File
@@ -0,0 +1,298 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.annotation.JsonInclude;
/**
* Per-element metadata produced by hybrid backends.
* Keyed by IObject.recognizedStructureId in a sidecar Map.
* All fields are optional — backends populate what they can.
*/
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public class ElementMetadata {
/** DLA AI score (0.0~1.0). Negative sentinel means "not provided". */
private double aiScore = -1.0;
/** Original Hancom AI label (0~17), -1 if not applicable. */
private int sourceLabel = -1;
/**
* The hybrid backend's per-page detection id for this element — the
* same value that surfaces as {@code object_id} in the raw DLA JSON
* and as {@code scored_items[].id} in the evidence report. Carrying
* it through to the corrected extraction tree lets downstream
* tooling line up a reading-ordered node with the original detection
* row without re-matching by bbox. Negative sentinel ({@code -1})
* means "not provided" — e.g. native pipeline output.
*/
private int dlaObjectId = -1;
/** Heading inference method: "bbox-height" or "fixed". */
private String headingInferenceMethod;
/** Bounding box height in pixels, used for heading inference. */
private Double bboxHeightPx;
/** Word-cell matching method: "bbox-intersection" or "tsr-text-fallback". */
private String wordMatchMethod;
/** Number of words matched in the cell. */
private int matchedWordCount;
/** TSR metadata for tables. */
private TsrMetadata tsr;
/** Caption metadata for images. */
private CaptionMetadata caption;
/** Regionlist resolution metadata for label 7. */
private RegionlistResolution regionlistResolution;
/** Text source decision: "stream", "ocr", or "ocr-fallback". */
private String textSource;
/** Stream vs OCR text similarity score (auto mode only). */
private Double streamOcrSimilarity;
public double getAiScore() {
return aiScore;
}
public ElementMetadata setAiScore(double aiScore) {
this.aiScore = aiScore;
return this;
}
public int getSourceLabel() {
return sourceLabel;
}
public ElementMetadata setSourceLabel(int sourceLabel) {
this.sourceLabel = sourceLabel;
return this;
}
public int getDlaObjectId() {
return dlaObjectId;
}
public ElementMetadata setDlaObjectId(int dlaObjectId) {
this.dlaObjectId = dlaObjectId;
return this;
}
public String getHeadingInferenceMethod() {
return headingInferenceMethod;
}
public ElementMetadata setHeadingInferenceMethod(String headingInferenceMethod) {
this.headingInferenceMethod = headingInferenceMethod;
return this;
}
public Double getBboxHeightPx() {
return bboxHeightPx;
}
public ElementMetadata setBboxHeightPx(Double bboxHeightPx) {
this.bboxHeightPx = bboxHeightPx;
return this;
}
public String getWordMatchMethod() {
return wordMatchMethod;
}
public ElementMetadata setWordMatchMethod(String wordMatchMethod) {
this.wordMatchMethod = wordMatchMethod;
return this;
}
public int getMatchedWordCount() {
return matchedWordCount;
}
public ElementMetadata setMatchedWordCount(int matchedWordCount) {
this.matchedWordCount = matchedWordCount;
return this;
}
public TsrMetadata getTsr() {
return tsr;
}
public ElementMetadata setTsr(TsrMetadata tsr) {
this.tsr = tsr;
return this;
}
public CaptionMetadata getCaption() {
return caption;
}
public ElementMetadata setCaption(CaptionMetadata caption) {
this.caption = caption;
return this;
}
public RegionlistResolution getRegionlistResolution() {
return regionlistResolution;
}
public ElementMetadata setRegionlistResolution(RegionlistResolution regionlistResolution) {
this.regionlistResolution = regionlistResolution;
return this;
}
public String getTextSource() {
return textSource;
}
public ElementMetadata setTextSource(String textSource) {
this.textSource = textSource;
return this;
}
public Double getStreamOcrSimilarity() {
return streamOcrSimilarity;
}
public ElementMetadata setStreamOcrSimilarity(Double streamOcrSimilarity) {
this.streamOcrSimilarity = streamOcrSimilarity;
return this;
}
/**
* TSR (Table Structure Recognition) metadata for table elements.
*/
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public static class TsrMetadata {
private int numCells;
private String html;
private long runTimeMs;
public int getNumCells() {
return numCells;
}
public TsrMetadata setNumCells(int numCells) {
this.numCells = numCells;
return this;
}
public String getHtml() {
return html;
}
public TsrMetadata setHtml(String html) {
this.html = html;
return this;
}
public long getRunTimeMs() {
return runTimeMs;
}
public TsrMetadata setRunTimeMs(long runTimeMs) {
this.runTimeMs = runTimeMs;
return this;
}
}
/**
* Caption metadata for image elements.
*/
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public static class CaptionMetadata {
private String text;
private String language;
private long runTimeMs;
public String getText() {
return text;
}
public CaptionMetadata setText(String text) {
this.text = text;
return this;
}
public String getLanguage() {
return language;
}
public CaptionMetadata setLanguage(String language) {
this.language = language;
return this;
}
public long getRunTimeMs() {
return runTimeMs;
}
public CaptionMetadata setRunTimeMs(long runTimeMs) {
this.runTimeMs = runTimeMs;
return this;
}
}
/**
* Resolution metadata for regionlist elements (label 7).
*/
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
public static class RegionlistResolution {
/** Strategy used: "table-first" or "list-only". */
private String strategy;
/** Whether TSR was attempted. */
private boolean tsrAttempted;
/** TSR result: "success", "no-cells", "failed", or null. */
private String tsrResult;
public String getStrategy() {
return strategy;
}
public RegionlistResolution setStrategy(String strategy) {
this.strategy = strategy;
return this;
}
public boolean isTsrAttempted() {
return tsrAttempted;
}
public RegionlistResolution setTsrAttempted(boolean tsrAttempted) {
this.tsrAttempted = tsrAttempted;
return this;
}
public String getTsrResult() {
return tsrResult;
}
public RegionlistResolution setTsrResult(String tsrResult) {
this.tsrResult = tsrResult;
return this;
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HancomAIClient.java
+895
View File
@@ -0,0 +1,895 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import okhttp3.MediaType;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* HTTP client for Hancom AI HOCR SDK API.
*
* <p>Pipeline:
* <ol>
* <li>pdf2img — convert each page to PNG image</li>
* <li>DOCUMENT_LAYOUT_WITH_OCR — layout analysis + OCR on full PDF</li>
* <li>TABLE_STRUCTURE_RECOGNITION — crop each Table/Regionlist from page image, send to TSR individually</li>
* <li>IMAGE_CAPTIONING — crop each Figure region from page image, caption individually</li>
* </ol>
*
* @see HancomAISchemaTransformer
*/
public class HancomAIClient implements HybridClient {
private static final Logger LOGGER = Logger.getLogger(HancomAIClient.class.getCanonicalName());
public static final String DEFAULT_URL = "http://localhost:18008";
private static final String SDK_ENDPOINT = "/hocr/sdk";
private static final String PDF2IMG_ENDPOINT = "/support/pdf2img";
private static final String PING_ENDPOINT = "/ping";
private static final String HEALTH_ENDPOINT = "/health";
private static final int HEALTH_CHECK_TIMEOUT_MS = 3000;
private static final String DEFAULT_FILENAME = "document.pdf";
private static final MediaType MEDIA_TYPE_PDF = MediaType.parse("application/pdf");
private static final MediaType MEDIA_TYPE_PNG = MediaType.parse("image/png");
// DLA label 7 = Regionlist (may be actual table or a list region)
private static final int LABEL_REGIONLIST = 7;
// DLA label 9 = Table
private static final int LABEL_TABLE = 9;
// DLA label 10 = Figure
private static final int LABEL_FIGURE = 10;
/** Padding (pixels) added around table crops before sending to TSR. */
private static final int TSR_CROP_PADDING = 20;
private final String baseUrl;
private final OkHttpClient httpClient;
private final ObjectMapper objectMapper;
private final HybridConfig config;
private String sourcePdfShaShort = "unknown";
private static final java.util.Map<String, String> MODULE_SHORT;
static {
java.util.Map<String, String> m = new java.util.HashMap<>();
m.put("DOCUMENT_LAYOUT_WITH_OCR", "dla-ocr");
m.put("DOCUMENT_LAYOUT_ANALYSIS", "dla");
m.put("TEXT_RECOGNITION", "ocr");
m.put("TABLE_STRUCTURE_RECOGNITION", "tsr");
m.put("IMAGE_CAPTIONING", "caption");
m.put("CHART_IMAGE_UNDERSTANDING", "chart");
MODULE_SHORT = java.util.Collections.unmodifiableMap(m);
}
private static String sha256ShortHex(byte[] data) {
try {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
byte[] hash = md.digest(data);
StringBuilder sb = new StringBuilder(12);
for (int i = 0; i < 6; i++) sb.append(String.format("%02x", hash[i]));
return sb.toString();
} catch (java.security.NoSuchAlgorithmException e) {
return "nohash000000";
}
}
// Test hook
void setSourcePdfShaShort(String s) { this.sourcePdfShaShort = s; }
public HancomAIClient(HybridConfig config) {
this.config = config;
this.baseUrl = config.getEffectiveUrl("hancom-ai");
this.objectMapper = new ObjectMapper();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.readTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.writeTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.build();
}
// Visible for testing
HancomAIClient(String baseUrl, OkHttpClient httpClient, ObjectMapper objectMapper) {
this.config = new HybridConfig();
this.baseUrl = baseUrl;
this.httpClient = httpClient;
this.objectMapper = objectMapper;
}
@Override
public JsonNode fetchHealth() {
OkHttpClient healthClient = httpClient.newBuilder()
.connectTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.readTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.build();
Request request = new Request.Builder()
.url(baseUrl + HEALTH_ENDPOINT)
.get()
.build();
try (Response response = healthClient.newCall(request).execute()) {
if (!response.isSuccessful()) return null;
ResponseBody body = response.body();
if (body == null) return null;
return objectMapper.readTree(body.string());
} catch (IOException e) {
LOGGER.log(Level.FINE,
"Hancom AI /health unavailable: {0}", e.getMessage());
return null;
}
}
@Override
public void checkAvailability() throws IOException {
OkHttpClient healthClient = httpClient.newBuilder()
.connectTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.readTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.build();
Request request = new Request.Builder()
.url(baseUrl + PING_ENDPOINT)
.get()
.build();
try (Response response = healthClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
throw new IOException("Hancom AI server at " + baseUrl +
" returned HTTP " + response.code());
}
} catch (IOException e) {
throw new IOException(
"Hancom AI server is not available at " + baseUrl + "\n"
+ "Check that the server is running and accessible.", e);
}
}
@Override
public HybridResponse convert(HybridRequest request) throws IOException {
byte[] pdfBytes = request.getPdfBytes();
this.sourcePdfShaShort = sha256ShortHex(pdfBytes);
LOGGER.log(Level.INFO, "Hancom AI: processing PDF ({0} bytes)", pdfBytes.length);
// Crop / page-image destination travels with the request, not the
// cached client's config, so the per-document target is correct even
// when the client is reused across documents (and is concurrency-safe
// since nothing shared is mutated).
CropOutput cropOutput = request.getCropOutput();
try (PageImageCache pageImageCache = createPageImageCache()) {
ObjectNode merged = objectMapper.createObjectNode();
ObjectNode timingsNode = objectMapper.createObjectNode();
// Step 1: DLA + OCR. This is required — downstream steps have nothing
// to process without it, so treat an empty response as a failure so the
// caller can fall back to the Java pipeline instead of silently emitting
// an empty document.
JsonNode dlaOcrResult = callModule(pdfBytes, "DOCUMENT_LAYOUT_WITH_OCR");
if (dlaOcrResult == null || !dlaOcrResult.isArray() || dlaOcrResult.size() == 0) {
throw new IOException(
"Hancom AI DOCUMENT_LAYOUT_WITH_OCR returned empty result — "
+ "backend unavailable or rejected the document");
}
merged.set("DOCUMENT_LAYOUT_WITH_OCR", dlaOcrResult);
addTimings(timingsNode, "DOCUMENT_LAYOUT_WITH_OCR", dlaOcrResult);
// Step 2: Table Structure — crop each Table region from page image, send to TSR individually
long tsrStartMs = System.currentTimeMillis();
ArrayNode tsrResults = recognizeTableStructures(pdfBytes, dlaOcrResult, pageImageCache, cropOutput);
long tsrMs = System.currentTimeMillis() - tsrStartMs;
merged.set("TABLE_STRUCTURE_RECOGNITION", tsrResults);
ObjectNode tsrTiming = objectMapper.createObjectNode();
tsrTiming.put("total_ms", tsrMs);
tsrTiming.put("count", tsrResults.size());
if (tsrResults.size() > 0) {
tsrTiming.put("avg_ms", tsrMs / tsrResults.size());
}
timingsNode.set("TABLE_STRUCTURE_RECOGNITION", tsrTiming);
// Step 3: Figure captioning — pdf2img → crop figures → caption each
long captionStartMs = System.currentTimeMillis();
ArrayNode figureCaptions = captionFigures(pdfBytes, dlaOcrResult, pageImageCache, cropOutput);
long captionMs = System.currentTimeMillis() - captionStartMs;
merged.set("FIGURE_CAPTIONS", figureCaptions);
// Evidence-report consumers need the same rendered page image that
// DLA bboxes are expressed against. TSR/FIGURE fetches already save
// their pages; this pass fills only pages that were not otherwise
// rendered.
saveDlaPageImages(pdfBytes, dlaOcrResult, pageImageCache, cropOutput);
ObjectNode captionTiming = objectMapper.createObjectNode();
captionTiming.put("total_ms", captionMs);
captionTiming.put("count", figureCaptions.size());
if (figureCaptions.size() > 0) {
captionTiming.put("avg_ms", captionMs / figureCaptions.size());
}
timingsNode.set("IMAGE_CAPTIONING", captionTiming);
merged.set("timings", timingsNode);
LOGGER.log(Level.INFO, "Hancom AI: completed — {0} table crops, {1} figure captions",
new Object[]{tsrResults.size(), figureCaptions.size()});
return new HybridResponse(null, null, merged, Collections.emptyMap(),
Collections.emptyList(), timingsNode);
}
}
/**
* Creates a PageImageCache based on config.
*/
private PageImageCache createPageImageCache() throws IOException {
if ("disk".equalsIgnoreCase(config.getImageCache())) {
return new DiskPageImageCache();
}
return new MemoryPageImageCache();
}
@Override
public CompletableFuture<HybridResponse> convertAsync(HybridRequest request) {
return CompletableFuture.supplyAsync(() -> {
try {
return convert(request);
} catch (IOException e) {
throw new IllegalStateException("Failed to convert via Hancom AI", e);
}
});
}
/**
* Captions each Figure found by DLA:
* 1. Get page images via pdf2img
* 2. Find Figure objects (label 10) from DLA results
* 3. Crop each Figure from page image
* 4. Send cropped image to IMAGE_CAPTIONING
*
* @return ArrayNode of {page_number, object_id, bbox, caption}
*/
private ArrayNode captionFigures(byte[] pdfBytes, JsonNode dlaResult,
PageImageCache pageImageCache, CropOutput cropOutput) {
ArrayNode captions = objectMapper.createArrayNode();
// Extract pages from DLA result
List<JsonNode> dlaPages = extractPages(dlaResult);
if (dlaPages.isEmpty()) return captions;
// Collect Figure objects per page
Map<Integer, List<JsonNode>> figuresByPage = new HashMap<>();
for (JsonNode page : dlaPages) {
int pageNum = page.has("page_number") ? page.get("page_number").asInt() : -1;
if (pageNum < 0) continue;
JsonNode objects = page.get("objects");
if (objects == null || !objects.isArray()) continue;
for (JsonNode obj : objects) {
int label = obj.has("label") ? obj.get("label").asInt() : -1;
if (label == LABEL_FIGURE) {
figuresByPage.computeIfAbsent(pageNum, k -> new ArrayList<>()).add(obj);
}
}
}
if (figuresByPage.isEmpty()) {
LOGGER.log(Level.INFO, "Hancom AI: no Figure objects found, skipping captioning");
return captions;
}
LOGGER.log(Level.INFO, "Hancom AI: captioning {0} figures across {1} pages",
new Object[]{figuresByPage.values().stream().mapToInt(List::size).sum(),
figuresByPage.size()});
for (Map.Entry<Integer, List<JsonNode>> entry : figuresByPage.entrySet()) {
int pageNum = entry.getKey();
List<JsonNode> figures = entry.getValue();
// Get page image via cache
BufferedImage pageImage;
try {
pageImage = pageImageCache.getOrFetch(pageNum,
idx -> fetchPageImage(pdfBytes, idx, cropOutput));
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to get page {0} image: {1}",
new Object[]{pageNum, e.getMessage()});
continue;
}
// Caption each figure
for (JsonNode fig : figures) {
JsonNode bboxNode = fig.get("bbox");
if (bboxNode == null || !bboxNode.isArray() || bboxNode.size() < 4) continue;
int left = bboxNode.get(0).asInt();
int top = bboxNode.get(1).asInt();
int right = bboxNode.get(2).asInt();
int bottom = bboxNode.get(3).asInt();
// Clamp to image bounds
left = Math.max(0, left);
top = Math.max(0, top);
right = Math.min(pageImage.getWidth(), right);
bottom = Math.min(pageImage.getHeight(), bottom);
if (right <= left || bottom <= top) continue;
try {
BufferedImage cropped = pageImage.getSubimage(left, top, right - left, bottom - top);
byte[] croppedPng = imageToPng(cropped);
// Save crop if configured
if (cropOutput.active()) {
int objId = fig.has("object_id") ? fig.get("object_id").asInt() : -1;
saveCropFile(cropOutput.directory(), pageNum, objId, "figure", croppedPng);
}
int objIdForCaption = fig.has("object_id") ? fig.get("object_id").asInt() : -1;
CaptionResult captionResult = callImageCaptioning(croppedPng, pageNum, objIdForCaption);
String caption = captionResult != null ? captionResult.caption : null;
ObjectNode capNode = objectMapper.createObjectNode();
capNode.put("page_number", pageNum);
capNode.put("object_id", fig.has("object_id") ? fig.get("object_id").asInt() : -1);
ArrayNode bboxArr = objectMapper.createArrayNode();
bboxArr.add(left).add(top).add(right).add(bottom);
capNode.set("bbox", bboxArr);
capNode.put("caption", caption != null ? caption : "");
if (captionResult != null && captionResult.confidence != null) {
capNode.put("confidence", captionResult.confidence);
}
captions.add(capNode);
LOGGER.log(Level.FINE, "Captioned figure page={0} bbox=[{1},{2},{3},{4}]: {5}",
new Object[]{pageNum, left, top, right, bottom,
caption != null ? caption.substring(0, Math.min(50, caption.length())) : ""});
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Failed to caption figure on page {0}: {1}",
new Object[]{pageNum, e.getMessage()});
}
}
// Done with this page — allow cache to reclaim memory
pageImageCache.evict(pageNum);
}
return captions;
}
/**
* For each Table region (label 7) found by DLA, crop the region from the
* page image and send the crop to TABLE_STRUCTURE_RECOGNITION individually.
*
* <p>When regionlist strategy is "list-only", label 7 is always treated as a
* list and TSR is skipped entirely.
*
* @param pdfBytes the original PDF bytes (needed for pdf2img)
* @param dlaResult the DLA+OCR result containing detected objects
* @param pageImageCache shared cache for page images
* @param cropOutput per-document destination for saved table crops
* @return ArrayNode of per-table results:
* [{page_number, object_id, label, dla_bbox, tsr: {cells, num_cells, html, ...}}]
*/
private ArrayNode recognizeTableStructures(byte[] pdfBytes, JsonNode dlaResult,
PageImageCache pageImageCache, CropOutput cropOutput) {
ArrayNode results = objectMapper.createArrayNode();
// In list-only mode, LABEL_REGIONLIST is always rendered as a list and
// does not need TSR. LABEL_TABLE still needs TSR — without it the
// transformer's LABEL_TABLE branch returns null and real tables drop out
// of the structured output entirely.
boolean skipRegionlistTsr = config.isRegionlistListOnly();
if (skipRegionlistTsr) {
LOGGER.log(Level.INFO, "Hancom AI: regionlist strategy is list-only, "
+ "skipping TSR for Regionlist (label 7); Table (label 9) TSR still runs");
}
List<JsonNode> dlaPages = extractPages(dlaResult);
if (dlaPages.isEmpty()) return results;
for (JsonNode page : dlaPages) {
int pageNum = page.has("page_number") ? page.get("page_number").asInt() : -1;
if (pageNum < 0) continue;
JsonNode objects = page.get("objects");
if (objects == null || !objects.isArray()) continue;
// Check if any table/regionlist objects exist on this page
boolean needsPageImage = false;
for (JsonNode obj : objects) {
int label = obj.has("label") ? obj.get("label").asInt() : -1;
if (label == LABEL_TABLE
|| (label == LABEL_REGIONLIST && !skipRegionlistTsr)) {
needsPageImage = true;
break;
}
}
if (!needsPageImage) continue;
// Get page image
BufferedImage pageImage;
try {
pageImage = pageImageCache.getOrFetch(pageNum,
idx -> fetchPageImage(pdfBytes, idx, cropOutput));
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to get page {0} image for TSR: {1}",
new Object[]{pageNum, e.getMessage()});
continue;
}
int imgWidth = pageImage.getWidth();
int imgHeight = pageImage.getHeight();
for (JsonNode obj : objects) {
int label = obj.has("label") ? obj.get("label").asInt() : -1;
if (label != LABEL_REGIONLIST && label != LABEL_TABLE) continue;
if (label == LABEL_REGIONLIST && skipRegionlistTsr) continue;
JsonNode bboxNode = obj.get("bbox");
if (bboxNode == null || !bboxNode.isArray() || bboxNode.size() < 4) continue;
int left = bboxNode.get(0).asInt();
int top = bboxNode.get(1).asInt();
int right = bboxNode.get(2).asInt();
int bottom = bboxNode.get(3).asInt();
// Add padding around crop
left = Math.max(0, left - TSR_CROP_PADDING);
top = Math.max(0, top - TSR_CROP_PADDING);
right = Math.min(imgWidth, right + TSR_CROP_PADDING);
bottom = Math.min(imgHeight, bottom + TSR_CROP_PADDING);
if (right <= left || bottom <= top) continue;
try {
BufferedImage crop = pageImage.getSubimage(left, top, right - left, bottom - top);
byte[] cropPng = imageToPng(crop);
// Save crop if configured
if (cropOutput.active()) {
int objectId = obj.has("object_id") ? obj.get("object_id").asInt() : -1;
saveCropFile(cropOutput.directory(), pageNum, objectId, "table", cropPng);
}
// Call TSR with crop image
int objId = obj.has("object_id") ? obj.get("object_id").asInt() : -1;
JsonNode tsrResult = callModuleImage(cropPng, "TABLE_STRUCTURE_RECOGNITION", pageNum, objId);
// Build result entry
ObjectNode entry = objectMapper.createObjectNode();
entry.put("page_number", pageNum);
entry.put("object_id",
obj.has("object_id") ? obj.get("object_id").asInt() : -1);
entry.put("label", label);
// Store the DLA bbox (padded, page-level pixels) for coordinate offset later
ArrayNode dlaBbox = objectMapper.createArrayNode();
dlaBbox.add(left).add(top).add(right).add(bottom);
entry.set("dla_bbox", dlaBbox);
// Extract TSR page result. The HOCR envelope wraps results
// as RESULT=[[page]], so the page node is where any
// top-level self-score lands.
List<JsonNode> tsrPages = extractPages(tsrResult);
if (!tsrPages.isEmpty()) {
JsonNode tsrPage = tsrPages.get(0);
JsonNode conf = tsrPage.get("confidence");
if (conf != null && conf.isNumber()) {
// doubleValue() returns the numeric value directly;
// asDouble() has a silent 0.0 fallback we don't want
// even though the isNumber() guard makes it unreachable.
entry.put("confidence", conf.doubleValue());
}
entry.set("tsr", tsrPage);
} else {
entry.set("tsr", objectMapper.createObjectNode());
}
results.add(entry);
} catch (Exception e) {
LOGGER.log(Level.WARNING, "TSR failed for page {0} object: {1}",
new Object[]{pageNum, e.getMessage()});
}
}
// Evict the page image here only if captionFigures() will not revisit
// this page. captionFigures() iterates pages that contain at least one
// LABEL_FIGURE object; for table-only pages (no figures) the cached
// full-resolution image would otherwise sit in memory until the
// try-with-resources in convert() closes the cache — ~25MB per page
// for the memory cache, which is costly on large table-heavy PDFs.
boolean hasFigure = false;
for (JsonNode obj : objects) {
int objLabel = obj.has("label") ? obj.get("label").asInt() : -1;
if (objLabel == LABEL_FIGURE) {
hasFigure = true;
break;
}
}
if (!hasFigure) {
pageImageCache.evict(pageNum);
}
}
LOGGER.log(Level.INFO, "Hancom AI: TSR processed {0} table crops", results.size());
return results;
}
/**
* Calls a single HOCR SDK module with image (PNG) input.
* Similar to {@link #callModule} but sends image data instead of PDF.
*/
private JsonNode callModuleImage(byte[] pngBytes, String moduleName, int pageNum, int objectId) throws IOException {
String moduleShort = MODULE_SHORT.getOrDefault(moduleName, moduleName);
String requestId = "odl-" + sourcePdfShaShort + "-p" + pageNum + "-o" + objectId + "-" + moduleShort;
MultipartBody body = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("REQUEST_ID", requestId)
.addFormDataPart("OPEN_API_NAME", moduleName)
.addFormDataPart("DATA_FORMAT", "image")
.addFormDataPart("FILE", "crop.png",
RequestBody.create(pngBytes, MEDIA_TYPE_PNG))
.build();
Request httpRequest = new Request.Builder()
.url(baseUrl + SDK_ENDPOINT)
.post(body)
.build();
LOGGER.log(Level.FINE, "Calling Hancom AI module (image): {0} [{1}]",
new Object[]{moduleName, requestId});
try (Response response = httpClient.newCall(httpRequest).execute()) {
if (!response.isSuccessful()) {
ResponseBody respBody = response.body();
String errorMsg = respBody != null ? respBody.string() : "";
LOGGER.log(Level.WARNING, "Hancom AI module {0} (image) returned HTTP {1}: {2}",
new Object[]{moduleName, response.code(), errorMsg});
return objectMapper.createArrayNode();
}
ResponseBody respBody = response.body();
if (respBody == null) {
return objectMapper.createArrayNode();
}
JsonNode root = objectMapper.readTree(respBody.string());
boolean success = root.has("SUCCESS") && root.get("SUCCESS").asBoolean();
if (!success) {
LOGGER.log(Level.WARNING, "Hancom AI module {0} (image) returned SUCCESS=false: {1}",
new Object[]{moduleName, root.has("MSG") ? root.get("MSG").asText() : ""});
return objectMapper.createArrayNode();
}
JsonNode result = root.get("RESULT");
return result != null ? result : objectMapper.createArrayNode();
}
}
/**
* Saves full-page render images for every DLA page when evidence image
* capture is enabled.
*/
private void saveDlaPageImages(byte[] pdfBytes, JsonNode dlaResult,
PageImageCache pageImageCache, CropOutput cropOutput) {
if (!cropOutput.active()) return;
for (JsonNode page : extractPages(dlaResult)) {
int pageNum = page.has("page_number") ? page.get("page_number").asInt() : -1;
if (pageNum < 0) continue;
if (isPageImageFileSaved(cropOutput.directory(), pageNum)) continue;
try {
pageImageCache.getOrFetch(pageNum, idx -> fetchPageImage(pdfBytes, idx, cropOutput));
} catch (IOException e) {
LOGGER.log(Level.FINE, "Failed to save DLA page image for page "
+ pageNum, e);
} finally {
pageImageCache.evict(pageNum);
}
}
}
private boolean isPageImageFileSaved(String outputDir, int pageNum) {
if (outputDir == null) return false;
File file = new File(new File(outputDir, "page-images"),
String.format("page-%d.png", pageNum));
return file.isFile();
}
/**
* Saves a cropped image to disk for debugging.
*/
private void saveCropFile(String outputDir, int pageNum, int objectId,
String labelName, byte[] pngBytes) {
try {
File dir = new File(outputDir, "crops");
dir.mkdirs();
String filename = String.format("page-%d_%s-o%d.png", pageNum, labelName, objectId);
Files.write(new File(dir, filename).toPath(), pngBytes);
} catch (IOException e) {
LOGGER.log(Level.FINE, "Failed to save crop file", e);
}
}
/**
* Saves a full rendered PDF page image to disk for evidence overlays.
*/
private void savePageImageFile(String outputDir, int pageNum, byte[] pngBytes) {
try {
File dir = new File(outputDir, "page-images");
dir.mkdirs();
String filename = String.format("page-%d.png", pageNum);
Files.write(new File(dir, filename).toPath(), pngBytes);
} catch (IOException e) {
LOGGER.log(Level.FINE, "Failed to save page image file", e);
}
}
/**
* Fetches a page image from the pdf2img endpoint.
*/
private BufferedImage fetchPageImage(byte[] pdfBytes, int pageIndex, CropOutput cropOutput)
throws IOException {
MultipartBody body = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("REQUEST_ID",
"odl-" + sourcePdfShaShort + "-pdf2img-p" + pageIndex)
.addFormDataPart("PAGE_INDEX", String.valueOf(pageIndex))
.addFormDataPart("FILE", DEFAULT_FILENAME,
RequestBody.create(pdfBytes, MEDIA_TYPE_PDF))
.build();
Request httpRequest = new Request.Builder()
.url(baseUrl + PDF2IMG_ENDPOINT)
.post(body)
.build();
try (Response response = httpClient.newCall(httpRequest).execute()) {
if (!response.isSuccessful()) {
throw new IOException("pdf2img returned HTTP " + response.code());
}
ResponseBody respBody = response.body();
if (respBody == null) {
throw new IOException("pdf2img returned empty body");
}
JsonNode root = objectMapper.readTree(respBody.string());
// Navigate: RESULT[0].RESULT.PAGE_PNG_DATA
JsonNode resultArr = root.get("RESULT");
if (resultArr == null || !resultArr.isArray() || resultArr.size() == 0) {
throw new IOException("pdf2img RESULT is empty");
}
JsonNode pageResult = resultArr.get(0);
JsonNode innerResult = pageResult.get("RESULT");
if (innerResult == null) {
throw new IOException("pdf2img inner RESULT is null");
}
String pngBase64 = innerResult.has("PAGE_PNG_DATA")
? innerResult.get("PAGE_PNG_DATA").asText() : null;
if (pngBase64 == null || pngBase64.isEmpty()) {
throw new IOException("pdf2img PAGE_PNG_DATA is empty");
}
byte[] pngBytes;
try {
pngBytes = Base64.getDecoder().decode(pngBase64);
} catch (IllegalArgumentException e) {
// fetchPageImage is declared to throw IOException and callers catch
// only IOException. Escaping IAE would abort the whole conversion
// instead of skipping the failed page.
throw new IOException("pdf2img PAGE_PNG_DATA is not valid Base64", e);
}
BufferedImage image = ImageIO.read(new ByteArrayInputStream(pngBytes));
if (image == null) {
throw new IOException("pdf2img PAGE_PNG_DATA is not a readable image");
}
if (cropOutput.active()) {
savePageImageFile(cropOutput.directory(), pageIndex, pngBytes);
}
return image;
}
}
/**
* Sends a cropped image to IMAGE_CAPTIONING and returns the caption text.
*/
/** Image-captioning result: caption text + the model's self-reported confidence. */
static final class CaptionResult {
final String caption;
final Double confidence;
CaptionResult(String caption, Double confidence) {
this.caption = caption;
this.confidence = confidence;
}
}
private CaptionResult callImageCaptioning(byte[] pngBytes, int pageNum, int objectId) throws IOException {
String requestId = "odl-" + sourcePdfShaShort + "-p" + pageNum + "-o" + objectId + "-caption";
MultipartBody body = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("REQUEST_ID", requestId)
.addFormDataPart("OPEN_API_NAME", "IMAGE_CAPTIONING")
.addFormDataPart("DATA_FORMAT", "image")
.addFormDataPart("FILE", "figure.png",
RequestBody.create(pngBytes, MEDIA_TYPE_PNG))
.build();
Request httpRequest = new Request.Builder()
.url(baseUrl + SDK_ENDPOINT)
.post(body)
.build();
try (Response response = httpClient.newCall(httpRequest).execute()) {
if (!response.isSuccessful()) return null;
ResponseBody respBody = response.body();
if (respBody == null) return null;
JsonNode root = objectMapper.readTree(respBody.string());
if (!root.has("SUCCESS") || !root.get("SUCCESS").asBoolean()) return null;
JsonNode result = root.get("RESULT");
if (result == null || !result.isArray() || result.size() == 0) return null;
JsonNode page = result.get(0);
if (page.isArray() && page.size() > 0) page = page.get(0);
String caption = page.has("caption") ? page.get("caption").asText("") : null;
JsonNode confNode = page.get("confidence");
Double confidence = confNode != null && confNode.isNumber()
? confNode.doubleValue() : null;
return new CaptionResult(caption, confidence);
}
}
/**
* Calls a single HOCR SDK module with PDF input.
*/
private JsonNode callModule(byte[] pdfBytes, String moduleName) throws IOException {
MultipartBody body = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("REQUEST_ID",
"odl-" + sourcePdfShaShort + "-" + MODULE_SHORT.getOrDefault(moduleName, moduleName))
.addFormDataPart("OPEN_API_NAME", moduleName)
.addFormDataPart("DATA_FORMAT", "pdf")
.addFormDataPart("FILE", DEFAULT_FILENAME,
RequestBody.create(pdfBytes, MEDIA_TYPE_PDF))
.build();
Request httpRequest = new Request.Builder()
.url(baseUrl + SDK_ENDPOINT)
.post(body)
.build();
LOGGER.log(Level.INFO, "Calling Hancom AI module: {0}", moduleName);
try (Response response = httpClient.newCall(httpRequest).execute()) {
if (!response.isSuccessful()) {
ResponseBody respBody = response.body();
String errorMsg = respBody != null ? respBody.string() : "";
LOGGER.log(Level.WARNING, "Hancom AI module {0} returned HTTP {1}: {2}",
new Object[]{moduleName, response.code(), errorMsg});
return objectMapper.createArrayNode();
}
ResponseBody respBody = response.body();
if (respBody == null) {
return objectMapper.createArrayNode();
}
JsonNode root = objectMapper.readTree(respBody.string());
boolean success = root.has("SUCCESS") && root.get("SUCCESS").asBoolean();
if (!success) {
LOGGER.log(Level.WARNING, "Hancom AI module {0} returned SUCCESS=false: {1}",
new Object[]{moduleName, root.has("MSG") ? root.get("MSG").asText() : ""});
return objectMapper.createArrayNode();
}
JsonNode result = root.get("RESULT");
return result != null ? result : objectMapper.createArrayNode();
}
}
// --- Helpers ---
private List<JsonNode> extractPages(JsonNode moduleResult) {
List<JsonNode> pages = new ArrayList<>();
if (moduleResult == null || !moduleResult.isArray()) return pages;
JsonNode inner = moduleResult.size() > 0 && moduleResult.get(0).isArray()
? moduleResult.get(0) : moduleResult;
for (JsonNode page : inner) {
if (page.isObject()) pages.add(page);
}
return pages;
}
private byte[] imageToPng(BufferedImage image) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
ImageIO.write(image, "png", baos);
return baos.toByteArray();
}
private void addTimings(ObjectNode timingsNode, String moduleName, JsonNode result) {
long totalMs = 0;
int pageCount = 0;
if (result.isArray() && result.size() > 0) {
JsonNode pages = result.get(0).isArray() ? result.get(0) : result;
for (JsonNode page : pages) {
if (page.has("run_time")) {
totalMs += page.get("run_time").asLong();
pageCount++;
}
}
}
ObjectNode timing = objectMapper.createObjectNode();
timing.put("total_ms", totalMs);
timing.put("count", pageCount);
if (pageCount > 0) timing.put("avg_ms", totalMs / pageCount);
timingsNode.set(moduleName, timing);
}
public String getBaseUrl() {
return baseUrl;
}
public void shutdown() {
httpClient.dispatcher().executorService().shutdown();
httpClient.connectionPool().evictAll();
if (httpClient.cache() != null) {
try { httpClient.cache().close(); } catch (Exception ignored) { }
}
}
// --- Test hooks (package-private) ---
void invokeCallModule(byte[] pdfBytes, String moduleName) throws IOException {
this.sourcePdfShaShort = sha256ShortHex(pdfBytes);
callModule(pdfBytes, moduleName);
}
void invokeCallImageCaptioning(byte[] pngBytes, int pageNum, int objectId) throws IOException {
callImageCaptioning(pngBytes, pageNum, objectId);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HancomAISchemaTransformer.java
+1352
View File
File diff suppressed because it is too large Load Diff
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HancomClient.java
+306
View File
@@ -0,0 +1,306 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import okhttp3.MediaType;
import okhttp3.MultipartBody;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.RequestBody;
import okhttp3.Response;
import okhttp3.ResponseBody;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* HTTP client for Hancom Document AI API.
*
* <p>This client communicates with the Hancom Document AI backend service
* for PDF processing. The workflow is:
* <ol>
* <li>Upload PDF file to /v1/dl/files/upload</li>
* <li>Request visual info extraction from /v1/dl/files/{fileId}/visualinfo</li>
* <li>Delete the file from server after processing</li>
* </ol>
*
* <p>The client ensures cleanup (file deletion) even when processing fails.
*
* @see HybridClient
* @see HybridConfig
*/
public class HancomClient implements HybridClient {
private static final Logger LOGGER = Logger.getLogger(HancomClient.class.getCanonicalName());
/** Default URL for Hancom Document AI API. */
public static final String DEFAULT_URL = "https://dataloader.cloud.hancom.com/studio-lite/api";
private static final String UPLOAD_ENDPOINT = "/v1/dl/files/upload";
private static final String VISUALINFO_ENDPOINT = "/v1/dl/files/%s/visualinfo";
private static final String DELETE_ENDPOINT = "/v1/dl/files/%s";
private static final String DEFAULT_FILENAME = "document.pdf";
private static final MediaType MEDIA_TYPE_PDF = MediaType.parse("application/pdf");
// Query parameters for visualinfo
private static final String ENGINE = "pdf_ai_dl";
private static final String DLA_MODE = "ENABLED";
private static final String OCR_MODE = "FORCE";
private static final int HEALTH_CHECK_TIMEOUT_MS = 3000;
private final String baseUrl;
private final OkHttpClient httpClient;
private final ObjectMapper objectMapper;
/**
* Creates a new HancomClient with the specified configuration.
*
* @param config The hybrid configuration containing URL and timeout settings.
*/
public HancomClient(HybridConfig config) {
String effectiveUrl = config.getEffectiveUrl("hancom");
this.baseUrl = effectiveUrl != null ? normalizeUrl(effectiveUrl) : DEFAULT_URL;
this.objectMapper = new ObjectMapper();
this.httpClient = new OkHttpClient.Builder()
.connectTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.readTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.writeTimeout(config.getTimeoutMs(), TimeUnit.MILLISECONDS)
.build();
}
/**
* Creates a new HancomClient with a custom OkHttpClient (for testing).
*
* @param baseUrl The base URL of the Hancom API.
* @param httpClient The OkHttp client to use for requests.
* @param objectMapper The Jackson ObjectMapper for JSON parsing.
*/
HancomClient(String baseUrl, OkHttpClient httpClient, ObjectMapper objectMapper) {
this.baseUrl = normalizeUrl(baseUrl);
this.httpClient = httpClient;
this.objectMapper = objectMapper;
}
@Override
public void checkAvailability() throws IOException {
OkHttpClient healthClient = httpClient.newBuilder()
.connectTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.readTimeout(HEALTH_CHECK_TIMEOUT_MS, TimeUnit.MILLISECONDS)
.build();
Request request = new Request.Builder()
.url(baseUrl)
.head()
.build();
try (Response response = healthClient.newCall(request).execute()) {
// Any HTTP response (including 4xx/5xx) means the server is reachable.
// Hancom API requires authentication for all endpoints, so a 401/403
// is expected and still proves connectivity.
} catch (IOException e) {
throw new IOException(
"Hybrid server is not available at " + baseUrl + "\n"
+ "Please check the server URL and ensure the Hancom API is accessible.\n"
+ "Or pass --hybrid-fallback to fall back to Java-only output for this run.\n"
+ "Or run without --hybrid flag for Java-only processing.", e);
}
}
@Override
public HybridResponse convert(HybridRequest request) throws IOException {
String fileId = null;
try {
// Step 1: Upload PDF
fileId = uploadFile(request.getPdfBytes());
LOGGER.log(Level.FINE, "Uploaded file with ID: {0}", fileId);
// Step 2: Get visual info
JsonNode visualInfo = getVisualInfo(fileId);
LOGGER.log(Level.FINE, "Retrieved visual info for file: {0}", fileId);
return new HybridResponse(null, visualInfo, null);
} finally {
// Step 3: Always cleanup
if (fileId != null) {
deleteFile(fileId);
}
}
}
@Override
public CompletableFuture<HybridResponse> convertAsync(HybridRequest request) {
return CompletableFuture.supplyAsync(() -> {
try {
return convert(request);
} catch (IOException e) {
throw new IllegalStateException("Failed to convert", e);
}
});
}
/**
* Gets the base URL of this client.
*
* @return The base URL.
*/
public String getBaseUrl() {
return baseUrl;
}
/**
* Uploads a PDF file to the Hancom API.
*
* @param pdfBytes The PDF file bytes.
* @return The file ID assigned by the server.
* @throws IOException If the upload fails.
*/
private String uploadFile(byte[] pdfBytes) throws IOException {
MultipartBody requestBody = new MultipartBody.Builder()
.setType(MultipartBody.FORM)
.addFormDataPart("file", DEFAULT_FILENAME,
RequestBody.create(pdfBytes, MEDIA_TYPE_PDF))
.build();
Request request = new Request.Builder()
.url(baseUrl + UPLOAD_ENDPOINT)
.post(requestBody)
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
ResponseBody body = response.body();
String bodyStr = body != null ? body.string() : "";
throw new IOException("Hancom upload failed with status " + response.code() + ": " + bodyStr);
}
ResponseBody body = response.body();
if (body == null) {
throw new IOException("Empty response body from upload");
}
JsonNode root = objectMapper.readTree(body.string());
// Response format: {"codeNum":0,"code":"file.upload.success","data":{"fileId":"...",...}}
JsonNode dataNode = root.get("data");
if (dataNode == null) {
throw new IOException("Invalid upload response: missing data field");
}
JsonNode fileIdNode = dataNode.get("fileId");
if (fileIdNode == null || !fileIdNode.isTextual()) {
throw new IOException("Invalid upload response: missing fileId in data");
}
return fileIdNode.asText();
}
}
/**
* Retrieves visual info for an uploaded file.
*
* @param fileId The file ID from upload.
* @return The visual info JSON response.
* @throws IOException If the request fails.
*/
private JsonNode getVisualInfo(String fileId) throws IOException {
String url = baseUrl + String.format(VISUALINFO_ENDPOINT, fileId) +
"?engine=" + ENGINE +
"&dlaMode=" + DLA_MODE +
"&ocrMode=" + OCR_MODE;
Request request = new Request.Builder()
.url(url)
.get()
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (!response.isSuccessful()) {
ResponseBody body = response.body();
String bodyStr = body != null ? body.string() : "";
throw new IOException("Hancom visualinfo failed with status " + response.code() + ": " + bodyStr);
}
ResponseBody body = response.body();
if (body == null) {
throw new IOException("Empty response body from visualinfo");
}
return objectMapper.readTree(body.string());
}
}
/**
* Deletes an uploaded file from the server.
*
* <p>This method silently ignores any errors to ensure cleanup
* doesn't interfere with the main processing result.
*
* @param fileId The file ID to delete.
*/
private void deleteFile(String fileId) {
String url = baseUrl + String.format(DELETE_ENDPOINT, fileId);
Request request = new Request.Builder()
.url(url)
.delete()
.build();
try (Response response = httpClient.newCall(request).execute()) {
if (response.isSuccessful()) {
LOGGER.log(Level.FINE, "Deleted file: {0}", fileId);
} else {
LOGGER.log(Level.WARNING, "Failed to delete file {0}: {1}",
new Object[]{fileId, response.code()});
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Error deleting file " + fileId, e);
}
}
/**
* Normalizes a URL by removing trailing slashes.
*/
private static String normalizeUrl(String url) {
if (url != null && url.endsWith("/")) {
return url.substring(0, url.length() - 1);
}
return url;
}
/**
* Shuts down the HTTP client and releases all resources.
*
* <p>This gracefully shuts down the dispatcher's executor service,
* allowing the JVM to exit cleanly. Idle connections are evicted
* from the connection pool.
*/
public void shutdown() {
httpClient.dispatcher().executorService().shutdown();
httpClient.connectionPool().evictAll();
if (httpClient.cache() != null) {
try {
httpClient.cache().close();
} catch (Exception ignored) {
// Ignore cache close errors
}
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HancomSchemaTransformer.java
+554
View File
@@ -0,0 +1,554 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.hybrid.HybridClient.HybridResponse;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Transforms Hancom VisualInfoDto JSON output to OpenDataLoader IObject hierarchy.
*
* <p>This transformer handles the Hancom Document AI response format and converts
* its elements (PARAGRAPH, HEADING, TABLE, FIGURE, etc.) to the equivalent IObject
* types used by OpenDataLoader's downstream processors and generators.
*
* <h2>Schema Mapping</h2>
* <ul>
* <li>PARAGRAPH → SemanticParagraph</li>
* <li>HEADING → SemanticHeading</li>
* <li>TABLE → TableBorder with rows and cells</li>
* <li>FIGURE → SemanticPicture</li>
* <li>FORMULA → SemanticFormula</li>
* <li>LIST_ITEM → SemanticParagraph</li>
* <li>PAGE_HEADER, PAGE_FOOTER → Filtered out (furniture)</li>
* </ul>
*
* <h2>Coordinate System</h2>
* <p>Hancom uses TOPLEFT origin with (left, top, width, height) format.
* OpenDataLoader uses BOTTOMLEFT origin with (left, bottom, right, top) format.
* This transformer handles the coordinate conversion.
*
* <h2>Thread Safety</h2>
* <p>This class is NOT thread-safe. The {@code transform()} method resets
* internal state (pictureIndex) at the start of each call. Concurrent calls
* to transform() on the same instance may produce incorrect results.
* Use separate instances for concurrent transformations.
*/
public class HancomSchemaTransformer implements HybridSchemaTransformer {
private static final Logger LOGGER = Logger.getLogger(HancomSchemaTransformer.class.getCanonicalName());
private static final String BACKEND_TYPE = "hancom";
// Picture index counter (reset per transform call)
private int pictureIndex;
// Hancom element types
private static final String TYPE_PARAGRAPH = "PARAGRAPH";
private static final String TYPE_HEADING = "HEADING";
private static final String TYPE_TABLE = "TABLE";
private static final String TYPE_FIGURE = "FIGURE";
private static final String TYPE_FORMULA = "FORMULA";
private static final String TYPE_LIST_ITEM = "LIST_ITEM";
private static final String TYPE_PAGE_HEADER = "PAGE_HEADER";
private static final String TYPE_PAGE_FOOTER = "PAGE_FOOTER";
@Override
public String getBackendType() {
return BACKEND_TYPE;
}
@Override
public List<List<IObject>> transform(HybridResponse response, Map<Integer, Double> pageHeights) {
JsonNode json = response.getJson();
if (json == null) {
LOGGER.log(Level.WARNING, "HybridResponse JSON is null, returning empty result");
return Collections.emptyList();
}
// Reset picture index for each transform call
pictureIndex = 0;
// Determine number of pages
int numPages = determinePageCount(json, pageHeights);
// Initialize result list
List<List<IObject>> result = new ArrayList<>(numPages);
for (int i = 0; i < numPages; i++) {
result.add(new ArrayList<>());
}
// Transform elements
JsonNode elements = json.get("elements");
if (elements != null && elements.isArray()) {
for (JsonNode element : elements) {
transformElement(element, result, pageHeights);
}
}
// Sort each page's contents by reading order (top to bottom, left to right)
for (List<IObject> pageContents : result) {
sortByReadingOrder(pageContents);
}
return result;
}
@Override
public List<IObject> transformPage(int pageNumber, JsonNode pageContent, double pageHeight) {
Map<Integer, Double> pageHeights = new HashMap<>();
pageHeights.put(pageNumber, pageHeight);
// Create a wrapper response with just this page's content
HybridResponse singlePageResponse = new HybridResponse("", pageContent, Collections.emptyMap());
List<List<IObject>> result = transform(singlePageResponse, pageHeights);
if (result.isEmpty()) {
return Collections.emptyList();
}
// Find the page in the result (pageIndex is 0-based, pageNumber is 1-based)
int pageIndex = pageNumber - 1;
if (pageIndex >= 0 && pageIndex < result.size()) {
return result.get(pageIndex);
}
// If result has content but page index doesn't match, return first page
if (!result.isEmpty() && !result.get(0).isEmpty()) {
return result.get(0);
}
return Collections.emptyList();
}
/**
* Determines the number of pages from the JSON response.
*/
private int determinePageCount(JsonNode json, Map<Integer, Double> pageHeights) {
// First check pageHeights if provided
if (pageHeights != null && !pageHeights.isEmpty()) {
return pageHeights.keySet().stream().mapToInt(Integer::intValue).max().orElse(0);
}
// Check pageSizes array
JsonNode pageSizes = json.get("pageSizes");
if (pageSizes != null && pageSizes.isArray()) {
return pageSizes.size();
}
// Scan elements for max pageIndex
return scanElementsForPageCount(json);
}
/**
* Scans elements to determine page count.
*/
private int scanElementsForPageCount(JsonNode json) {
int maxPage = 0;
JsonNode elements = json.get("elements");
if (elements != null && elements.isArray()) {
for (JsonNode element : elements) {
JsonNode pageIndex = element.get("pageIndex");
if (pageIndex != null && pageIndex.isInt()) {
maxPage = Math.max(maxPage, pageIndex.asInt() + 1); // pageIndex is 0-based
}
}
}
return Math.max(maxPage, 1); // At least 1 page
}
/**
* Transforms a single Hancom element to an IObject.
*/
private void transformElement(JsonNode element, List<List<IObject>> result, Map<Integer, Double> pageHeights) {
// Get category type
JsonNode category = element.get("category");
if (category == null) {
LOGGER.log(Level.FINE, "Element missing category, skipping");
return;
}
String type = getTextValue(category, "type");
if (type == null) {
LOGGER.log(Level.FINE, "Element category missing type, skipping");
return;
}
// Skip furniture elements (page headers/footers)
if (TYPE_PAGE_HEADER.equals(type) || TYPE_PAGE_FOOTER.equals(type)) {
return;
}
// Get page index (0-based)
int pageIndex = element.has("pageIndex") ? element.get("pageIndex").asInt() : 0;
// Ensure result list is large enough
while (result.size() <= pageIndex) {
result.add(new ArrayList<>());
}
// Get bounding box
JsonNode bboxNode = element.get("bbox");
if (bboxNode == null) {
LOGGER.log(Level.FINE, "Element missing bbox, skipping");
return;
}
// Get page height for coordinate conversion
Double pageHeight = pageHeights != null ? pageHeights.get(pageIndex + 1) : null;
if (pageHeight == null) {
// Try to get from pageSizes
pageHeight = 842.0; // Default A4 height
}
BoundingBox bbox = extractBoundingBox(bboxNode, pageIndex, pageHeight);
// Get content
JsonNode contentNode = element.get("content");
String text = contentNode != null ? getTextValue(contentNode, "text") : null;
if (text == null) {
text = "";
}
// Create appropriate IObject based on type
IObject object = null;
switch (type) {
case TYPE_PARAGRAPH:
case TYPE_LIST_ITEM:
object = createParagraph(text, bbox);
break;
case TYPE_HEADING:
object = createHeading(text, bbox);
break;
case TYPE_TABLE:
object = transformTable(element, bbox, pageIndex, pageHeight);
break;
case TYPE_FIGURE:
object = createPicture(bbox);
break;
case TYPE_FORMULA:
object = createFormula(text, bbox);
break;
default:
// Unknown type, treat as paragraph if has text
if (!text.isEmpty()) {
object = createParagraph(text, bbox);
}
break;
}
if (object != null) {
result.get(pageIndex).add(object);
}
}
/**
* Creates a SemanticParagraph.
*/
private SemanticParagraph createParagraph(String text, BoundingBox bbox) {
TextChunk textChunk = new TextChunk(bbox, text, 12.0, 12.0);
textChunk.adjustSymbolEndsToBoundingBox(null);
TextLine textLine = new TextLine(textChunk);
SemanticParagraph paragraph = new SemanticParagraph();
paragraph.add(textLine);
paragraph.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
// Set semantic score to avoid NullPointerException in ListUtils.isContainsHeading()
paragraph.setCorrectSemanticScore(1.0);
return paragraph;
}
/**
* Creates a SemanticHeading.
*/
private SemanticHeading createHeading(String text, BoundingBox bbox) {
TextChunk textChunk = new TextChunk(bbox, text, 12.0, 12.0);
textChunk.adjustSymbolEndsToBoundingBox(null);
TextLine textLine = new TextLine(textChunk);
SemanticHeading heading = new SemanticHeading();
heading.add(textLine);
heading.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
heading.setHeadingLevel(1); // Default level
// Set semantic score to avoid NullPointerException in ListUtils.isContainsHeading()
heading.setCorrectSemanticScore(1.0);
return heading;
}
/**
* Creates a SemanticFormula.
*/
private SemanticFormula createFormula(String latex, BoundingBox bbox) {
SemanticFormula formula = new SemanticFormula(bbox, latex);
formula.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return formula;
}
/**
* Creates a SemanticPicture.
*/
private SemanticPicture createPicture(BoundingBox bbox) {
SemanticPicture picture = new SemanticPicture(bbox, ++pictureIndex, null);
picture.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return picture;
}
/**
* Transforms a Hancom table element to a TableBorder.
*
* <p>Hancom API returns table data in this structure:
* <pre>
* {
* "content": {
* "text": "...",
* "html": "&lt;table&gt;...&lt;/table&gt;",
* "table": {
* "cells": [
* {"cellId": "0", "rowspan": [0], "colspan": [0], "bbox": {...}, "text": "..."},
* ...
* ]
* }
* }
* }
* </pre>
*/
private TableBorder transformTable(JsonNode element, BoundingBox tableBbox,
int pageIndex, double pageHeight) {
// Get table cells from content.table.cells
JsonNode contentNode = element.get("content");
if (contentNode == null) {
LOGGER.log(Level.FINE, "Table element missing content, skipping");
return null;
}
// Hancom API: cells are in content.table.cells
JsonNode tableNode = contentNode.get("table");
if (tableNode == null) {
LOGGER.log(Level.FINE, "Table element missing content.table, skipping");
return null;
}
JsonNode cellsNode = tableNode.get("cells");
if (cellsNode == null || !cellsNode.isArray() || cellsNode.size() == 0) {
LOGGER.log(Level.FINE, "Table missing cells, skipping");
return null;
}
// Determine table dimensions from cells
int numRows = 0;
int numCols = 0;
Map<String, JsonNode> cellMap = new HashMap<>();
for (JsonNode cell : cellsNode) {
JsonNode rowspanNode = cell.get("rowspan");
JsonNode colspanNode = cell.get("colspan");
if (rowspanNode != null && rowspanNode.isArray() && rowspanNode.size() > 0) {
int maxRow = 0;
for (JsonNode r : rowspanNode) {
maxRow = Math.max(maxRow, r.asInt() + 1);
}
numRows = Math.max(numRows, maxRow);
}
if (colspanNode != null && colspanNode.isArray() && colspanNode.size() > 0) {
int maxCol = 0;
for (JsonNode c : colspanNode) {
maxCol = Math.max(maxCol, c.asInt() + 1);
}
numCols = Math.max(numCols, maxCol);
}
// Store cell by row,col key
int row = rowspanNode != null && rowspanNode.size() > 0 ? rowspanNode.get(0).asInt() : 0;
int col = colspanNode != null && colspanNode.size() > 0 ? colspanNode.get(0).asInt() : 0;
String key = row + "," + col;
cellMap.put(key, cell);
}
if (numRows == 0 || numCols == 0) {
return null;
}
// Create TableBorder
TableBorder table = new TableBorder(numRows, numCols);
table.setBoundingBox(tableBbox);
table.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
// Build table structure
double rowHeight = (tableBbox.getTopY() - tableBbox.getBottomY()) / numRows;
double colWidth = (tableBbox.getRightX() - tableBbox.getLeftX()) / numCols;
// Pass 1: Initialize all row containers to prevent NPE when backfilling spans
for (int row = 0; row < numRows; row++) {
TableBorderRow borderRow = new TableBorderRow(row, numCols, 0L);
double rowTop = tableBbox.getTopY() - (row * rowHeight);
double rowBottom = rowTop - rowHeight;
borderRow.setBoundingBox(new BoundingBox(pageIndex,
tableBbox.getLeftX(), rowBottom, tableBbox.getRightX(), rowTop));
table.getRows()[row] = borderRow;
}
// Pass 2: Fill cells with proper span backfill
for (int row = 0; row < numRows; row++) {
TableBorderRow borderRow = table.getRows()[row];
for (int col = 0; col < numCols; col++) {
if (borderRow.getCells()[col] != null) {
continue;
}
String key = row + "," + col;
JsonNode cellNode = cellMap.get(key);
int rowSpan = 1;
int colSpan = 1;
String cellText = "";
if (cellNode != null) {
JsonNode rowspanNode = cellNode.get("rowspan");
JsonNode colspanNode = cellNode.get("colspan");
rowSpan = rowspanNode != null ? rowspanNode.size() : 1;
colSpan = colspanNode != null ? colspanNode.size() : 1;
cellText = getTextValue(cellNode, "text");
if (cellText == null) {
cellText = "";
}
}
int effectiveRowSpan = Math.max(1, Math.min(rowSpan, numRows - row));
int effectiveColSpan = Math.max(1, Math.min(colSpan, numCols - col));
TableBorderCell cell = new TableBorderCell(row, col, effectiveRowSpan, effectiveColSpan, 0L);
cell.setSemanticType(row == 0 ? SemanticType.TABLE_HEADER : SemanticType.TABLE_CELL);
double cellLeft = tableBbox.getLeftX() + (col * colWidth);
double cellRight = cellLeft + (effectiveColSpan * colWidth);
double cellTop = tableBbox.getTopY() - (row * rowHeight);
double cellBottom = cellTop - (effectiveRowSpan * rowHeight);
cell.setBoundingBox(new BoundingBox(pageIndex, cellLeft, cellBottom, cellRight, cellTop));
// Add cell content if present
if (!cellText.isEmpty()) {
SemanticParagraph content = createParagraph(cellText, cell.getBoundingBox());
cell.addContentObject(content);
}
borderRow.getCells()[col] = cell;
// Backfill: assign same cell instance to all slots it covers
// All target rows are guaranteed to exist from Pass 1
for (int r = row; r < row + effectiveRowSpan; r++) {
for (int c = col; c < col + effectiveColSpan; c++) {
table.getRows()[r].getCells()[c] = cell;
}
}
}
}
return table;
}
/**
* Extracts a BoundingBox from Hancom bbox JSON.
*
* <p>Hancom uses TOPLEFT origin with {left, top, width, height} format.
* OpenDataLoader uses BOTTOMLEFT origin with {left, bottom, right, top} format.
*
* @param bboxNode The bbox JSON node with left, top, width, height fields
* @param pageIndex The 0-indexed page number
* @param pageHeight The page height for coordinate transformation
* @return A BoundingBox in OpenDataLoader format
*/
private BoundingBox extractBoundingBox(JsonNode bboxNode, int pageIndex, double pageHeight) {
if (bboxNode == null) {
return new BoundingBox(pageIndex, 0, 0, 0, 0);
}
double left = bboxNode.has("left") ? bboxNode.get("left").asDouble() : 0;
double top = bboxNode.has("top") ? bboxNode.get("top").asDouble() : 0;
double width = bboxNode.has("width") ? bboxNode.get("width").asDouble() : 0;
double height = bboxNode.has("height") ? bboxNode.get("height").asDouble() : 0;
// Convert from TOPLEFT to BOTTOMLEFT origin
// In TOPLEFT: top is distance from top of page
// In BOTTOMLEFT: bottom is distance from bottom of page
double right = left + width;
double bottomY = pageHeight - top - height; // Convert top distance to bottom coordinate
double topY = pageHeight - top; // Convert top distance to top coordinate
return new BoundingBox(pageIndex, left, bottomY, right, topY);
}
/**
* Gets a text value from a JSON node.
*/
private String getTextValue(JsonNode node, String fieldName) {
if (node != null && node.has(fieldName)) {
JsonNode field = node.get(fieldName);
if (field.isTextual()) {
return field.asText();
}
}
return null;
}
/**
* Sorts page contents by reading order (top to bottom, left to right).
*/
private void sortByReadingOrder(List<IObject> contents) {
contents.sort(new Comparator<IObject>() {
@Override
public int compare(IObject o1, IObject o2) {
// Sort by top Y (descending - higher on page first)
double topDiff = o2.getTopY() - o1.getTopY();
if (Math.abs(topDiff) > 5.0) { // Use tolerance for same-line detection
return topDiff > 0 ? 1 : -1;
}
// Same line, sort by left X (ascending)
return Double.compare(o1.getLeftX(), o2.getLeftX());
}
});
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HybridClient.java
+435
View File
@@ -0,0 +1,435 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
/**
* Interface for hybrid PDF processing backends.
*
* <p>Hybrid processing routes pages to external AI backends (like docling, hancom, azure)
* for advanced document parsing capabilities such as table structure extraction and OCR.
*
* <p>Implementations of this interface provide HTTP client integration with specific backends.
*/
public interface HybridClient {
/**
* Optional: fetch the backend's {@code /health} (or equivalent) payload
* for reference-environment metadata (hardware, model identifiers,
* backend version). Best-effort — returns {@code null} when the backend
* has no such endpoint or the call fails. The default returns
* {@code null}; implementations that have a health endpoint should
* override.
*/
default com.fasterxml.jackson.databind.JsonNode fetchHealth() {
return null;
}
/**
* Per-document destination for crop / page-image artifacts.
*
* <p>Clients are cached and reused across documents
* ({@link HybridClientFactory#getOrCreate}), so the {@link HybridConfig}
* captured at construction reflects only the <em>first</em> document and
* cannot be the source of truth for where the <em>current</em> document's
* crops belong. This value travels with each {@link HybridRequest}
* instead, so the destination tracks the document being processed without
* any shared mutable state — safe even if convert calls run concurrently.
*
* <p>{@code enabled} controls whether crop / page-image artifacts are
* written; {@code directory} is the destination for this document (or
* {@code null} when none).
*/
final class CropOutput {
/** Sentinel meaning "do not write crops for this request". */
public static final CropOutput DISABLED = new CropOutput(false, null);
private final boolean enabled;
private final String directory;
public CropOutput(boolean enabled, String directory) {
this.enabled = enabled;
this.directory = directory;
}
/** Whether crop / page-image artifacts should be written. */
public boolean enabled() {
return enabled;
}
/** Destination directory, or {@code null} when none is set. */
public String directory() {
return directory;
}
/** True when crops should be written and a destination is set. */
public boolean active() {
return enabled && directory != null;
}
}
/**
* Output formats that can be requested from the hybrid backend.
*/
enum OutputFormat {
/** JSON structured document format (DoclingDocument). */
JSON("json"),
/** Markdown text format. */
MARKDOWN("md"),
/** HTML format. */
HTML("html");
private final String apiValue;
OutputFormat(String apiValue) {
this.apiValue = apiValue;
}
/** Returns the API parameter value for this format. */
public String getApiValue() {
return apiValue;
}
}
/**
* Request class containing PDF bytes and processing options.
*
* <p>Note: OCR and table structure detection are always enabled on the server side.
* The DocumentConverter is initialized once at startup with fixed options for performance.
*/
final class HybridRequest {
private final byte[] pdfBytes;
private final Set<Integer> pageNumbers;
private final Set<OutputFormat> outputFormats;
private final CropOutput cropOutput;
/**
* Creates a new HybridRequest.
*
* @param pdfBytes The raw PDF file bytes to process.
* @param pageNumbers Set of 1-indexed page numbers to process. If empty, process all pages.
* @param outputFormats Set of output formats to request. If empty, defaults to all formats.
*/
public HybridRequest(byte[] pdfBytes, Set<Integer> pageNumbers,
Set<OutputFormat> outputFormats) {
this(pdfBytes, pageNumbers, outputFormats, CropOutput.DISABLED);
}
private HybridRequest(byte[] pdfBytes, Set<Integer> pageNumbers,
Set<OutputFormat> outputFormats, CropOutput cropOutput) {
this.pdfBytes = pdfBytes != null ? Arrays.copyOf(pdfBytes, pdfBytes.length) : null;
this.pageNumbers = pageNumbers != null ? pageNumbers : Collections.emptySet();
this.outputFormats = outputFormats != null && !outputFormats.isEmpty()
? EnumSet.copyOf(outputFormats)
: EnumSet.allOf(OutputFormat.class);
this.cropOutput = cropOutput != null ? cropOutput : CropOutput.DISABLED;
}
/**
* Returns a copy of this request with the given crop-output destination.
* Lets the caller attach a per-document crop target without mutating the
* cached client. The original request is unchanged.
*
* @param cropOutput where to write crops / page images for this document
* @return a new request carrying {@code cropOutput}
*/
public HybridRequest withCropOutput(CropOutput cropOutput) {
return new HybridRequest(pdfBytes, pageNumbers, outputFormats, cropOutput);
}
/**
* Per-document crop / page-image destination. Never {@code null};
* defaults to {@link CropOutput#DISABLED}.
*/
public CropOutput getCropOutput() {
return cropOutput;
}
/**
* Creates a request to process all pages with default options.
*
* @param pdfBytes The PDF file bytes.
* @return A new HybridRequest for all pages with all output formats.
*/
public static HybridRequest allPages(byte[] pdfBytes) {
return new HybridRequest(pdfBytes, Collections.emptySet(), null);
}
/**
* Creates a request to process all pages with specified output formats.
*
* @param pdfBytes The PDF file bytes.
* @param outputFormats The output formats to request.
* @return A new HybridRequest for all pages.
*/
public static HybridRequest allPages(byte[] pdfBytes, Set<OutputFormat> outputFormats) {
return new HybridRequest(pdfBytes, Collections.emptySet(), outputFormats);
}
/**
* Creates a request to process specific pages.
*
* @param pdfBytes The PDF file bytes.
* @param pageNumbers The 1-indexed page numbers to process.
* @return A new HybridRequest for the specified pages.
*/
public static HybridRequest forPages(byte[] pdfBytes, Set<Integer> pageNumbers) {
return new HybridRequest(pdfBytes, pageNumbers, null);
}
/**
* Creates a request to process specific pages with specified output formats.
*
* @param pdfBytes The PDF file bytes.
* @param pageNumbers The 1-indexed page numbers to process.
* @param outputFormats The output formats to request.
* @return A new HybridRequest for the specified pages.
*/
public static HybridRequest forPages(byte[] pdfBytes, Set<Integer> pageNumbers,
Set<OutputFormat> outputFormats) {
return new HybridRequest(pdfBytes, pageNumbers, outputFormats);
}
public byte[] getPdfBytes() {
return pdfBytes != null ? Arrays.copyOf(pdfBytes, pdfBytes.length) : null;
}
public Set<Integer> getPageNumbers() {
return pageNumbers;
}
/**
* Returns the output formats to request from the backend.
*
* @return Set of output formats. Never empty.
*/
public Set<OutputFormat> getOutputFormats() {
return outputFormats;
}
/**
* Checks if JSON output is requested.
*
* @return true if JSON format is included.
*/
public boolean wantsJson() {
return outputFormats.contains(OutputFormat.JSON);
}
/**
* Checks if Markdown output is requested.
*
* @return true if Markdown format is included.
*/
public boolean wantsMarkdown() {
return outputFormats.contains(OutputFormat.MARKDOWN);
}
/**
* Checks if HTML output is requested.
*
* @return true if HTML format is included.
*/
public boolean wantsHtml() {
return outputFormats.contains(OutputFormat.HTML);
}
}
/**
* Response class containing parsed document content.
*/
final class HybridResponse {
private final String markdown;
private final String html;
private final JsonNode json;
private final Map<Integer, JsonNode> pageContents;
private final List<Integer> failedPages;
private final JsonNode timings;
/**
* Creates a new HybridResponse.
*
* @param markdown The markdown representation of the document.
* @param html The HTML representation of the document.
* @param json The full structured JSON output (DoclingDocument format).
* @param pageContents Per-page JSON content, keyed by 1-indexed page number.
* @param failedPages List of 1-indexed page numbers that failed during backend processing.
* @param timings Per-step pipeline timings from the hybrid server (may be null).
*/
public HybridResponse(String markdown, String html, JsonNode json,
Map<Integer, JsonNode> pageContents, List<Integer> failedPages,
JsonNode timings) {
this.markdown = markdown != null ? markdown : "";
this.html = html != null ? html : "";
this.json = json;
this.pageContents = pageContents != null ? pageContents : Collections.emptyMap();
this.failedPages = failedPages != null
? Collections.unmodifiableList(new ArrayList<>(failedPages))
: Collections.emptyList();
this.timings = timings;
}
/**
* Creates a new HybridResponse (without timings, backward compatible).
*/
public HybridResponse(String markdown, String html, JsonNode json,
Map<Integer, JsonNode> pageContents, List<Integer> failedPages) {
this(markdown, html, json, pageContents, failedPages, null);
}
/**
* Creates a new HybridResponse (backward compatible constructor).
*
* @param markdown The markdown representation of the document.
* @param html The HTML representation of the document.
* @param json The full structured JSON output (DoclingDocument format).
* @param pageContents Per-page JSON content, keyed by 1-indexed page number.
*/
public HybridResponse(String markdown, String html, JsonNode json, Map<Integer, JsonNode> pageContents) {
this(markdown, html, json, pageContents, Collections.emptyList());
}
/**
* Creates a new HybridResponse (backward compatible constructor).
*
* @param markdown The markdown representation of the document.
* @param json The full structured JSON output (DoclingDocument format).
* @param pageContents Per-page JSON content, keyed by 1-indexed page number.
*/
public HybridResponse(String markdown, JsonNode json, Map<Integer, JsonNode> pageContents) {
this(markdown, "", json, pageContents, Collections.emptyList());
}
/**
* Creates an empty response.
*
* @return A new HybridResponse with empty/null values.
*/
public static HybridResponse empty() {
return new HybridResponse("", "", null, Collections.emptyMap());
}
public String getMarkdown() {
return markdown;
}
public String getHtml() {
return html;
}
public JsonNode getJson() {
return json;
}
public Map<Integer, JsonNode> getPageContents() {
return pageContents;
}
/**
* Returns per-step pipeline timings from the hybrid server.
*
* <p>Keys are step names (e.g. "layout", "ocr", "table_structure",
* "picture_description"). Each value contains "total_s", "avg_s", and "count".
*
* @return Timings JSON node, or null if profiling was not enabled on the server.
*/
public JsonNode getTimings() {
return timings;
}
/**
* Returns the list of 1-indexed page numbers that failed during backend processing.
*
* <p>When the backend returns partial_success, some pages may have failed due to
* issues like invalid code points in PDF font encoding. These pages can be retried
* via the Java processing path as a fallback.
*
* @return List of failed page numbers (1-indexed), or empty list if all pages succeeded.
*/
public List<Integer> getFailedPages() {
return failedPages;
}
/**
* Returns whether the backend reported any failed pages.
*
* @return true if at least one page failed during backend processing.
*/
public boolean hasFailedPages() {
return !failedPages.isEmpty();
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
HybridResponse that = (HybridResponse) o;
return Objects.equals(markdown, that.markdown) &&
Objects.equals(html, that.html) &&
Objects.equals(json, that.json) &&
Objects.equals(pageContents, that.pageContents) &&
Objects.equals(failedPages, that.failedPages) &&
Objects.equals(timings, that.timings);
}
@Override
public int hashCode() {
return Objects.hash(markdown, html, json, pageContents, failedPages, timings);
}
}
/**
* Checks if the backend server is available and ready to accept requests.
*
* <p>This performs a lightweight health check (e.g., HTTP GET to /health) with a short
* timeout to verify connectivity before sending actual conversion requests.
*
* @throws IOException If the server is unreachable or not ready.
*/
void checkAvailability() throws IOException;
/**
* Converts a PDF document synchronously.
*
* @param request The conversion request containing PDF bytes and options.
* @return The conversion response with parsed content.
* @throws IOException If an I/O error occurs during the request.
*/
HybridResponse convert(HybridRequest request) throws IOException;
/**
* Converts a PDF document asynchronously.
*
* <p>This method is useful for parallel processing where multiple pages
* can be processed concurrently with the Java backend.
*
* @param request The conversion request containing PDF bytes and options.
* @return A CompletableFuture that completes with the conversion response.
*/
CompletableFuture<HybridResponse> convertAsync(HybridRequest request);
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HybridClientFactory.java
+188
View File
@@ -0,0 +1,188 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* Factory for creating and managing hybrid client instances.
*
* <p>This factory provides a central point for instantiating HybridClient
* implementations based on the specified backend type. Clients are cached
* and reused to avoid creating multiple thread pools per document.
*
* <p>Supported backends:
* <ul>
* <li>{@code docling-fast} - Optimized docling SDK server</li>
* </ul>
*
* <p>Future backends (not yet implemented):
* <ul>
* <li>{@code hancom} - Hancom document parsing service</li>
* <li>{@code azure} - Azure Document Intelligence</li>
* <li>{@code google} - Google Document AI</li>
* </ul>
*
* @see HybridClient
* @see HybridConfig
*/
public class HybridClientFactory {
/** Backend type constant for Docling Fast Server. */
public static final String BACKEND_DOCLING_FAST = "docling-fast";
/** Backend type constant for Hancom (Studio Lite API). */
public static final String BACKEND_HANCOM = "hancom";
/** Backend type constant for Hancom AI (HOCR SDK individual modules). */
public static final String BACKEND_HANCOM_AI = "hancom-ai";
/** Backend type constant for Azure (not yet implemented). */
public static final String BACKEND_AZURE = "azure";
/** Backend type constant for Google (not yet implemented). */
public static final String BACKEND_GOOGLE = "google";
/** Cache of created clients, keyed by backend type. */
private static final Map<String, HybridClient> CLIENT_CACHE = new ConcurrentHashMap<>();
private HybridClientFactory() {
// Private constructor to prevent instantiation
}
/**
* Gets or creates a hybrid client for the specified backend.
*
* <p>Clients are cached and reused across multiple documents to avoid
* creating new thread pools for each document. Call {@link #shutdown()}
* when processing is complete to release resources.
*
* @param hybrid The backend type (e.g., "docling", "hancom", "azure", "google").
* @param config The configuration for the hybrid client.
* @return A HybridClient instance for the specified backend.
* @throws IllegalArgumentException If the backend type is unknown or not supported.
*/
public static HybridClient getOrCreate(String hybrid, HybridConfig config) {
if (hybrid == null || hybrid.isEmpty()) {
throw new IllegalArgumentException("Hybrid backend type cannot be null or empty");
}
String lowerHybrid = hybrid.toLowerCase();
return CLIENT_CACHE.computeIfAbsent(lowerHybrid, key -> createClient(key, config));
}
/**
* Creates a new hybrid client instance.
*/
private static HybridClient createClient(String hybrid, HybridConfig config) {
if (BACKEND_DOCLING_FAST.equals(hybrid)) {
return new DoclingFastServerClient(config);
} else if (BACKEND_HANCOM.equals(hybrid)) {
return new HancomClient(config);
} else if (BACKEND_HANCOM_AI.equals(hybrid)) {
return new HancomAIClient(config);
} else if (BACKEND_AZURE.equals(hybrid)) {
throw new UnsupportedOperationException("Azure Document Intelligence backend is not yet implemented");
} else if (BACKEND_GOOGLE.equals(hybrid)) {
throw new UnsupportedOperationException("Google Document AI backend is not yet implemented");
} else {
throw new IllegalArgumentException("Unknown hybrid backend: " + hybrid +
". Supported backends: " + getSupportedBackends());
}
}
/**
* Creates a hybrid client for the specified backend.
*
* @param hybrid The backend type (e.g., "docling", "hancom", "azure", "google").
* @param config The configuration for the hybrid client.
* @return A new HybridClient instance for the specified backend.
* @throws IllegalArgumentException If the backend type is unknown or not supported.
* @deprecated Use {@link #getOrCreate(String, HybridConfig)} instead to reuse clients.
*/
@Deprecated
public static HybridClient create(String hybrid, HybridConfig config) {
return getOrCreate(hybrid, config);
}
/**
* Creates a hybrid client for the specified backend with default configuration.
*
* @param hybrid The backend type (e.g., "docling").
* @return A new HybridClient instance for the specified backend.
* @throws IllegalArgumentException If the backend type is unknown or not supported.
* @deprecated Use {@link #getOrCreate(String, HybridConfig)} instead to reuse clients.
*/
@Deprecated
public static HybridClient create(String hybrid) {
return getOrCreate(hybrid, new HybridConfig());
}
/**
* Shuts down all cached clients and releases resources.
*
* <p>This method should be called when all processing is complete,
* typically at the end of the CLI main method.
*/
public static void shutdown() {
for (HybridClient client : CLIENT_CACHE.values()) {
if (client instanceof DoclingFastServerClient) {
((DoclingFastServerClient) client).shutdown();
} else if (client instanceof HancomClient) {
((HancomClient) client).shutdown();
} else if (client instanceof HancomAIClient) {
((HancomAIClient) client).shutdown();
}
}
CLIENT_CACHE.clear();
}
/**
* Checks if a backend type is supported and implemented.
*
* @param hybrid The backend type to check.
* @return true if the backend is supported and implemented, false otherwise.
*/
public static boolean isSupported(String hybrid) {
if (hybrid == null || hybrid.isEmpty()) {
return false;
}
String lowerHybrid = hybrid.toLowerCase();
return BACKEND_DOCLING_FAST.equals(lowerHybrid) || BACKEND_HANCOM.equals(lowerHybrid)
|| BACKEND_HANCOM_AI.equals(lowerHybrid);
}
/**
* Gets a comma-separated list of supported backend types.
*
* @return A string listing all supported backends.
*/
public static String getSupportedBackends() {
return String.join(", ", BACKEND_DOCLING_FAST, BACKEND_HANCOM, BACKEND_HANCOM_AI);
}
/**
* Gets a comma-separated list of all known backend types (including not yet implemented).
*
* @return A string listing all known backends.
*/
public static String getAllKnownBackends() {
return String.join(", ", BACKEND_DOCLING_FAST, BACKEND_HANCOM, BACKEND_AZURE, BACKEND_GOOGLE);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HybridConfig.java
+379
View File
@@ -0,0 +1,379 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
/**
* Configuration class for hybrid PDF processing with external AI backends.
*
* <p>Hybrid processing routes pages to either Java-based processing or external
* AI backends (like docling, hancom, azure, google) based on page triage decisions.
*/
public class HybridConfig {
/** Default timeout for backend requests in milliseconds. */
public static final int DEFAULT_TIMEOUT_MS = 0;
/** Default maximum concurrent requests to the backend. */
public static final int DEFAULT_MAX_CONCURRENT_REQUESTS = 4;
/** Default URL for docling-serve. */
public static final String DOCLING_DEFAULT_URL = "http://localhost:5001";
/** Default URL for docling-fast-server. */
public static final String DOCLING_FAST_DEFAULT_URL = "http://localhost:5002";
/** Default URL for Hancom Document AI API. */
public static final String HANCOM_DEFAULT_URL = "https://dataloader.cloud.hancom.com/studio-lite/api";
/** Default URL for Hancom AI HOCR SDK API. */
public static final String HANCOM_AI_DEFAULT_URL = "http://localhost:18008/api/v1";
private String url;
private int timeoutMs = DEFAULT_TIMEOUT_MS;
private boolean fallbackToJava = false;
private int maxConcurrentRequests = DEFAULT_MAX_CONCURRENT_REQUESTS;
/** Hybrid triage mode: auto (dynamic triage based on page content). */
public static final String MODE_AUTO = "auto";
/** Hybrid triage mode: full (skip triage, send all pages to backend). */
public static final String MODE_FULL = "full";
private String mode = MODE_AUTO;
/** Regionlist strategy: table-first (default) — check TSR overlap, skip if TSR exists. */
public static final String REGIONLIST_TABLE_FIRST = "table-first";
/** Regionlist strategy: list-only — always treat label 7 as list, skip TSR check. */
public static final String REGIONLIST_LIST_ONLY = "list-only";
private String regionlistStrategy = REGIONLIST_TABLE_FIRST;
/** OCR strategy: off (stream only, no OCR fallback). */
public static final String OCR_OFF = "off";
/** OCR strategy: auto (stream first, OCR fallback when enrichment fails). */
public static final String OCR_AUTO = "auto";
/** OCR strategy: force (OCR only, skip stream-based enrichment). */
public static final String OCR_FORCE = "force";
private String ocrStrategy = OCR_AUTO;
/** Page image cache strategy: "memory" (default) or "disk". */
private String imageCache = "memory";
/** Whether to save cropped figure images to disk for debugging. */
private boolean saveCrops = false;
/** Output directory for saved crops (set by CLI when --save-crops is used). */
private String cropOutputDir = null;
/**
* Default constructor initializing the configuration with default values.
*/
public HybridConfig() {
}
/**
* Gets the backend server URL.
*
* @return The backend URL, or null if using default for the backend type.
*/
public String getUrl() {
return url;
}
/**
* Sets the backend server URL.
*
* @param url The backend URL to use.
*/
public void setUrl(String url) {
this.url = url;
}
/**
* Gets the request timeout in milliseconds.
*
* @return The timeout in milliseconds.
*/
public int getTimeoutMs() {
return timeoutMs;
}
/**
* Sets the request timeout in milliseconds. Use 0 for no timeout.
*
* @param timeoutMs The timeout in milliseconds (0 = no timeout).
* @throws IllegalArgumentException if timeout is negative.
*/
public void setTimeoutMs(int timeoutMs) {
if (timeoutMs < 0) {
throw new IllegalArgumentException("Timeout must be non-negative: " + timeoutMs);
}
this.timeoutMs = timeoutMs;
}
/**
* Checks if fallback to Java processing is enabled when backend fails.
*
* @return true if fallback is enabled, false otherwise.
*/
public boolean isFallbackToJava() {
return fallbackToJava;
}
/**
* Sets whether to fallback to Java processing when backend fails.
*
* @param fallbackToJava true to enable fallback, false to fail on backend error.
*/
public void setFallbackToJava(boolean fallbackToJava) {
this.fallbackToJava = fallbackToJava;
}
/**
* Gets the maximum number of concurrent requests to the backend.
*
* @return The maximum concurrent requests.
*/
public int getMaxConcurrentRequests() {
return maxConcurrentRequests;
}
/**
* Sets the maximum number of concurrent requests to the backend.
*
* @param maxConcurrentRequests The maximum concurrent requests.
* @throws IllegalArgumentException if the value is not positive.
*/
public void setMaxConcurrentRequests(int maxConcurrentRequests) {
if (maxConcurrentRequests <= 0) {
throw new IllegalArgumentException("Max concurrent requests must be positive: " + maxConcurrentRequests);
}
this.maxConcurrentRequests = maxConcurrentRequests;
}
/**
* Gets the default URL for a given hybrid backend.
*
* @param hybrid The hybrid backend name (docling, docling-fast, hancom, azure, google).
* @return The default URL, or null if the backend requires explicit URL.
*/
public static String getDefaultUrl(String hybrid) {
if (hybrid == null) {
return null;
}
String lowerHybrid = hybrid.toLowerCase();
// Both "docling" and "docling-fast" (deprecated) use the same server
if ("docling".equals(lowerHybrid) || "docling-fast".equals(lowerHybrid)) {
return DOCLING_FAST_DEFAULT_URL;
}
if ("hancom".equals(lowerHybrid)) {
return HANCOM_DEFAULT_URL;
}
if ("hancom-ai".equals(lowerHybrid)) {
return HANCOM_AI_DEFAULT_URL;
}
// azure, google require explicit URL
return null;
}
/**
* Gets the effective URL for a given hybrid backend.
* Returns the configured URL if set, otherwise returns the default URL for the backend.
*
* @param hybrid The hybrid backend name.
* @return The effective URL to use for the backend.
*/
public String getEffectiveUrl(String hybrid) {
if (url != null && !url.isEmpty()) {
return url;
}
return getDefaultUrl(hybrid);
}
/**
* Gets the hybrid triage mode.
*
* @return The mode (auto or full).
*/
public String getMode() {
return mode;
}
/**
* Sets the hybrid triage mode.
*
* @param mode The mode (auto or full).
*/
public void setMode(String mode) {
this.mode = mode;
}
/**
* Checks if full mode is enabled (skip triage, send all pages to backend).
*
* @return true if mode is full, false otherwise.
*/
public boolean isFullMode() {
return MODE_FULL.equals(mode);
}
/**
* Gets the regionlist strategy for label 7 (Table region) handling.
*
* @return The regionlist strategy (table-first or list-only).
*/
public String getRegionlistStrategy() {
return regionlistStrategy;
}
/**
* Sets the regionlist strategy for label 7 (Table region) handling.
*
* <ul>
* <li>{@code "table-first"} (default): check TSR overlap, skip if TSR exists, else treat as list</li>
* <li>{@code "list-only"}: always treat as list, skip TSR check entirely</li>
* </ul>
*
* @param regionlistStrategy The regionlist strategy to use.
*/
public void setRegionlistStrategy(String regionlistStrategy) {
if (regionlistStrategy != null
&& !REGIONLIST_TABLE_FIRST.equals(regionlistStrategy)
&& !REGIONLIST_LIST_ONLY.equals(regionlistStrategy)) {
throw new IllegalArgumentException("Invalid regionlistStrategy: "
+ regionlistStrategy + " (expected " + REGIONLIST_TABLE_FIRST
+ " or " + REGIONLIST_LIST_ONLY + ")");
}
this.regionlistStrategy = regionlistStrategy;
}
/**
* Checks if regionlist strategy is list-only (always treat label 7 as list).
*
* @return true if strategy is list-only, false otherwise.
*/
public boolean isRegionlistListOnly() {
return REGIONLIST_LIST_ONLY.equals(regionlistStrategy);
}
/**
* Gets the page image cache strategy.
*
* @return "memory" or "disk".
*/
public String getImageCache() {
return imageCache;
}
/**
* Sets the page image cache strategy.
*
* @param imageCache "memory" (in-heap HashMap) or "disk" (temp PNG files).
*/
public void setImageCache(String imageCache) {
if (imageCache != null
&& !"memory".equals(imageCache) && !"disk".equals(imageCache)) {
throw new IllegalArgumentException("Invalid imageCache: "
+ imageCache + " (expected \"memory\" or \"disk\")");
}
this.imageCache = imageCache;
}
/**
* Checks if cropped figure images should be saved to disk.
*
* @return true if save-crops is enabled.
*/
public boolean isSaveCrops() {
return saveCrops;
}
/**
* Sets whether to save cropped figure images to disk.
*
* @param saveCrops true to save crops.
*/
public void setSaveCrops(boolean saveCrops) {
this.saveCrops = saveCrops;
}
/**
* Gets the output directory for saved crops.
*
* @return the crop output directory path, or null if not set.
*/
public String getCropOutputDir() {
return cropOutputDir;
}
/**
* Sets the output directory for saved crops.
*
* @param cropOutputDir the directory path.
*/
public void setCropOutputDir(String cropOutputDir) {
this.cropOutputDir = cropOutputDir;
}
/**
* Gets the OCR strategy for enrichment fallback.
*
* @return The OCR strategy (off, auto, or force).
*/
public String getOcrStrategy() {
return ocrStrategy;
}
/**
* Sets the OCR strategy for enrichment fallback.
*
* <ul>
* <li>{@code "off"}: stream-based enrichment only, no OCR fallback</li>
* <li>{@code "auto"} (default): try stream enrichment first, fall back to OCR words when no match</li>
* <li>{@code "force"}: skip stream enrichment, always use OCR words</li>
* </ul>
*
* @param ocrStrategy The OCR strategy to use.
*/
public void setOcrStrategy(String ocrStrategy) {
if (ocrStrategy != null
&& !OCR_OFF.equals(ocrStrategy)
&& !OCR_AUTO.equals(ocrStrategy)
&& !OCR_FORCE.equals(ocrStrategy)) {
throw new IllegalArgumentException("Invalid ocrStrategy: "
+ ocrStrategy + " (expected " + OCR_OFF + ", " + OCR_AUTO
+ ", or " + OCR_FORCE + ")");
}
this.ocrStrategy = ocrStrategy;
}
/**
* Checks if OCR strategy is auto (stream first, OCR fallback).
*
* @return true if strategy is auto, false otherwise.
*/
public boolean isOcrAuto() {
return OCR_AUTO.equals(ocrStrategy);
}
/**
* Checks if OCR strategy is force (OCR only).
*
* @return true if strategy is force, false otherwise.
*/
public boolean isOcrForce() {
return OCR_FORCE.equals(ocrStrategy);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/HybridSchemaTransformer.java
+110
View File
@@ -0,0 +1,110 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.hybrid.HybridClient.HybridResponse;
import org.verapdf.wcag.algorithms.entities.IObject;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
/**
* Interface for transforming hybrid backend responses to IObject hierarchy.
*
* <p>Implementations of this interface convert backend-specific JSON output
* (e.g., Docling's DoclingDocument format) to the OpenDataLoader IObject
* structure that downstream processors and generators expect.
*
* <p>The transformer ensures schema compatibility between different backends
* and the Java processing path, allowing seamless integration of results.
*/
public interface HybridSchemaTransformer {
/**
* Transforms a hybrid backend response to a list of IObjects per page.
*
* <p>The returned structure matches the format expected by downstream
* processors: a list indexed by page number (0-based), where each entry
* contains the IObjects for that page.
*
* @param response The hybrid backend response containing JSON output.
* @param pageHeights Map of page number (1-indexed) to page height in PDF points.
* Used for coordinate transformation if needed.
* @return A list of IObject lists, one per page (0-indexed).
*/
List<List<IObject>> transform(HybridResponse response, Map<Integer, Double> pageHeights);
/**
* Transforms per-page JSON content to IObjects for a specific page.
*
* <p>This method is useful when processing pages individually or when
* the backend provides separate responses per page.
*
* @param pageNumber The 1-indexed page number.
* @param pageContent The JSON content for the page.
* @param pageHeight The page height in PDF points.
* @return A list of IObjects for the specified page.
*/
List<IObject> transformPage(int pageNumber, JsonNode pageContent, double pageHeight);
/**
* Returns the backend type this transformer handles.
*
* @return The backend name (e.g., "docling", "hancom").
*/
String getBackendType();
/**
* Returns per-element metadata produced during the last {@link #transform} call.
* Keys are {@code IObject.recognizedStructureId} values.
*
* @return unmodifiable map of element metadata, empty by default
*/
default Map<Long, ElementMetadata> getElementMetadata() {
return Collections.emptyMap();
}
/**
* Re-key the in-memory {@link ElementMetadata} map after the host
* pipeline renumbered structure IDs. {@code oldToNew} maps the
* transformer-assigned ID to the post-{@code setIDs} ID for IObjects
* the caller just renumbered; entries for unmapped IDs are left alone.
*
* <p>Backends that don't expose mutable metadata should leave the
* default no-op; their downstream metadata lookups will continue to
* miss.
*/
default void rekeyMetadata(Map<Long, Long> oldToNew) {
// no-op
}
/**
* Returns per-page OCR word data collected during the last {@link #transform} call.
* Keys are 0-indexed page numbers. Each list contains word-level OCR data with
* text and bounding box in PDF coordinate space.
*
* <p>Only populated by backends that perform OCR (e.g., hancom-ai).
* Used by OCR enrichment fallback when Java TextChunks are not available.
*
* @return unmodifiable map of page number to OCR words, empty by default
*/
default Map<Integer, List<OcrWordInfo>> getOcrWordsByPage() {
return Collections.emptyMap();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/MemoryPageImageCache.java
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
/**
* In-memory page image cache. Stores images in a HashMap; evict() removes
* entries so GC can reclaim memory (~25MB per page image).
*/
public class MemoryPageImageCache implements PageImageCache {
private final Map<Integer, BufferedImage> cache = new HashMap<>();
@Override
public BufferedImage getOrFetch(int pageIndex, PageImageFetcher fetcher) throws IOException {
BufferedImage image = cache.get(pageIndex);
if (image == null) {
image = fetcher.fetch(pageIndex);
if (image == null) {
throw new IOException("Page image fetcher returned null for page " + pageIndex);
}
cache.put(pageIndex, image);
}
return image;
}
@Override
public void evict(int pageIndex) {
cache.remove(pageIndex);
}
@Override
public void close() {
cache.clear();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/OcrWordInfo.java
+54
View File
@@ -0,0 +1,54 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
/**
* Holds OCR-recognized text and its bounding box in PDF coordinate space.
*
* <p>Used to preserve word-level OCR data from the DLA+OCR pipeline so that
* the enrichment step can create invisible text operators when Java TextChunks
* are not available (e.g., scanned/image-based pages).
*/
public class OcrWordInfo {
private final String text;
private final BoundingBox bbox;
public OcrWordInfo(String text, BoundingBox bbox) {
this.text = text;
this.bbox = bbox;
}
/**
* Gets the OCR-recognized text for this word.
*
* @return the recognized text
*/
public String getText() {
return text;
}
/**
* Gets the bounding box in PDF coordinate space (72 DPI, bottom-left origin).
*
* @return the bounding box
*/
public BoundingBox getBbox() {
return bbox;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/PageImageCache.java
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import java.awt.image.BufferedImage;
import java.io.IOException;
/**
* Cache for page images (pdf2img results). Implementations control where
* images are stored and when they are evicted.
*/
public interface PageImageCache extends AutoCloseable {
/**
* Get a cached page image or fetch it using the supplier.
*
* @param pageIndex 0-indexed page number
* @param fetcher called if image not in cache; may throw IOException
* @return the page image, never null
* @throws IOException if the fetcher fails
*/
BufferedImage getOrFetch(int pageIndex, PageImageFetcher fetcher) throws IOException;
/**
* Hint that this page is no longer needed.
* Memory impl evicts immediately; disk impl keeps for potential re-read.
*
* @param pageIndex 0-indexed page number
*/
void evict(int pageIndex);
/**
* Clean up all resources. Disk impl deletes temp files.
*/
@Override
void close() throws IOException;
/**
* Functional interface for fetching a page image on cache miss.
*/
@FunctionalInterface
interface PageImageFetcher {
BufferedImage fetch(int pageIndex) throws IOException;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TextSimilarity.java
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
/**
* Computes text similarity for stream vs OCR comparison.
* Used by enrichment to decide whether stream text is corrupted.
*/
public final class TextSimilarity {
/** Default similarity threshold. Below this, stream text is considered corrupted. */
public static final double DEFAULT_THRESHOLD = 0.5;
private TextSimilarity() {}
/**
* Computes normalized similarity between two strings (0.0 = completely different, 1.0 = identical).
* Uses Levenshtein distance normalized by the longer string's length.
*/
public static double similarity(String a, String b) {
if (a == null || b == null) return 0.0;
if (a.equals(b)) return 1.0;
if (a.isEmpty() || b.isEmpty()) return 0.0;
int distance = levenshteinDistance(a, b);
int maxLen = Math.max(a.length(), b.length());
return 1.0 - ((double) distance / maxLen);
}
/**
* Returns true if stream text should be trusted over OCR text.
*/
public static boolean trustStream(String streamText, String ocrText, double threshold) {
if (streamText == null || streamText.isEmpty()) return false;
if (ocrText == null || ocrText.isEmpty()) return true;
return similarity(streamText, ocrText) >= threshold;
}
private static int levenshteinDistance(String a, String b) {
int[] prev = new int[b.length() + 1];
int[] curr = new int[b.length() + 1];
for (int j = 0; j <= b.length(); j++) prev[j] = j;
for (int i = 1; i <= a.length(); i++) {
curr[0] = i;
for (int j = 1; j <= b.length(); j++) {
int cost = a.charAt(i - 1) == b.charAt(j - 1) ? 0 : 1;
curr[j] = Math.min(Math.min(curr[j - 1] + 1, prev[j] + 1), prev[j - 1] + cost);
}
int[] temp = prev; prev = curr; curr = temp;
}
return prev[b.length()];
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TriageLogger.java
+225
View File
@@ -0,0 +1,225 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.hybrid;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.opendataloader.pdf.hybrid.TriageProcessor.TriageDecision;
import org.opendataloader.pdf.hybrid.TriageProcessor.TriageResult;
import org.opendataloader.pdf.hybrid.TriageProcessor.TriageSignals;
import java.io.IOException;
import java.io.Writer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
/**
* Logger for triage decisions to JSON format for benchmark evaluation.
*
* <p>Output format:
* <pre>
* {
* "document": "example.pdf",
* "hybrid": "docling",
* "triage": [
* {
* "page": 1,
* "decision": "JAVA",
* "confidence": 0.95,
* "signals": {
* "lineChunkCount": 2,
* "textChunkCount": 45,
* "lineToTextRatio": 0.04,
* "alignedLineGroups": 0,
* "hasTableBorder": false,
* "hasSuspiciousPattern": false
* }
* }
* ],
* "summary": {
* "totalPages": 10,
* "javaPages": 8,
* "backendPages": 2
* }
* }
* </pre>
*/
public class TriageLogger {
private static final Logger LOGGER = Logger.getLogger(TriageLogger.class.getCanonicalName());
/** Default filename for triage log output. */
public static final String DEFAULT_FILENAME = "triage.json";
private final ObjectMapper objectMapper;
/**
* Creates a new TriageLogger with default settings.
*/
public TriageLogger() {
this.objectMapper = new ObjectMapper();
this.objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
}
/**
* Logs triage results to a JSON file.
*
* @param outputDir The output directory path.
* @param documentName The name of the processed document.
* @param hybridBackend The hybrid backend used (e.g., "docling").
* @param triageResults Map of page number to triage result.
* @throws IOException If writing the file fails.
*/
public void logToFile(
Path outputDir,
String documentName,
String hybridBackend,
Map<Integer, TriageResult> triageResults) throws IOException {
Path outputPath = outputDir.resolve(DEFAULT_FILENAME);
Files.createDirectories(outputDir);
ObjectNode root = createTriageJson(documentName, hybridBackend, triageResults);
try (Writer writer = Files.newBufferedWriter(outputPath)) {
objectMapper.writeValue(writer, root);
}
LOGGER.log(Level.INFO, "Triage log written to {0}", outputPath);
}
/**
* Writes triage results to a Writer.
*
* @param writer The Writer to write to.
* @param documentName The name of the processed document.
* @param hybridBackend The hybrid backend used (e.g., "docling").
* @param triageResults Map of page number to triage result.
* @throws IOException If writing fails.
*/
public void logToWriter(
Writer writer,
String documentName,
String hybridBackend,
Map<Integer, TriageResult> triageResults) throws IOException {
ObjectNode root = createTriageJson(documentName, hybridBackend, triageResults);
objectMapper.writeValue(writer, root);
}
/**
* Creates the triage JSON structure.
*
* @param documentName The name of the processed document.
* @param hybridBackend The hybrid backend used.
* @param triageResults Map of page number to triage result.
* @return The root ObjectNode containing all triage data.
*/
public ObjectNode createTriageJson(
String documentName,
String hybridBackend,
Map<Integer, TriageResult> triageResults) {
ObjectNode root = objectMapper.createObjectNode();
root.put("document", documentName);
root.put("hybrid", hybridBackend);
// Create triage array
ArrayNode triageArray = objectMapper.createArrayNode();
int javaCount = 0;
int backendCount = 0;
List<Map.Entry<Integer, TriageResult>> sortedEntries = triageResults.entrySet().stream()
.sorted(Map.Entry.comparingByKey())
.collect(Collectors.toList());
for (Map.Entry<Integer, TriageResult> entry : sortedEntries) {
int pageNumber = entry.getKey();
TriageResult result = entry.getValue();
ObjectNode pageNode = objectMapper.createObjectNode();
pageNode.put("page", pageNumber + 1); // Convert to 1-indexed for output
pageNode.put("decision", result.getDecision().name());
pageNode.put("confidence", result.getConfidence());
// Add signals
ObjectNode signalsNode = createSignalsNode(result.getSignals());
pageNode.set("signals", signalsNode);
triageArray.add(pageNode);
// Count decisions
if (result.getDecision() == TriageDecision.JAVA) {
javaCount++;
} else {
backendCount++;
}
}
root.set("triage", triageArray);
// Create summary
ObjectNode summaryNode = objectMapper.createObjectNode();
summaryNode.put("totalPages", triageResults.size());
summaryNode.put("javaPages", javaCount);
summaryNode.put("backendPages", backendCount);
root.set("summary", summaryNode);
return root;
}
/**
* Creates a JSON node for triage signals.
*
* @param signals The triage signals.
* @return The ObjectNode containing signal data.
*/
private ObjectNode createSignalsNode(TriageSignals signals) {
ObjectNode signalsNode = objectMapper.createObjectNode();
signalsNode.put("lineChunkCount", signals.getLineChunkCount());
signalsNode.put("textChunkCount", signals.getTextChunkCount());
signalsNode.put("lineToTextRatio", signals.getLineToTextRatio());
signalsNode.put("alignedLineGroups", signals.getAlignedLineGroups());
signalsNode.put("hasTableBorder", signals.hasTableBorder());
signalsNode.put("hasSuspiciousPattern", signals.hasSuspiciousPattern());
return signalsNode;
}
/**
* Converts triage results to JSON string.
*
* @param documentName The name of the processed document.
* @param hybridBackend The hybrid backend used.
* @param triageResults Map of page number to triage result.
* @return JSON string representation.
* @throws IOException If serialization fails.
*/
public String toJsonString(
String documentName,
String hybridBackend,
Map<Integer, TriageResult> triageResults) throws IOException {
ObjectNode root = createTriageJson(documentName, hybridBackend, triageResults);
return objectMapper.writeValueAsString(root);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/hybrid/TriageProcessor.java
+1132
View File
File diff suppressed because it is too large Load Diff
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/JsonName.java
+81
View File
@@ -0,0 +1,81 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json;
public class JsonName {
public static final String PAGE_NUMBER = "page number";
public static final String LEVEL = "level";
public static final String BOUNDING_BOX = "bounding box";
public static final String TYPE = "type";
public static final String ID = "id";
public static final String IMAGE_CHUNK_TYPE = "image";
public static final String LIST_ITEM_TYPE = "list item";
public static final String LINE_CHUNK_TYPE = "line";
public static final String CONTENT = "content";
public static final String HIDDEN_TEXT = "hidden text";
public static final String TEXT_CHUNK_TYPE = "text chunk";
public static final String TABLE_TYPE = "table";
public static final String TEXT_BLOCK = "text block";
public static final String LIST_TYPE = "list";
public static final String TOC_TYPE = "toc";
public static final String TOC_ITEM_TYPE = "toc item";
public static final String TABLE_CELL_TYPE = "table cell";
public static final String ROW_TYPE = "table row";
public static final String FONT_TYPE = "font";
public static final String FONT_SIZE = "font size";
public static final String TEXT_COLOR = "text color";
public static final String PARAGRAPH_TYPE = "paragraph";
public static final String HEADING_TYPE = "heading";
public static final String CAPTION_TYPE = "caption";
public static final String KIDS = "kids";
public static final String LIST_ITEMS = "list items";
public static final String TOC_ITEMS = "toc items";
public static final String NUMBER_OF_LIST_ITEMS = "number of list items";
public static final String PREVIOUS_LIST_ID = "previous list id";
public static final String NEXT_LIST_ID = "next list id";
public static final String PREVIOUS_TOC_ID = "previous toc id";
public static final String NEXT_TOC_ID = "next toc id";
public static final String PREVIOUS_TABLE_ID = "previous table id";
public static final String NEXT_TABLE_ID = "next table id";
public static final String AUTHOR = "author";
public static final String TITLE = "title";
public static final String COLUMN_NUMBER = "column number";
public static final String ROW_NUMBER = "row number";
public static final String COLUMN_SPAN = "column span";
public static final String ROW_SPAN = "row span";
public static final String IS_HEADER = "is_header";
public static final String NUMBER_OF_ROWS = "number of rows";
public static final String NUMBER_OF_COLUMNS = "number of columns";
public static final String NUMBER_OF_PAGES = "number of pages";
public static final String FILE_NAME = "file name";
public static final String HEADING_LEVEL = "heading level";
public static final String CREATION_DATE = "creation date";
public static final String MODIFICATION_DATE = "modification date";
public static final String ROWS = "rows";
public static final String CELLS = "cells";
public static final String NUMBERING_STYLE = "numbering style";
public static final String SOURCE = "source";
public static final String DATA = "data";
public static final String IMAGE_FORMAT = "format";
public static final String FORMULA_TYPE = "formula";
public static final String DESCRIPTION = "description";
public static final String ALT = "alt";
public static final String ALT_SOURCE = "alt_source";
public static final String AI_SCORE = "ai_score";
public static final String PDFUA_TAG = "pdfua_tag";
public static final String SOURCE_LABEL = "source label";
public static final String HYBRID = "hybrid";
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/JsonWriter.java
+134
View File
@@ -0,0 +1,134 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json;
import com.fasterxml.jackson.core.JsonEncoding;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.verapdf.as.ASAtom;
import org.verapdf.cos.COSDictionary;
import org.verapdf.cos.COSObjType;
import org.verapdf.cos.COSObject;
import org.verapdf.cos.COSTrailer;
import org.verapdf.gf.model.impl.cos.GFCosInfo;
import org.verapdf.pd.PDDocument;
import org.verapdf.tools.StaticResources;
import org.opendataloader.pdf.hybrid.ElementMetadata;
import org.opendataloader.pdf.json.serializers.SerializerUtil;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeaderOrFooter;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
public class JsonWriter {
private static final Logger LOGGER = Logger.getLogger(JsonWriter.class.getCanonicalName());
private static JsonGenerator getJsonGenerator(String fileName) throws IOException {
JsonFactory jsonFactory = new JsonFactory();
return jsonFactory.createGenerator(new File(fileName), JsonEncoding.UTF8)
.setPrettyPrinter(new DefaultPrettyPrinter())
.setCodec(ObjectMapperHolder.getObjectMapper());
}
public static void writeToJson(File inputPDF, String outputFolder, List<List<IObject>> contents) throws IOException {
writeToJson(inputPDF, outputFolder, contents, Collections.emptyMap(), null);
}
public static void writeToJson(File inputPDF, String outputFolder, List<List<IObject>> contents,
Map<Long, ElementMetadata> elementMetadata) throws IOException {
writeToJson(inputPDF, outputFolder, contents, elementMetadata, null);
}
public static void writeToJson(File inputPDF, String outputFolder, List<List<IObject>> contents,
Map<Long, ElementMetadata> elementMetadata,
Map<String, Object> hybridInfo) throws IOException {
writeToJson(inputPDF, outputFolder, contents, elementMetadata, hybridInfo, false);
}
public static void writeToJson(File inputPDF, String outputFolder, List<List<IObject>> contents,
Map<Long, ElementMetadata> elementMetadata,
Map<String, Object> hybridInfo,
boolean includeHeaderFooter) throws IOException {
StaticLayoutContainers.resetImageIndex();
String jsonFileName = outputFolder + File.separator + inputPDF.getName().substring(0, inputPDF.getName().length() - 3) + "json";
try (JsonGenerator jsonGenerator = getJsonGenerator(jsonFileName)) {
jsonGenerator.writeStartObject();
writeDocumentInfo(jsonGenerator, inputPDF.getName());
if (hybridInfo != null && !hybridInfo.isEmpty()) {
writeHybridBlock(jsonGenerator, hybridInfo);
}
SerializerUtil.setElementMetadata(elementMetadata);
try {
jsonGenerator.writeArrayFieldStart(JsonName.KIDS);
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
for (IObject content : contents.get(pageNumber)) {
if (content instanceof LineArtChunk) {
continue;
}
if (!includeHeaderFooter && content instanceof SemanticHeaderOrFooter) {
continue;
}
jsonGenerator.writePOJO(content);
}
}
jsonGenerator.writeEndArray();
} finally {
SerializerUtil.clearElementMetadata();
}
jsonGenerator.writeEndObject();
LOGGER.log(Level.INFO, "Created {0}", jsonFileName);
} catch (Exception ex) {
LOGGER.log(Level.WARNING, "Unable to create JSON output: " + ex.getMessage());
}
}
private static void writeHybridBlock(JsonGenerator generator, Map<String, Object> hybridInfo) throws IOException {
generator.writeObjectFieldStart(JsonName.HYBRID);
for (Map.Entry<String, Object> entry : hybridInfo.entrySet()) {
generator.writePOJOField(entry.getKey(), entry.getValue());
}
generator.writeEndObject();
}
private static void writeDocumentInfo(JsonGenerator generator, String pdfName) throws IOException {
PDDocument document = StaticResources.getDocument();
generator.writeStringField(JsonName.FILE_NAME, pdfName);
generator.writeNumberField(JsonName.NUMBER_OF_PAGES, document.getNumberOfPages());
COSTrailer trailer = document.getDocument().getTrailer();
COSObject object = trailer.getKey(ASAtom.INFO);
GFCosInfo info = new GFCosInfo((COSDictionary)
(object != null && object.getType() == COSObjType.COS_DICT ?
object.getDirectBase() : COSDictionary.construct().get()));
generator.writeStringField(JsonName.AUTHOR, info.getAuthor() != null ? info.getAuthor() : info.getXMPCreator());
generator.writeStringField(JsonName.TITLE, info.getTitle() != null ? info.getTitle() : info.getXMPTitle());
generator.writeStringField(JsonName.CREATION_DATE, info.getCreationDate() != null ?
info.getCreationDate() : info.getXMPCreateDate());
generator.writeStringField(JsonName.MODIFICATION_DATE, info.getModDate() != null ?
info.getModDate() : info.getXMPModifyDate());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/ObjectMapperHolder.java
+103
View File
@@ -0,0 +1,103 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json;
import com.fasterxml.jackson.core.Version;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.json.serializers.*;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.*;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
public class ObjectMapperHolder {
private static final ObjectMapper objectMapper = new ObjectMapper();
static {
SimpleModule module = new SimpleModule("NodeSerializer", new Version(2, 1,
3, null, null, null));
TextChunkSerializer textChunkSerializer = new TextChunkSerializer(TextChunk.class);
module.addSerializer(TextChunk.class, textChunkSerializer);
TextLineSerializer textLineSerializer = new TextLineSerializer(TextLine.class);
module.addSerializer(TextLine.class, textLineSerializer);
ImageSerializer imageSerializer = new ImageSerializer(ImageChunk.class);
module.addSerializer(ImageChunk.class, imageSerializer);
TableSerializer tableSerializer = new TableSerializer(TableBorder.class);
module.addSerializer(TableBorder.class, tableSerializer);
TableCellSerializer tableCellSerializer = new TableCellSerializer(TableBorderCell.class);
module.addSerializer(TableBorderCell.class, tableCellSerializer);
ListSerializer listSerializer = new ListSerializer(PDFList.class);
module.addSerializer(PDFList.class, listSerializer);
ListItemSerializer listItemSerializer = new ListItemSerializer(ListItem.class);
module.addSerializer(ListItem.class, listItemSerializer);
TOCSerializer tocSerializer = new TOCSerializer(SemanticTOC.class);
module.addSerializer(SemanticTOC.class, tocSerializer);
TOCItemSerializer tocItemSerializer = new TOCItemSerializer(SemanticTOCI.class);
module.addSerializer(SemanticTOCI.class, tocItemSerializer);
LineChunkSerializer lineChunkSerializer = new LineChunkSerializer(LineChunk.class);
module.addSerializer(LineChunk.class, lineChunkSerializer);
SemanticTextNodeSerializer semanticTextNodeSerializer = new SemanticTextNodeSerializer(SemanticTextNode.class);
module.addSerializer(SemanticTextNode.class, semanticTextNodeSerializer);
TableRowSerializer tableRowSerializer = new TableRowSerializer(TableBorderRow.class);
module.addSerializer(TableBorderRow.class, tableRowSerializer);
HeadingSerializer headingSerializer = new HeadingSerializer(SemanticHeading.class);
module.addSerializer(SemanticHeading.class, headingSerializer);
CaptionSerializer captionSerializer = new CaptionSerializer(SemanticCaption.class);
module.addSerializer(SemanticCaption.class, captionSerializer);
DoubleSerializer doubleSerializer = new DoubleSerializer(Double.class);
module.addSerializer(Double.class, doubleSerializer);
HeaderFooterSerializer headerFooterSerializer = new HeaderFooterSerializer(SemanticHeaderOrFooter.class);
module.addSerializer(SemanticHeaderOrFooter.class, headerFooterSerializer);
FormulaSerializer formulaSerializer = new FormulaSerializer(SemanticFormula.class);
module.addSerializer(SemanticFormula.class, formulaSerializer);
PictureSerializer pictureSerializer = new PictureSerializer(SemanticPicture.class);
module.addSerializer(SemanticPicture.class, pictureSerializer);
//ParagraphSerializer paragraphSerializer = new ParagraphSerializer(SemanticParagraph.class);
//module.addSerializer(SemanticParagraph.class, paragraphSerializer);
objectMapper.registerModule(module);
}
public static ObjectMapper getObjectMapper() {
return objectMapper;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/CaptionSerializer.java
+53
View File
@@ -0,0 +1,53 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.SemanticCaption;
import java.io.IOException;
/**
* Jackson serializer for SemanticCaption objects.
* Serializes captions with their essential info and linked content ID.
*/
public class CaptionSerializer extends StdSerializer<SemanticCaption> {
/**
* Creates a new CaptionSerializer.
*
* @param t the class type for SemanticCaption
*/
public CaptionSerializer(Class<SemanticCaption> t) {
super(t);
}
@Override
public void serialize(SemanticCaption caption, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, caption, JsonName.CAPTION_TYPE);
if (caption.getLinkedContentId() != null) {
jsonGenerator.writeNumberField("linked content id", caption.getLinkedContentId());
}
SerializerUtil.writeTextInfo(jsonGenerator, caption);
SerializerUtil.writeMetadataIfPresent(jsonGenerator, caption);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/DoubleSerializer.java
+58
View File
@@ -0,0 +1,58 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.RoundingMode;
/**
* Jackson serializer for Double values.
* Rounds double values to 3 decimal places for cleaner JSON output.
*/
public class DoubleSerializer extends StdSerializer<Double> {
/**
* Creates a new DoubleSerializer.
*
* @param t the class type for Double
*/
public DoubleSerializer(Class<Double> t) {
super(t);
}
private static final int DEFAULT_ROUNDING_VALUE = 3;
@Override
public void serialize(Double number, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeNumber(round(number, DEFAULT_ROUNDING_VALUE));
}
private static double round(double value, int decimalPlaces) {
if (decimalPlaces < 0) {
throw new IllegalArgumentException();
}
BigDecimal bigDecimalValue = new BigDecimal(Double.toString(value));
bigDecimalValue = bigDecimalValue.setScale(decimalPlaces, RoundingMode.HALF_UP);
return bigDecimalValue.doubleValue();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/FormulaSerializer.java
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.json.JsonName;
import java.io.IOException;
/**
* JSON serializer for SemanticFormula objects.
*
* <p>Produces JSON output in the format:
* <pre>
* {
* "type": "formula",
* "id": 123,
* "page number": 1,
* "bounding box": [x1, y1, x2, y2],
* "content": "\\frac{a}{b}"
* }
* </pre>
*/
public class FormulaSerializer extends StdSerializer<SemanticFormula> {
public FormulaSerializer(Class<SemanticFormula> t) {
super(t);
}
@Override
public void serialize(SemanticFormula formula, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, formula, JsonName.FORMULA_TYPE);
jsonGenerator.writeStringField(JsonName.CONTENT, formula.getLatex());
SerializerUtil.writeMetadataIfPresent(jsonGenerator, formula);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/HeaderFooterSerializer.java
+57
View File
@@ -0,0 +1,57 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeaderOrFooter;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import java.io.IOException;
/**
* Jackson serializer for SemanticHeaderOrFooter objects.
* Serializes headers and footers with their child contents.
*/
public class HeaderFooterSerializer extends StdSerializer<SemanticHeaderOrFooter> {
/**
* Creates a new HeaderFooterSerializer.
*
* @param t the class type for SemanticHeaderOrFooter
*/
public HeaderFooterSerializer(Class<SemanticHeaderOrFooter> t) {
super(t);
}
@Override
public void serialize(SemanticHeaderOrFooter header, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, header, header.getSemanticType().getValue().toLowerCase());
jsonGenerator.writeArrayFieldStart(JsonName.KIDS);
for (IObject content : header.getContents()) {
if (!(content instanceof LineArtChunk)) {
jsonGenerator.writePOJO(content);
}
}
jsonGenerator.writeEndArray();
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/HeadingSerializer.java
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import java.io.IOException;
/**
* Jackson serializer for SemanticHeading objects.
* Serializes headings with their level and text content.
*/
public class HeadingSerializer extends StdSerializer<SemanticHeading> {
/**
* Creates a new HeadingSerializer.
*
* @param t the class type for SemanticHeading
*/
public HeadingSerializer(Class<SemanticHeading> t) {
super(t);
}
@Override
public void serialize(SemanticHeading heading, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, heading, JsonName.HEADING_TYPE);
jsonGenerator.writeNumberField(JsonName.HEADING_LEVEL, heading.getHeadingLevel());
SerializerUtil.writeTextInfo(jsonGenerator, heading);
SerializerUtil.writeMetadataIfPresent(jsonGenerator, heading);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/ImageSerializer.java
+82
View File
@@ -0,0 +1,82 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.EnrichedImageChunk;
import org.opendataloader.pdf.json.JsonName;
import org.opendataloader.pdf.markdown.MarkdownSyntax;
import org.opendataloader.pdf.utils.Base64ImageUtils;
import org.opendataloader.pdf.utils.ImagesUtils;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
import java.io.File;
import java.io.IOException;
public class ImageSerializer extends StdSerializer<ImageChunk> {
public ImageSerializer(Class<ImageChunk> t) {
super(t);
}
@Override
public void serialize(ImageChunk imageChunk, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
String imageFormat = StaticLayoutContainers.getImageFormat();
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, imageChunk.getIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", imageChunk.getIndex(), imageFormat);
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, imageChunk, JsonName.IMAGE_CHUNK_TYPE);
// alt / alt_source policy (PDF/UA semantics):
// - original /Alt present → alt = original, alt_source = "original"
// - no /Alt, AI caption → alt = AI caption, alt_source = "ai-generated"
// - neither present → no alt field, alt_source = "missing"
// We never synthesize a placeholder like "Image N": PDF/UA forbids false
// alternatives and a synthetic string defeats the evidence-report
// signal a reviewer needs to find genuinely missing alt text.
String alt = "";
String altSource = "missing";
if (imageChunk instanceof EnrichedImageChunk) {
EnrichedImageChunk eic = (EnrichedImageChunk) imageChunk;
alt = eic.sanitizeDescription();
if (!alt.isEmpty()) {
altSource = eic.getAltSource() == EnrichedImageChunk.AltSource.AI_GENERATED
? "ai-generated" : "original";
}
}
if (!alt.isEmpty()) {
jsonGenerator.writeStringField(JsonName.ALT, alt);
}
jsonGenerator.writeStringField(JsonName.ALT_SOURCE, altSource);
if (ImagesUtils.isImageFileExists(absolutePath)) {
if (StaticLayoutContainers.isEmbedImages()) {
File imageFile = new File(absolutePath);
String dataUri = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (dataUri != null) {
jsonGenerator.writeStringField(JsonName.DATA, dataUri);
jsonGenerator.writeStringField(JsonName.IMAGE_FORMAT, imageFormat);
}
} else {
jsonGenerator.writeStringField(JsonName.SOURCE, relativePath);
}
}
SerializerUtil.writeMetadataIfPresent(jsonGenerator, imageChunk);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/LineChunkSerializer.java
+39
View File
@@ -0,0 +1,39 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import java.io.IOException;
public class LineChunkSerializer extends StdSerializer<LineChunk> {
public LineChunkSerializer(Class<LineChunk> t) {
super(t);
}
@Override
public void serialize(LineChunk lineChunk, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, lineChunk, JsonName.LINE_CHUNK_TYPE);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/ListItemSerializer.java
+49
View File
@@ -0,0 +1,49 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import java.io.IOException;
public class ListItemSerializer extends StdSerializer<ListItem> {
public ListItemSerializer(Class<ListItem> t) {
super(t);
}
@Override
public void serialize(ListItem item, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, item, JsonName.LIST_ITEM_TYPE);
SerializerUtil.writeTextInfo(jsonGenerator, item);
jsonGenerator.writeArrayFieldStart(JsonName.KIDS);
for (IObject content : item.getContents()) {
if (!(content instanceof LineArtChunk)) {
jsonGenerator.writePOJO(content);
}
}
jsonGenerator.writeEndArray();
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/ListSerializer.java
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import java.io.IOException;
public class ListSerializer extends StdSerializer<PDFList> {
public ListSerializer(Class<PDFList> t) {
super(t);
}
@Override
public void serialize(PDFList list, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, list, JsonName.LIST_TYPE);
jsonGenerator.writeStringField(JsonName.NUMBERING_STYLE, list.getNumberingStyle());
jsonGenerator.writeNumberField(JsonName.NUMBER_OF_LIST_ITEMS, list.getNumberOfListItems());
if (list.getPreviousListId() != null) {
jsonGenerator.writeNumberField(JsonName.PREVIOUS_LIST_ID, list.getPreviousListId());
}
if (list.getNextListId() != null) {
jsonGenerator.writeNumberField(JsonName.NEXT_LIST_ID, list.getNextListId());
}
jsonGenerator.writeArrayFieldStart(JsonName.LIST_ITEMS);
for (ListItem item : list.getListItems()) {
jsonGenerator.writePOJO(item);
}
jsonGenerator.writeEndArray();
SerializerUtil.writeMetadataIfPresent(jsonGenerator, list);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/ParagraphSerializer.java
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import java.io.IOException;
public class ParagraphSerializer extends StdSerializer<SemanticParagraph> {
public ParagraphSerializer(Class<SemanticParagraph> t) {
super(t);
}
@Override
public void serialize(SemanticParagraph textParagraph, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, textParagraph, JsonName.PARAGRAPH_TYPE);
SerializerUtil.writeTextInfo(jsonGenerator, textParagraph);
SerializerUtil.writeMetadataIfPresent(jsonGenerator, textParagraph);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/PictureSerializer.java
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.json.JsonName;
import org.opendataloader.pdf.markdown.MarkdownSyntax;
import org.opendataloader.pdf.utils.Base64ImageUtils;
import org.opendataloader.pdf.utils.ImagesUtils;
import java.io.File;
import java.io.IOException;
/**
* JSON serializer for SemanticPicture elements.
*
* <p>Serializes pictures with their description (alt text) and image source.
*/
public class PictureSerializer extends StdSerializer<SemanticPicture> {
public PictureSerializer(Class<SemanticPicture> t) {
super(t);
}
@Override
public void serialize(SemanticPicture picture, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
String imageFormat = StaticLayoutContainers.getImageFormat();
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, picture.getPictureIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", picture.getPictureIndex(), imageFormat);
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, picture, JsonName.IMAGE_CHUNK_TYPE);
// alt / alt_source — same policy as ImageSerializer. A SemanticPicture
// only reaches this serializer when enrichBackendResults could not
// match it to a Java ImageChunk, i.e. the backend (always AI for
// SemanticPicture today) is the only source of alt text. Drop the
// legacy `description` field in favor of the unified `alt` schema.
String alt = picture.hasDescription() ? picture.sanitizeDescription() : "";
if (!alt.isEmpty()) {
jsonGenerator.writeStringField(JsonName.ALT, alt);
jsonGenerator.writeStringField(JsonName.ALT_SOURCE, "ai-generated");
} else {
jsonGenerator.writeStringField(JsonName.ALT_SOURCE, "missing");
}
if (ImagesUtils.isImageFileExists(absolutePath)) {
if (StaticLayoutContainers.isEmbedImages()) {
File imageFile = new File(absolutePath);
String dataUri = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (dataUri != null) {
jsonGenerator.writeStringField(JsonName.DATA, dataUri);
jsonGenerator.writeStringField(JsonName.IMAGE_FORMAT, imageFormat);
}
} else {
jsonGenerator.writeStringField(JsonName.SOURCE, relativePath);
}
}
SerializerUtil.writeMetadataIfPresent(jsonGenerator, picture);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/SemanticTextNodeSerializer.java
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import java.io.IOException;
//For now this class is used to process headers, footers, headings, captions.
public class SemanticTextNodeSerializer extends StdSerializer<SemanticTextNode> {
public SemanticTextNodeSerializer(Class<SemanticTextNode> t) {
super(t);
}
@Override
public void serialize(SemanticTextNode textNode, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, textNode, textNode.getSemanticType().toString().toLowerCase());
SerializerUtil.writeTextInfo(jsonGenerator, textNode);
SerializerUtil.writeMetadataIfPresent(jsonGenerator, textNode);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/SerializerUtil.java
+219
View File
@@ -0,0 +1,219 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import org.opendataloader.pdf.hybrid.ElementMetadata;
import org.opendataloader.pdf.json.JsonName;
import org.opendataloader.pdf.utils.TextNodeUtils;
import org.verapdf.tools.TaggedPDFConstants;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.TextBlock;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Map;
public class SerializerUtil {
private static final ThreadLocal<Map<Long, ElementMetadata>> ELEMENT_METADATA =
ThreadLocal.withInitial(Collections::emptyMap);
public static void setElementMetadata(Map<Long, ElementMetadata> metadata) {
ELEMENT_METADATA.set(metadata != null ? metadata : Collections.emptyMap());
}
public static void clearElementMetadata() {
ELEMENT_METADATA.remove();
}
/**
* Writes element-level metadata fields (confidence, source label, etc.) if available.
* Call this before writeEndObject() in each serializer.
*/
public static void writeMetadataIfPresent(JsonGenerator gen, IObject object) throws IOException {
if (object == null || object.getRecognizedStructureId() == null) return;
Map<Long, ElementMetadata> metadata = ELEMENT_METADATA.get();
if (metadata.isEmpty()) return;
ElementMetadata meta = metadata.get(object.getRecognizedStructureId());
if (meta == null) return;
if (meta.getAiScore() >= 0.0) {
gen.writeNumberField(JsonName.AI_SCORE, meta.getAiScore());
}
if (meta.getSourceLabel() >= 0) {
gen.writeNumberField(JsonName.SOURCE_LABEL, meta.getSourceLabel());
}
if (meta.getHeadingInferenceMethod() != null) {
gen.writeObjectFieldStart("heading inference");
gen.writeStringField("method", meta.getHeadingInferenceMethod());
if (meta.getBboxHeightPx() != null) {
gen.writeNumberField("bbox height px", meta.getBboxHeightPx());
}
gen.writeEndObject();
}
if (meta.getTsr() != null) {
gen.writeObjectFieldStart("tsr");
gen.writeNumberField("num cells", meta.getTsr().getNumCells());
if (meta.getTsr().getHtml() != null && !meta.getTsr().getHtml().isEmpty()) {
gen.writeStringField("html", meta.getTsr().getHtml());
}
if (meta.getTsr().getRunTimeMs() > 0) {
gen.writeNumberField("run time ms", meta.getTsr().getRunTimeMs());
}
gen.writeEndObject();
}
if (meta.getCaption() != null) {
gen.writeObjectFieldStart("caption");
if (meta.getCaption().getText() != null) {
gen.writeStringField("text", meta.getCaption().getText());
}
if (meta.getCaption().getLanguage() != null) {
gen.writeStringField("language", meta.getCaption().getLanguage());
}
if (meta.getCaption().getRunTimeMs() > 0) {
gen.writeNumberField("run time ms", meta.getCaption().getRunTimeMs());
}
gen.writeEndObject();
}
if (meta.getRegionlistResolution() != null) {
gen.writeObjectFieldStart("regionlist resolution");
gen.writeStringField("strategy", meta.getRegionlistResolution().getStrategy());
gen.writeBooleanField("tsr attempted", meta.getRegionlistResolution().isTsrAttempted());
if (meta.getRegionlistResolution().getTsrResult() != null) {
gen.writeStringField("tsr result", meta.getRegionlistResolution().getTsrResult());
}
gen.writeEndObject();
}
if (meta.getWordMatchMethod() != null) {
gen.writeObjectFieldStart("word match");
gen.writeStringField("method", meta.getWordMatchMethod());
gen.writeNumberField("matched words", meta.getMatchedWordCount());
gen.writeEndObject();
}
if (meta.getTextSource() != null) {
gen.writeStringField("text source", meta.getTextSource());
}
if (meta.getStreamOcrSimilarity() != null) {
gen.writeNumberField("stream ocr similarity", meta.getStreamOcrSimilarity());
}
}
public static void writeEssentialInfo(JsonGenerator jsonGenerator, IObject object, String type) throws IOException {
jsonGenerator.writeStringField(JsonName.TYPE, type);
String pdfuaTag = pdfuaTagFor(type, object);
if (pdfuaTag != null) {
jsonGenerator.writeStringField(JsonName.PDFUA_TAG, pdfuaTag);
}
Long id = object.getRecognizedStructureId();
if (id != null && id != 0L) {
jsonGenerator.writeNumberField(JsonName.ID, id);
}
if (object.getLevel() != null) {
jsonGenerator.writeStringField(JsonName.LEVEL, object.getLevel());
}
jsonGenerator.writeNumberField(JsonName.PAGE_NUMBER, object.getPageNumber() + 1);
jsonGenerator.writeArrayFieldStart(JsonName.BOUNDING_BOX);
jsonGenerator.writePOJO(object.getLeftX());
jsonGenerator.writePOJO(object.getBottomY());
jsonGenerator.writePOJO(object.getRightX());
jsonGenerator.writePOJO(object.getTopY());
jsonGenerator.writeEndArray();
}
/**
* Maps an extraction JSON `type` (plus heading level when relevant) to the
* PDF/UA structure tag that AutoTaggingProcessor will emit for the node.
* Returns null when the node has no canonical PDF/UA tag (e.g. text chunks
* that live below the structure-element granularity).
*/
static String pdfuaTagFor(String type, IObject object) {
if (type == null) {
return null;
}
switch (type) {
case JsonName.HEADING_TYPE:
if (object instanceof SemanticHeading) {
int level = ((SemanticHeading) object).getHeadingLevel();
if (level >= 1 && level <= 6) {
return "H" + level;
}
}
// Fallback for unleveled headings. PDF/UA-2 deprecates the
// plain "H" tag in favor of H1..H6; emit it only because we
// cannot infer a level. Downstream remediation may upgrade
// this when more context is available.
return TaggedPDFConstants.H;
case JsonName.PARAGRAPH_TYPE:
return TaggedPDFConstants.P;
case JsonName.IMAGE_CHUNK_TYPE:
return TaggedPDFConstants.FIGURE;
case JsonName.FORMULA_TYPE:
return TaggedPDFConstants.FORMULA;
case JsonName.CAPTION_TYPE:
return TaggedPDFConstants.CAPTION;
case JsonName.LIST_TYPE:
return TaggedPDFConstants.L;
case JsonName.LIST_ITEM_TYPE:
return TaggedPDFConstants.LI;
case JsonName.TOC_TYPE:
return TaggedPDFConstants.TOC;
case JsonName.TOC_ITEM_TYPE:
return TaggedPDFConstants.TOCI;
case JsonName.TABLE_TYPE:
return TaggedPDFConstants.TABLE;
case JsonName.TABLE_CELL_TYPE:
return ((TableBorderCell)object).isHeaderCell() ? TaggedPDFConstants.TH : TaggedPDFConstants.TD;
default:
// header/footer/footnote/line/text-chunk/text-block
// either become Artifact or are not promoted to their own
// PDF/UA structure element. Leave the tag unset for now.
return null;
}
}
public static void writeTextInfo(JsonGenerator jsonGenerator, SemanticTextNode textNode) throws IOException {
jsonGenerator.writeStringField(JsonName.FONT_TYPE, textNode.getFontName());
jsonGenerator.writePOJOField(JsonName.FONT_SIZE, textNode.getFontSize());
double[] textColor = TextNodeUtils.getTextColorOrNull(textNode);
if (textColor != null) {
jsonGenerator.writeStringField(JsonName.TEXT_COLOR, Arrays.toString(textColor));
}
jsonGenerator.writeStringField(JsonName.CONTENT, textNode.getValue());
if (textNode.isHiddenText()) {
jsonGenerator.writeBooleanField(JsonName.HIDDEN_TEXT, true);
}
}
public static void writeTextInfo(JsonGenerator jsonGenerator, TextBlock textBlock) throws IOException {
jsonGenerator.writeStringField(JsonName.FONT_TYPE, textBlock.getFirstLine().getFirstTextChunk().getFontName());
jsonGenerator.writePOJOField(JsonName.FONT_SIZE, textBlock.getFontSize());
jsonGenerator.writeStringField(JsonName.TEXT_COLOR, Arrays.toString(
textBlock.getFirstLine().getFirstTextChunk().getFontColor()));
jsonGenerator.writeStringField(JsonName.CONTENT, textBlock.toString());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TOCItemSerializer.java
+49
View File
@@ -0,0 +1,49 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticTOCI;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import java.io.IOException;
public class TOCItemSerializer extends StdSerializer<SemanticTOCI> {
public TOCItemSerializer(Class<SemanticTOCI> t) {
super(t);
}
@Override
public void serialize(SemanticTOCI item, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, item, JsonName.TOC_ITEM_TYPE);
SerializerUtil.writeTextInfo(jsonGenerator, item);
jsonGenerator.writeArrayFieldStart(JsonName.KIDS);
for (IObject content : item.getContents()) {
if (!(content instanceof LineArtChunk)) {
jsonGenerator.writePOJO(content);
}
}
jsonGenerator.writeEndArray();
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TOCSerializer.java
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticTOC;
import java.io.IOException;
public class TOCSerializer extends StdSerializer<SemanticTOC> {
public TOCSerializer(Class<SemanticTOC> t) {
super(t);
}
@Override
public void serialize(SemanticTOC toc, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, toc, JsonName.TOC_TYPE);
if (toc.getPreviousTOCId() != null) {
jsonGenerator.writeNumberField(JsonName.PREVIOUS_TOC_ID, toc.getPreviousTOCId());
}
if (toc.getNextTOCId() != null) {
jsonGenerator.writeNumberField(JsonName.NEXT_TOC_ID, toc.getNextTOCId());
}
jsonGenerator.writeArrayFieldStart(JsonName.TOC_ITEMS);
for (IObject child : toc.getTOCItems()) {
jsonGenerator.writePOJO(child);
}
jsonGenerator.writeEndArray();
SerializerUtil.writeMetadataIfPresent(jsonGenerator, toc);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TableCellSerializer.java
+55
View File
@@ -0,0 +1,55 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import java.io.IOException;
public class TableCellSerializer extends StdSerializer<TableBorderCell> {
public TableCellSerializer(Class<TableBorderCell> t) {
super(t);
}
@Override
public void serialize(TableBorderCell cell, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, cell, JsonName.TABLE_CELL_TYPE);
jsonGenerator.writeNumberField(JsonName.ROW_NUMBER, cell.getRowNumber() + 1);
jsonGenerator.writeNumberField(JsonName.COLUMN_NUMBER, cell.getColNumber() + 1);
jsonGenerator.writeNumberField(JsonName.ROW_SPAN, cell.getRowSpan());
jsonGenerator.writeNumberField(JsonName.COLUMN_SPAN, cell.getColSpan());
if (cell.isHeaderCell()) {
jsonGenerator.writeBooleanField(JsonName.IS_HEADER, true);
}
jsonGenerator.writeArrayFieldStart(JsonName.KIDS);
for (IObject content : cell.getContents()) {
if (!(content instanceof LineArtChunk)) {
jsonGenerator.writePOJO(content);
}
}
jsonGenerator.writeEndArray();
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TableRowSerializer.java
+51
View File
@@ -0,0 +1,51 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.io.IOException;
public class TableRowSerializer extends StdSerializer<TableBorderRow> {
public TableRowSerializer(Class<TableBorderRow> t) {
super(t);
}
@Override
public void serialize(TableBorderRow row, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
jsonGenerator.writeStringField(JsonName.TYPE, JsonName.ROW_TYPE);
jsonGenerator.writeNumberField(JsonName.ROW_NUMBER, row.getRowNumber() + 1);
jsonGenerator.writeArrayFieldStart(JsonName.CELLS);
TableBorderCell[] cells = row.getCells();
for (int columnNumber = 0; columnNumber < cells.length; columnNumber++) {
TableBorderCell cell = cells[columnNumber];
if (cell.getColNumber() == columnNumber && cell.getRowNumber() == row.getRowNumber()) {
jsonGenerator.writePOJO(cell);
}
}
jsonGenerator.writeEndArray();
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TableSerializer.java
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.io.IOException;
public class TableSerializer extends StdSerializer<TableBorder> {
public TableSerializer(Class<TableBorder> t) {
super(t);
}
@Override
public void serialize(TableBorder table, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, table, table.isTextBlock() ? JsonName.TEXT_BLOCK : JsonName.TABLE_TYPE);
if (table.isTextBlock()) {
jsonGenerator.writeArrayFieldStart(JsonName.KIDS);
for (IObject content : table.getCell(0, 0).getContents()) {
if (!(content instanceof LineArtChunk)) {
jsonGenerator.writePOJO(content);
}
}
jsonGenerator.writeEndArray();
} else {
jsonGenerator.writeNumberField(JsonName.NUMBER_OF_ROWS, table.getNumberOfRows());
jsonGenerator.writeNumberField(JsonName.NUMBER_OF_COLUMNS, table.getNumberOfColumns());
if (table.getPreviousTableId() != null) {
jsonGenerator.writeNumberField(JsonName.PREVIOUS_TABLE_ID, table.getPreviousTableId());
}
if (table.getNextTableId() != null) {
jsonGenerator.writeNumberField(JsonName.NEXT_TABLE_ID, table.getNextTableId());
}
jsonGenerator.writeArrayFieldStart(JsonName.ROWS);
for (TableBorderRow row : table.getRows()) {
jsonGenerator.writePOJO(row);
}
jsonGenerator.writeEndArray();
}
SerializerUtil.writeMetadataIfPresent(jsonGenerator, table);
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TextChunkSerializer.java
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import java.io.IOException;
public class TextChunkSerializer extends StdSerializer<TextChunk> {
public TextChunkSerializer(Class<TextChunk> t) {
super(t);
}
@Override
public void serialize(TextChunk textChunk, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, textChunk, JsonName.TEXT_CHUNK_TYPE);
jsonGenerator.writeStringField(JsonName.CONTENT, textChunk.getValue());
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/json/serializers/TextLineSerializer.java
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.json.serializers;
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import org.opendataloader.pdf.json.JsonName;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import java.io.IOException;
public class TextLineSerializer extends StdSerializer<TextLine> {
public TextLineSerializer(Class<TextLine> t) {
super(t);
}
@Override
public void serialize(TextLine textLine, JsonGenerator jsonGenerator, SerializerProvider serializerProvider)
throws IOException {
jsonGenerator.writeStartObject();
SerializerUtil.writeEssentialInfo(jsonGenerator, textLine, JsonName.TEXT_CHUNK_TYPE);
jsonGenerator.writeStringField(JsonName.CONTENT, textLine.getValue());
jsonGenerator.writeEndObject();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/markdown/MarkdownGenerator.java
+461
View File
@@ -0,0 +1,461 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.markdown;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.SemanticFormula;
import org.opendataloader.pdf.entities.EnrichedImageChunk;
import org.opendataloader.pdf.entities.SemanticPicture;
import org.opendataloader.pdf.utils.Base64ImageUtils;
import org.opendataloader.pdf.utils.GeneratorUtils;
import org.opendataloader.pdf.utils.ImagesUtils;
import org.opendataloader.pdf.utils.OutputType;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.*;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.io.Closeable;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
public class MarkdownGenerator implements Closeable {
protected static final Logger LOGGER = Logger.getLogger(MarkdownGenerator.class.getCanonicalName());
protected final java.io.Writer markdownWriter;
protected final String markdownFileName;
protected int tableNesting = 0;
protected boolean isImageSupported;
protected String markdownPageSeparator;
/**
* Page numbers (1-based) selected by --pages; an empty set means all pages.
* Sourced from the raw {@link Config#getPageNumbers()} list (not the
* validated set built by {@code DocumentProcessor.getValidPageNumbers}).
* Safe to compare against {@code pageNumber + 1} because the surrounding
* loop is bounded by the document's actual page count, so out-of-range
* values from the raw list are never tested for membership.
*/
protected final Set<Integer> selectedPageNumbers;
protected boolean embedImages = false;
protected String imageFormat = Config.IMAGE_FORMAT_PNG;
protected boolean includeHeaderFooter = false;
protected static final String strikethroughTextMD = "~~";
MarkdownGenerator(File inputPdf, Config config) throws IOException {
String cutPdfFileName = inputPdf.getName();
this.markdownFileName = config.getOutputFolder() + File.separator + cutPdfFileName.substring(0, cutPdfFileName.length() - 3) + "md";
this.markdownWriter = new FileWriter(markdownFileName, StandardCharsets.UTF_8);
this.isImageSupported = !config.isImageOutputOff() && config.isGenerateMarkdown();
this.markdownPageSeparator = config.getMarkdownPageSeparator();
this.selectedPageNumbers = new HashSet<>(config.getPageNumbers());
this.embedImages = config.isEmbedImages();
this.imageFormat = config.getImageFormat();
this.includeHeaderFooter = config.isIncludeHeaderFooter();
}
/**
* Creates a MarkdownGenerator that writes to an arbitrary Writer (e.g., stdout).
*/
public MarkdownGenerator(java.io.Writer writer, Config config) {
this.markdownFileName = null;
this.markdownWriter = writer;
this.isImageSupported = false;
this.markdownPageSeparator = config.getMarkdownPageSeparator();
this.selectedPageNumbers = new HashSet<>(config.getPageNumbers());
this.embedImages = false;
this.imageFormat = config.getImageFormat();
this.includeHeaderFooter = config.isIncludeHeaderFooter();
}
public void writeToMarkdown(List<List<IObject>> contents) {
try {
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
if (selectedPageNumbers.isEmpty() || selectedPageNumbers.contains(pageNumber + 1)) {
writePageSeparator(pageNumber);
}
for (IObject content : contents.get(pageNumber)) {
if (!isSupportedContent(content)) {
continue;
}
this.write(content);
writeContentsSeparator();
}
}
LOGGER.log(Level.INFO, "Created {0}", markdownFileName);
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Unable to create markdown output: " + e.getMessage());
}
}
protected void writePageSeparator(int pageNumber) throws IOException {
if (!markdownPageSeparator.isEmpty()) {
markdownWriter.write(markdownPageSeparator.contains(Config.PAGE_NUMBER_STRING)
? markdownPageSeparator.replace(Config.PAGE_NUMBER_STRING, String.valueOf(pageNumber + 1))
: markdownPageSeparator);
writeContentsSeparator();
}
}
protected boolean isSupportedContent(IObject content) {
if (content instanceof SemanticHeaderOrFooter) {
return includeHeaderFooter;
}
return content instanceof SemanticTextNode || // Heading, Paragraph etc...
content instanceof SemanticFormula ||
content instanceof SemanticPicture ||
content instanceof TableBorder ||
content instanceof PDFList ||
content instanceof SemanticTOC ||
(content instanceof ImageChunk && isImageSupported);
}
protected void writeContentsSeparator() throws IOException {
writeLineBreak();
writeLineBreak();
}
protected void write(IObject object) throws IOException {
if (object instanceof SemanticHeaderOrFooter) {
writeHeaderOrFooter((SemanticHeaderOrFooter) object);
} else if (object instanceof SemanticPicture) {
writePicture((SemanticPicture) object);
} else if (object instanceof ImageChunk) {
writeImage((ImageChunk) object);
} else if (object instanceof SemanticFormula) {
writeFormula((SemanticFormula) object);
} else if (object instanceof SemanticHeading) {
writeHeading((SemanticHeading) object);
} else if (object instanceof SemanticParagraph) {
writeParagraph((SemanticParagraph) object);
} else if (object instanceof SemanticTextNode) {
writeSemanticTextNode((SemanticTextNode) object);
} else if (object instanceof TableBorder) {
writeTable((TableBorder) object);
} else if (object instanceof PDFList) {
writeList((PDFList) object);
} else if (object instanceof SemanticTOC) {
writeTOC((SemanticTOC) object);
}
}
/**
* Wraps an image relative path as a CommonMark angle-bracket link destination
* (`<...>`). The bare form `(my paper.png)` is terminated by the first space or
* unbalanced parenthesis, so paths inheriting filenames with spaces, parens, or
* brackets break in renderers (#405). The angle-bracket form is the
* spec-recommended way to embed such paths and lets the on-disk path stay
* byte-identical to the rendered link, which preserves user intent for both the
* default `<stem>_images/` directory and any `--image-dir` value.
*
* Only `<`, `>`, and `\` are reserved inside the angle-bracket form; escape
* those with a backslash. Newlines have no representable form in a link
* destination — replace them with spaces so the destination stays well-formed.
*/
static String formatMarkdownLinkDestination(String path) {
if (path == null) {
return null;
}
StringBuilder sb = new StringBuilder(path.length() + 2);
sb.append('<');
for (int i = 0; i < path.length(); i++) {
char c = path.charAt(i);
if (c == '<' || c == '>' || c == '\\') {
sb.append('\\').append(c);
} else if (c == '\n' || c == '\r') {
sb.append(' ');
} else {
sb.append(c);
}
}
sb.append('>');
return sb.toString();
}
protected void writeImage(ImageChunk image) {
try {
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, image.getIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", image.getIndex(), imageFormat);
if (ImagesUtils.isImageFileExists(absolutePath)) {
String imageSource;
if (embedImages) {
File imageFile = new File(absolutePath);
imageSource = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (imageSource == null) {
LOGGER.log(Level.WARNING, "Failed to convert image to Base64: {0}", absolutePath);
}
} else {
imageSource = formatMarkdownLinkDestination(relativePath);
}
if (imageSource != null) {
// No "image N" fallback: PDF/UA forbids false alternatives,
// and an empty Markdown alt lets screen readers skip the
// image as a decorative element rather than reading a
// meaningless synthetic label.
String altText = (image instanceof EnrichedImageChunk && ((EnrichedImageChunk) image).hasDescription())
? ((EnrichedImageChunk) image).sanitizeDescription()
: "";
String imageString = String.format(MarkdownSyntax.IMAGE_FORMAT, altText, imageSource);
markdownWriter.write(getCorrectMarkdownString(imageString));
}
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to write image for markdown output: " + e.getMessage());
}
}
/**
* Writes a SemanticPicture with its description as alt text.
*
* @param picture The picture to write
*/
protected void writePicture(SemanticPicture picture) {
try {
String absolutePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectory(), File.separator, picture.getPictureIndex(), imageFormat);
String relativePath = String.format(MarkdownSyntax.IMAGE_FILE_NAME_FORMAT, StaticLayoutContainers.getImagesDirectoryName(), "/", picture.getPictureIndex(), imageFormat);
if (ImagesUtils.isImageFileExists(absolutePath)) {
String imageSource;
if (embedImages) {
File imageFile = new File(absolutePath);
imageSource = Base64ImageUtils.toDataUri(imageFile, imageFormat);
if (imageSource == null) {
LOGGER.log(Level.WARNING, "Failed to convert image to Base64: {0}", absolutePath);
}
} else {
imageSource = formatMarkdownLinkDestination(relativePath);
}
if (imageSource != null) {
String altText = picture.hasDescription()
? picture.sanitizeDescription()
: "";
String imageString = String.format(MarkdownSyntax.IMAGE_FORMAT, altText, imageSource);
markdownWriter.write(getCorrectMarkdownString(imageString));
}
}
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to write picture for markdown output: " + e.getMessage());
}
}
/**
* Writes a formula in LaTeX format wrapped in $$ delimiters.
*
* @param formula The formula to write
*/
protected void writeFormula(SemanticFormula formula) throws IOException {
markdownWriter.write(MarkdownSyntax.MATH_BLOCK_START);
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
markdownWriter.write(formula.getLatex());
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
markdownWriter.write(MarkdownSyntax.MATH_BLOCK_END);
}
protected void writeHeaderOrFooter(SemanticHeaderOrFooter headerOrFooter) throws IOException {
for (IObject content : headerOrFooter.getContents()) {
if (isSupportedContent(content)) {
write(content);
writeContentsSeparator();
}
}
}
protected void writeList(PDFList list) throws IOException {
for (ListItem item : list.getListItems()) {
if (!isInsideTable()) {
markdownWriter.write(MarkdownSyntax.LIST_ITEM);
markdownWriter.write(MarkdownSyntax.SPACE);
}
markdownWriter.write(getCorrectMarkdownString(GeneratorUtils.getTextFromLines(item.getLines(), OutputType.MD)));
writeLineBreak();
List<IObject> itemContents = item.getContents();
if (!itemContents.isEmpty()) {
writeLineBreak();
writeContents(itemContents, false);
}
}
}
protected void writeTOC(SemanticTOC toc) throws IOException {
for (IObject item : toc.getTOCItems()) {
if (item instanceof SemanticTOC) {
writeTOC((SemanticTOC)item);
} else if (item instanceof SemanticTOCI) {
SemanticTOCI tocItem = (SemanticTOCI)item;
markdownWriter.write(getCorrectMarkdownString(GeneratorUtils.getTextFromLines(tocItem.getLines(), OutputType.MD)));
writeLineBreak();
List<IObject> itemContents = tocItem.getContents();
if (!itemContents.isEmpty()) {
writeLineBreak();
writeContents(itemContents, false);
}
}
}
}
protected void writeSemanticTextNode(SemanticTextNode textNode) throws IOException {
String value = GeneratorUtils.getTextFromTextNode(textNode, OutputType.MD);
if (StaticContainers.isKeepLineBreaks()) {
if (textNode instanceof SemanticHeading) {
value = value.replace(MarkdownSyntax.LINE_BREAK, MarkdownSyntax.SPACE);
} else if (isInsideTable()) {
value = value.replace(MarkdownSyntax.LINE_BREAK, getLineBreak());
}
} else if (isInsideTable()) {
// Always replace line breaks with space in table cells for proper markdown table formatting
value = value.replace(MarkdownSyntax.LINE_BREAK, MarkdownSyntax.SPACE);
}
markdownWriter.write(getCorrectMarkdownString(value));
}
protected void writeTable(TableBorder table) throws IOException {
enterTable();
for (int rowNumber = 0; rowNumber < table.getNumberOfRows(); rowNumber++) {
TableBorderRow row = table.getRow(rowNumber);
markdownWriter.write(MarkdownSyntax.TABLE_COLUMN_SEPARATOR);
for (int colNumber = 0; colNumber < table.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = row.getCell(colNumber);
if (cell.getRowNumber() == rowNumber && cell.getColNumber() == colNumber) {
List<IObject> cellContents = cell.getContents();
writeContents(cellContents, true);
} else {
writeSpace();
}
markdownWriter.write(MarkdownSyntax.TABLE_COLUMN_SEPARATOR);
}
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
//Due to markdown syntax we have to separate column headers
if (rowNumber == 0) {
markdownWriter.write(MarkdownSyntax.TABLE_COLUMN_SEPARATOR);
for (int i = 0; i < table.getNumberOfColumns(); i++) {
markdownWriter.write(MarkdownSyntax.TABLE_HEADER_SEPARATOR);
markdownWriter.write(MarkdownSyntax.TABLE_COLUMN_SEPARATOR);
}
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
}
}
leaveTable();
}
protected void writeContents(List<IObject> contents, boolean isTable) throws IOException {
boolean wroteAnyContent = false;
for (int i = 0; i < contents.size(); i++) {
IObject content = contents.get(i);
if (!isSupportedContent(content)) {
continue;
}
this.write(content);
boolean isLastContent = i == contents.size() - 1;
if (!isTable || !isLastContent) {
writeContentsSeparator();
}
wroteAnyContent = true;
}
if (!wroteAnyContent && isTable) {
writeSpace();
}
}
protected void writeParagraph(SemanticParagraph textNode) throws IOException {
writeSemanticTextNode(textNode);
}
protected void writeHeading(SemanticHeading heading) throws IOException {
if (!isInsideTable()) {
// Cap heading level to 1-6 per Markdown specification
int headingLevel = Math.min(6, Math.max(1, heading.getHeadingLevel()));
for (int i = 0; i < headingLevel; i++) {
markdownWriter.write(MarkdownSyntax.HEADING_LEVEL);
}
markdownWriter.write(MarkdownSyntax.SPACE);
}
writeSemanticTextNode(heading);
}
protected void enterTable() {
tableNesting++;
}
protected void leaveTable() {
if (tableNesting > 0) {
tableNesting--;
}
}
protected boolean isInsideTable() {
return tableNesting > 0;
}
protected String getLineBreak() {
if (isInsideTable()) {
return MarkdownSyntax.HTML_LINE_BREAK_TAG;
} else {
return MarkdownSyntax.LINE_BREAK;
}
}
protected void writeLineBreak() throws IOException {
markdownWriter.write(getLineBreak());
}
protected void writeSpace() throws IOException {
markdownWriter.write(MarkdownSyntax.SPACE);
}
protected String getCorrectMarkdownString(String value) {
if (value != null) {
return value.replace("\u0000", " ");
}
return null;
}
public static void getTextFromLineForMarkdown(TextLine line, StringBuilder stringBuilder) {
for (TextChunk chunk : line.getTextChunks()) {
if (chunk.getIsStrikethroughText()) {
stringBuilder.append(strikethroughTextMD);
}
stringBuilder.append(chunk.getValue());
if (chunk.getIsStrikethroughText()) {
stringBuilder.append(strikethroughTextMD);
}
}
}
@Override
public void close() throws IOException {
if (markdownWriter != null) {
markdownWriter.close();
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/markdown/MarkdownGeneratorFactory.java
+31
View File
@@ -0,0 +1,31 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.markdown;
import org.opendataloader.pdf.api.Config;
import java.io.File;
import java.io.IOException;
public class MarkdownGeneratorFactory {
public static MarkdownGenerator getMarkdownGenerator(File inputPdf,
Config config) throws IOException {
if (config.isUseHTMLInMarkdown()) {
return new MarkdownHTMLGenerator(inputPdf, config);
}
return new MarkdownGenerator(inputPdf, config);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/markdown/MarkdownHTMLGenerator.java
+92
View File
@@ -0,0 +1,92 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.markdown;
import org.opendataloader.pdf.api.Config;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.io.File;
import java.io.IOException;
import java.util.List;
public class MarkdownHTMLGenerator extends MarkdownGenerator {
protected MarkdownHTMLGenerator(File inputPdf, Config config) throws IOException {
super(inputPdf, config);
}
@Override
protected void writeTable(TableBorder table) throws IOException {
enterTable();
markdownWriter.write(MarkdownSyntax.HTML_TABLE_TAG);
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
for (int rowNumber = 0; rowNumber < table.getNumberOfRows(); rowNumber++) {
TableBorderRow row = table.getRow(rowNumber);
markdownWriter.write(MarkdownSyntax.INDENT);
markdownWriter.write(MarkdownSyntax.HTML_TABLE_ROW_TAG);
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
for (int colNumber = 0; colNumber < table.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = row.getCell(colNumber);
if (cell.getRowNumber() == rowNumber && cell.getColNumber() == colNumber) {
writeCellTagBegin(cell);
List<IObject> cellContents = cell.getContents();
writeContents(cellContents, true);
writeCellTagEnd(cell.isHeaderCell());
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
}
}
markdownWriter.write(MarkdownSyntax.INDENT);
markdownWriter.write(MarkdownSyntax.HTML_TABLE_ROW_CLOSE_TAG);
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
}
markdownWriter.write(MarkdownSyntax.HTML_TABLE_CLOSE_TAG);
markdownWriter.write(MarkdownSyntax.LINE_BREAK);
leaveTable();
}
private void writeCellTagBegin(TableBorderCell cell) throws IOException {
markdownWriter.write(MarkdownSyntax.INDENT);
markdownWriter.write(MarkdownSyntax.INDENT);
String tag = cell.isHeaderCell() ? "<th" : "<td";
StringBuilder cellTag = new StringBuilder(tag);
int colSpan = cell.getColSpan();
if (colSpan != 1) {
cellTag.append(" colspan=\"").append(colSpan).append("\"");
}
int rowSpan = cell.getRowSpan();
if (rowSpan != 1) {
cellTag.append(" rowspan=\"").append(rowSpan).append("\"");
}
cellTag.append(">");
markdownWriter.write(getCorrectMarkdownString(cellTag.toString()));
}
private void writeCellTagEnd(boolean isHeader) throws IOException {
if (isHeader) {
markdownWriter.write(MarkdownSyntax.HTML_TABLE_HEADER_CLOSE_TAG);
} else {
markdownWriter.write(MarkdownSyntax.HTML_TABLE_CELL_CLOSE_TAG);
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/markdown/MarkdownSyntax.java
+50
View File
@@ -0,0 +1,50 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.markdown;
public class MarkdownSyntax {
public static final String TABLE_COLUMN_SEPARATOR = "|";
public static final String TABLE_HEADER_SEPARATOR = "---";
public static final String DOUBLE_LINE_BREAK = "\n\n";
public static final String LINE_BREAK = "\n";
public static final String SPACE = " ";
public static final String INDENT = " ";
public static final String HEADING_LEVEL = "#";
public static final String LIST_ITEM = "-";
public static final String IMAGES_DIRECTORY_SUFFIX = "_images";
public static final String IMAGE_FILE_NAME_FORMAT = "%s%simageFile%d.%s";
public static final String IMAGE_FORMAT = "![%s](%s)";
public static final String HTML_TABLE_TAG = "<table>";
public static final String HTML_TABLE_CLOSE_TAG = "</table>";
public static final String HTML_TABLE_ROW_TAG = "<tr>";
public static final String HTML_TABLE_ROW_CLOSE_TAG = "</tr>";
public static final String HTML_TABLE_CELL_TAG = "<td>";
public static final String HTML_TABLE_CELL_CLOSE_TAG = "</td>";
public static final String HTML_TABLE_HEADER_TAG = "<th>";
public static final String HTML_TABLE_HEADER_CLOSE_TAG = "</th>";
public static final String HTML_ORDERED_LIST_TAG = "<ol style=\"list-style-type:none;\">";
public static final String HTML_ORDERED_LIST_CLOSE_TAG = "</ol>";
public static final String HTML_UNORDERED_LIST_TAG = "<ul style=\"list-style-type:none;\">";
public static final String HTML_UNORDERED_LIST_CLOSE_TAG = "</ul>";
public static final String HTML_LIST_ITEM_TAG = "<li>";
public static final String HTML_LIST_ITEM_CLOSE_TAG = "</li>";
public static final String HTML_LINE_BREAK_TAG = "<br>";
public static final String HTML_INDENT = "&nbsp;&nbsp;&nbsp;&nbsp;";
public static final String HTML_PARAGRAPH_TAG = "<p>";
public static final String HTML_PARAGRAPH_CLOSE_TAG = "</p>";
public static final String MATH_BLOCK_START = "$$";
public static final String MATH_BLOCK_END = "$$";
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/pdf/PDFLayer.java
+38
View File
@@ -0,0 +1,38 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.pdf;
public enum PDFLayer {
CONTENT("content"),
TABLE_CELLS("table cells"),
LIST_ITEMS("list items"),
TOC_ITEMS("toc items"),
TABLE_CONTENT("table content"),
LIST_CONTENT("list content"),
TOC_CONTENT("toc content"),
TEXT_BLOCK_CONTENT("text blocks content"),
HEADER_AND_FOOTER_CONTENT("header and footer content");
private final String value;
PDFLayer(String value) {
this.value = value;
}
public String getValue() {
return value;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/pdf/PDFWriter.java
+347
View File
@@ -0,0 +1,347 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.pdf;
import org.opendataloader.pdf.processors.DocumentProcessor;
import org.opendataloader.pdf.utils.FileUtils;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.cos.COSDictionary;
import org.apache.pdfbox.cos.COSName;
import org.apache.pdfbox.cos.COSString;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.PDDocumentCatalog;
import org.apache.pdfbox.pdmodel.common.PDRectangle;
import org.apache.pdfbox.pdmodel.documentinterchange.markedcontent.PDPropertyList;
import org.apache.pdfbox.pdmodel.graphics.color.PDColor;
import org.apache.pdfbox.pdmodel.graphics.color.PDDeviceRGB;
import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentGroup;
import org.apache.pdfbox.pdmodel.graphics.optionalcontent.PDOptionalContentProperties;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotation;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationSquare;
import org.apache.pdfbox.pdmodel.interactive.annotation.PDAnnotationSquareCircle;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.geometry.MultiBoundingBox;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
public class PDFWriter {
private static final Logger LOGGER = Logger.getLogger(PDFWriter.class.getCanonicalName());
private final Map<PDFLayer, PDOptionalContentGroup> optionalContents = new HashMap<>();
private final List<List<PDAnnotation>> annotations = new ArrayList<>();
private final List<BoundingBox> pageBoundingBoxes = new ArrayList<>();
public void updatePDF(File inputPDF, String password, String outputFolder, List<List<IObject>> contents) throws IOException {
try (PDDocument document = Loader.loadPDF(inputPDF, password)) {
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
annotations.add(new ArrayList<>());
pageBoundingBoxes.add(DocumentProcessor.getPageBoundingBox(pageNumber));
}
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
for (IObject content : contents.get(pageNumber)) {
drawContent(content, PDFLayer.CONTENT);
}
}
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
document.getPage(pageNumber).getAnnotations().addAll(annotations.get(pageNumber));
}
annotations.clear();
pageBoundingBoxes.clear();
createOptContentsForAnnotations(document);
document.setAllSecurityToBeRemoved(true);
String outputFileName = outputFolder + File.separator +
FileUtils.getBaseName(inputPDF.getName()) + "_annotated.pdf";
document.save(outputFileName);
LOGGER.log(Level.INFO, "Created {0}", outputFileName);
} catch (Exception ex) {
LOGGER.log(Level.WARNING, "Unable to create annotated PDF output: " + ex.getMessage());
}
}
private void drawContent(IObject content, PDFLayer layer) throws IOException {
drawContent(content, layer, null);
}
private void drawContent(IObject content, PDFLayer layer, Map<Integer, PDAnnotation> linkedAnnots) throws IOException {
if ((content instanceof LineChunk)) {
return;
}
Map<Integer, PDAnnotation> annots = draw(content.getBoundingBox(), getColor(content), getContents(content),
content.getRecognizedStructureId(), linkedAnnots, content.getLevel(), layer);
if (content instanceof TableBorder) {
drawTableCells((TableBorder) content, annots);
} else if (content instanceof PDFList) {
drawListItems((PDFList) content, annots);
} else if (content instanceof SemanticTOC) {
drawTOCItems((SemanticTOC) content, layer, annots);
} else if (content instanceof SemanticHeaderOrFooter) {
for (IObject contentItem : ((SemanticHeaderOrFooter) content).getContents()) {
drawContent(contentItem, PDFLayer.HEADER_AND_FOOTER_CONTENT, annots);
}
}
}
private void drawTableCells(TableBorder table, Map<Integer, PDAnnotation> annots) throws IOException {
if (table.isTextBlock()) {
for (IObject content : table.getCell(0, 0).getContents()) {
drawContent(content, PDFLayer.TEXT_BLOCK_CONTENT);
}
return;
}
for (int rowNumber = 0; rowNumber < table.getNumberOfRows(); rowNumber++) {
TableBorderRow row = table.getRow(rowNumber);
for (int colNumber = 0; colNumber < table.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = row.getCell(colNumber);
if (cell.getRowNumber() == rowNumber && cell.getColNumber() == colNumber) {
StringBuilder contentValue = new StringBuilder();
for (IObject object : cell.getContents()) {
if (object instanceof SemanticTextNode) {
contentValue.append(((SemanticTextNode) object).getValue());
}
}
String cellValue = String.format("Table %scell: row number %s, column number %s, row span %s, column span %s, text content \"%s\"",
cell.isHeaderCell() ? "header " : "", cell.getRowNumber() + 1, cell.getColNumber() + 1,
cell.getRowSpan(), cell.getColSpan(), contentValue);
draw(cell.getBoundingBox(), getColor(SemanticType.TABLE), cellValue, null, annots, cell.getLevel(), PDFLayer.TABLE_CELLS);
for (IObject content : cell.getContents()) {
drawContent(content, PDFLayer.TABLE_CONTENT);
}
}
}
}
}
private void drawListItems(PDFList list, Map<Integer, PDAnnotation> annots) throws IOException {
for (ListItem listItem : list.getListItems()) {
String contentValue = String.format("List item: text content \"%s\"", listItem.toString());
draw(listItem.getBoundingBox(), getColor(SemanticType.LIST), contentValue, null, annots, listItem.getLevel(), PDFLayer.LIST_ITEMS);
for (IObject content : listItem.getContents()) {
drawContent(content, PDFLayer.LIST_CONTENT);
}
}
}
private void drawTOCItems(SemanticTOC toc, PDFLayer layer, Map<Integer, PDAnnotation> annots) throws IOException {
for (IObject tocItem : toc.getTOCItems()) {
if (tocItem instanceof SemanticTOC) {
drawContent(tocItem, layer, annots);
} else if (tocItem instanceof SemanticTOCI) {
String contentValue = String.format("TOC item: text content \"%s\"", tocItem.toString());
draw(tocItem.getBoundingBox(), getColor(SemanticType.TABLE_OF_CONTENT), contentValue, null, annots, tocItem.getLevel(), PDFLayer.TOC_ITEMS);
for (IObject content : ((SemanticTOCI)tocItem).getContents()) {
drawContent(content, PDFLayer.TOC_CONTENT);
}
}
}
}
public Map<Integer, PDAnnotation> draw(BoundingBox boundingBox, float[] colorArray, String contents, Long id,
Map<Integer, PDAnnotation> linkedAnnots, String level, PDFLayer layerName) {
Map<Integer, PDAnnotation> result = new HashMap<>();
if (!Objects.equals(boundingBox.getPageNumber(), boundingBox.getLastPageNumber())) {
if (boundingBox instanceof MultiBoundingBox) {
for (int pageNumber = boundingBox.getPageNumber(); pageNumber <= boundingBox.getLastPageNumber(); pageNumber++) {
BoundingBox boundingBoxForPage = boundingBox.getBoundingBox(pageNumber);
if (boundingBoxForPage != null) {
result.putAll(draw(boundingBoxForPage, colorArray, contents, id, linkedAnnots, level, layerName));
}
}
return result;
} else {
LOGGER.log(Level.WARNING, "Bounding box on several pages cannot be split");
}
}
BoundingBox movedBoundingBox = new BoundingBox(boundingBox);
BoundingBox pageBoundingBox = pageBoundingBoxes.get(boundingBox.getPageNumber());
if (pageBoundingBox != null) {
movedBoundingBox.move(pageBoundingBox.getLeftX(), pageBoundingBox.getBottomY());
}
PDAnnotationSquareCircle square = new PDAnnotationSquare();
square.setRectangle(new PDRectangle(getFloat(movedBoundingBox.getLeftX()), getFloat(movedBoundingBox.getBottomY()),
getFloat(movedBoundingBox.getWidth()), getFloat(movedBoundingBox.getHeight())));
square.setConstantOpacity(0.4f);
PDColor color = new PDColor(colorArray, PDDeviceRGB.INSTANCE);
square.setColor(color);
square.setInteriorColor(color);
square.setContents((id != null ? "id = " + id + ", " : "") + (level != null ? "level = " + level + ", " : "") + contents);
if (linkedAnnots != null) {
square.setInReplyTo(linkedAnnots.get(boundingBox.getPageNumber()));
}
square.setOptionalContent(getOptionalContent(layerName));
annotations.get(boundingBox.getPageNumber()).add(square);
result.put(boundingBox.getPageNumber(), square);
return result;
}
private static float getFloat(double value) {
float floatValue = (float) value;
if (floatValue == Float.POSITIVE_INFINITY) {
return Float.MAX_VALUE;
}
if (floatValue == Float.NEGATIVE_INFINITY) {
return -Float.MAX_VALUE;
}
return floatValue;
}
public static String getContents(IObject content) {
if (content instanceof TableBorder) {
TableBorder border = (TableBorder) content;
if (border.isTextBlock()) {
return "Text block";
}
return String.format("Table: %s rows, %s columns, previous table id %s, next table id %s",
border.getNumberOfRows(), border.getNumberOfColumns(), border.getPreviousTableId(), border.getNextTableId());
}
if (content instanceof PDFList) {
PDFList list = (PDFList) content;
return String.format("List: number of items %s, previous list id %s, next list id %s",
list.getNumberOfListItems(), list.getPreviousListId(), list.getNextListId());
}
if (content instanceof SemanticTOC) {
SemanticTOC toc = (SemanticTOC) content;
return String.format("TOC: previous toc id %s, next toc id %s", toc.getPreviousTOCId(), toc.getNextTOCId());
}
if (content instanceof INode) {
INode node = (INode) content;
if (node.getSemanticType() == SemanticType.HEADER || node.getSemanticType() == SemanticType.FOOTER) {
return node.getSemanticType().getValue();
}
if (node.getSemanticType() == SemanticType.CAPTION) {
SemanticCaption caption = (SemanticCaption) node;
return DocumentProcessor.getContentsValueForTextNode(caption) + ", connected with object with id = " +
caption.getLinkedContentId();
}
if (node.getSemanticType() == SemanticType.HEADING) {
SemanticHeading heading = (SemanticHeading) node;
return DocumentProcessor.getContentsValueForTextNode(heading) +
", heading level " + heading.getHeadingLevel();
}
if (node instanceof SemanticTextNode) {
return DocumentProcessor.getContentsValueForTextNode((SemanticTextNode) node);
}
}
if (content instanceof ImageChunk) {
return String.format("Image: height %.2f, width %.2f", content.getHeight(), content.getWidth());
}
if (content instanceof LineArtChunk) {
return String.format("Line Art: height %.2f, width %.2f", content.getHeight(), content.getWidth());
}
if (content instanceof LineChunk) {
return "Line";
}
return "";
}
public static float[] getColor(IObject content) {
if (content instanceof TableBorder) {
return getColor(SemanticType.TABLE);
}
if (content instanceof PDFList) {
return getColor(SemanticType.LIST);
}
if (content instanceof SemanticTOC) {
return getColor(SemanticType.TABLE_OF_CONTENT);
}
if (content instanceof INode) {
INode node = (INode) content;
return getColor(node.getSemanticType());
}
if (content instanceof ImageChunk) {
return getColor(SemanticType.FIGURE);
}
if (content instanceof LineArtChunk || content instanceof LineChunk) {
return getColor(SemanticType.PART);
}
return new float[]{};
}
public static float[] getColor(SemanticType semanticType) {
if (semanticType == SemanticType.HEADING || semanticType == SemanticType.HEADER || semanticType == SemanticType.FOOTER) {
return new float[]{0, 0, 1};
}
if (semanticType == SemanticType.LIST) {
return new float[]{0, 1, 0};
}
if (semanticType == SemanticType.PARAGRAPH) {
return new float[]{0, 1, 1};
}
if (semanticType == SemanticType.FIGURE) {
return new float[]{1, 0, 0};
}
if (semanticType == SemanticType.TABLE) {
return new float[]{1, 0, 1};
}
if (semanticType == SemanticType.TABLE_OF_CONTENT) {
return new float[]{0.8f, 0.2f, 0.8f};
}
if (semanticType == SemanticType.CAPTION) {
return new float[]{1, 1, 0};
}
if (semanticType == SemanticType.PART) {
return new float[]{0.9f, 0.9f, 0.9f};
}
return null;
}
private void createOptContentsForAnnotations(PDDocument document) {
if (optionalContents.isEmpty()) {
return;
}
PDDocumentCatalog catalog = document.getDocumentCatalog();
PDOptionalContentProperties oldOCProperties = catalog.getOCProperties();
if (oldOCProperties == null) {
oldOCProperties = new PDOptionalContentProperties();
catalog.setOCProperties(oldOCProperties);
}
for (PDOptionalContentGroup group : optionalContents.values()) {
oldOCProperties.addGroup(group);
oldOCProperties.setGroupEnabled(group, true);
}
optionalContents.clear();
}
public PDOptionalContentGroup getOptionalContent(PDFLayer layer) {
PDOptionalContentGroup group = optionalContents.get(layer);
if (group == null) {
COSDictionary cosDictionary = new COSDictionary();
cosDictionary.setItem(COSName.TYPE, COSName.OCG);
cosDictionary.setItem(COSName.NAME, new COSString(layer.getValue()));
group = (PDOptionalContentGroup) PDPropertyList.create(cosDictionary);
optionalContents.put(layer, group);
}
return group;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/AbstractTableProcessor.java
+132
View File
@@ -0,0 +1,132 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.tables.TableBordersCollection;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedSet;
/**
* Abstract base class for table detection processors.
* Provides common functionality for detecting and processing tables in PDF documents.
*/
public abstract class AbstractTableProcessor {
private static final double Y_DIFFERENCE_EPSILON = 0.1;
private static final double X_DIFFERENCE_EPSILON = 3;
private static final double TABLE_INTERSECTION_PERCENT = 0.01;
/**
* Processes tables across all pages that may contain tables.
*
* @param contents the document contents organized by page
*/
public void processTables(List<List<IObject>> contents) {
List<Integer> pageNumbers = getPagesWithPossibleTables(contents);
processTables(contents, pageNumbers);
}
/**
* Processes tables on specified pages.
*
* @param contents the document contents organized by page
* @param pageNumbers the list of page numbers to process
*/
public void processTables(List<List<IObject>> contents, List<Integer> pageNumbers) {
if (!pageNumbers.isEmpty()) {
List<List<TableBorder>> tables = getTables(contents, pageNumbers);
addTablesToTableCollection(tables, pageNumbers);
}
}
/**
* Detects tables on the specified pages.
*
* @param contents the document contents organized by page
* @param pageNumbers the list of page numbers to process
* @return a list of detected tables for each page
*/
protected abstract List<List<TableBorder>> getTables(List<List<IObject>> contents, List<Integer> pageNumbers);
private static void addTablesToTableCollection(List<List<TableBorder>> detectedTables, List<Integer> pageNumbers) {
if (detectedTables != null) {
TableBordersCollection tableCollection = StaticContainers.getTableBordersCollection();
for (int index = 0; index < pageNumbers.size(); index++) {
SortedSet<TableBorder> tables = tableCollection.getTableBorders(pageNumbers.get(index));
for (TableBorder border : detectedTables.get(index)) {
boolean hasIntersections = false;
for (TableBorder table : tables) {
if (table.getBoundingBox().getIntersectionPercent(border.getBoundingBox()) > TABLE_INTERSECTION_PERCENT) {
hasIntersections = true;
break;
}
}
if (!hasIntersections) {
tables.add(border);
}
}
}
}
}
/**
* Identifies pages that may contain tables based on text chunk patterns.
*
* @param contents the document contents organized by page
* @return a list of page numbers that may contain tables
*/
public static List<Integer> getPagesWithPossibleTables(List<List<IObject>> contents) {
List<Integer> pageNumbers = new ArrayList<>();
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
TextChunk previousTextChunk = null;
for (IObject content : contents.get(pageNumber)) {
if (content instanceof TextChunk) {
TextChunk currentTextChunk = (TextChunk) content;
if (currentTextChunk.isWhiteSpaceChunk()) {
continue;
}
if (previousTextChunk != null && areSuspiciousTextChunks(previousTextChunk, currentTextChunk)) {
pageNumbers.add(pageNumber);
break;
}
previousTextChunk = currentTextChunk;
}
}
}
return pageNumbers;
}
private static boolean areSuspiciousTextChunks(TextChunk previousTextChunk, TextChunk currentTextChunk) {
if (previousTextChunk.getTopY() < currentTextChunk.getBottomY()) {
return true;
}
if (NodeUtils.areCloseNumbers(previousTextChunk.getBaseLine(), currentTextChunk.getBaseLine(),
currentTextChunk.getHeight() * Y_DIFFERENCE_EPSILON)) {
if (currentTextChunk.getLeftX() - previousTextChunk.getRightX() >
currentTextChunk.getHeight() * X_DIFFERENCE_EPSILON) {
return true;
}
}
return false;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/AutoTaggingProcessor.java
+1178
View File
File diff suppressed because it is too large Load Diff
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/CaptionProcessor.java
+151
View File
@@ -0,0 +1,151 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticCaption;
import org.verapdf.wcag.algorithms.entities.SemanticFigure;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.CaptionUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.util.List;
/**
* Processor for detecting and linking captions to figures and tables.
* Identifies text nodes that are likely captions based on proximity and content.
*/
public class CaptionProcessor {
private static final double CAPTION_PROBABILITY = 0.75;
private static final double CAPTION_VERTICAL_OFFSET_RATIO = 1;
private static final double CAPTION_HORIZONTAL_OFFSET_RATIO = 1;
private static final double SUBTLE_IMAGE_RATIO_THRESHOLD = 0.01;
/**
* Processes content to identify and link captions to images and tables.
*
* @param contents the list of content objects to process
*/
public static void processCaptions(List<IObject> contents) {
DocumentProcessor.setIndexesForContentsList(contents);
SemanticFigure imageNode = null;
SemanticTextNode lastTextNode = null;
for (IObject content : contents) {
if (content == null) {
continue;
}
if (content instanceof SemanticTextNode) {
SemanticTextNode textNode = (SemanticTextNode) content;
if (textNode.isSpaceNode() || textNode.isEmpty()) {
continue;
}
if (imageNode != null && isTextNotContainedInImage(imageNode, textNode)) {
acceptImageCaption(contents, imageNode, lastTextNode, textNode);
imageNode = null;
}
lastTextNode = textNode;
} else if (content instanceof ImageChunk && !isImageSubtle((ImageChunk) content)) {
if (imageNode != null && isTextNotContainedInImage(imageNode, lastTextNode)) {
acceptImageCaption(contents, imageNode, lastTextNode, null);
lastTextNode = null;
}
imageNode = new SemanticFigure((ImageChunk) content);
imageNode.setRecognizedStructureId(content.getRecognizedStructureId());
} else if (content instanceof TableBorder && !((TableBorder) content).isTextBlock()) {
if (imageNode != null && isTextNotContainedInImage(imageNode, lastTextNode)) {
acceptImageCaption(contents, imageNode, lastTextNode, null);
lastTextNode = null;
}
ImageChunk imageChunk = new ImageChunk(content.getBoundingBox());
imageChunk.setRecognizedStructureId(content.getRecognizedStructureId());
imageNode = new SemanticFigure(imageChunk);
imageNode.setRecognizedStructureId(content.getRecognizedStructureId());
}
}
if (imageNode != null) {
acceptImageCaption(contents, imageNode, lastTextNode, null);
}
// for (IObject content1 : contents) {
// if (content1 instanceof SemanticTextNode) {
// SemanticTextNode textNode = (SemanticTextNode)content1;
// for (IObject content2 : contents) {
// if (content2 instanceof ImageChunk) {
// SemanticFigure imageNode = new SemanticFigure((ImageChunk) content2);
// acceptImageCaption(imageNode, textNode, textNode);
// }
// }
// }
// }
}
private static boolean isImageSubtle(ImageChunk imageChunk) {
double imageHeight = imageChunk.getHeight();
double imageWidth = imageChunk.getWidth();
if (NodeUtils.areCloseNumbers(imageWidth, 0) || NodeUtils.areCloseNumbers(imageHeight, 0)) {
return true;
}
double aspectRatio = Math.min(imageWidth, imageHeight) / Math.max(imageWidth, imageHeight);
return aspectRatio < SUBTLE_IMAGE_RATIO_THRESHOLD;
}
/**
* Checks if a text node is not contained within an image's bounding box.
*
* @param image the image to check against
* @param text the text node to check
* @return true if the text is outside the image bounds, false otherwise
*/
public static boolean isTextNotContainedInImage(SemanticFigure image, SemanticTextNode text) {
if (text == null) {
return true;
}
double textSize = text.getFontSize();
return !image.getBoundingBox().contains(text.getBoundingBox(),
textSize * CAPTION_HORIZONTAL_OFFSET_RATIO,
textSize * CAPTION_VERTICAL_OFFSET_RATIO);
}
private static void acceptImageCaption(List<IObject> contents, SemanticFigure imageNode,
SemanticTextNode previousNode, SemanticTextNode nextNode) {
if (imageNode.getImages().isEmpty()) {
return;
}
double previousCaptionProbability = CaptionUtils.imageCaptionProbability(previousNode, imageNode);
double nextCaptionProbability = CaptionUtils.imageCaptionProbability(nextNode, imageNode);
double captionProbability;
SemanticTextNode captionNode;
if (previousCaptionProbability > nextCaptionProbability) {
captionProbability = previousCaptionProbability;
captionNode = previousNode;
} else {
captionProbability = nextCaptionProbability;
captionNode = nextNode;
}
if (captionProbability >= CAPTION_PROBABILITY) {
SemanticCaption semanticCaption = new SemanticCaption(captionNode);
contents.set(captionNode.getIndex(), semanticCaption);
semanticCaption.setLinkedContentId(imageNode.getRecognizedStructureId());
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ClusterTableProcessor.java
+120
View File
@@ -0,0 +1,120 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.tables.Table;
import org.verapdf.wcag.algorithms.entities.tables.TableToken;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.semanticalgorithms.consumers.ClusterTableConsumer;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.util.ArrayList;
import java.util.List;
/**
* Table processor that uses clustering algorithms to detect tables.
* Identifies tables by analyzing spatial relationships between text chunks.
*/
public class ClusterTableProcessor extends AbstractTableProcessor {
@Override
protected List<List<TableBorder>> getTables(List<List<IObject>> contents, List<Integer> pageNumbers) {
List<List<TableBorder>> tables = new ArrayList<>();
for (int pageNumber : pageNumbers) {
tables.add(processClusterDetectionTables(contents.get(pageNumber)));
}
return tables;
}
/**
* Detects tables on a single page using cluster-based detection.
*
* @param contents the page contents to analyze
* @return a list of detected table borders
*/
public static List<TableBorder> processClusterDetectionTables(List<IObject> contents) {
ClusterTableConsumer clusterTableConsumer = new ClusterTableConsumer();
for (IObject content : contents) {
if (content instanceof TextChunk) {
TextChunk textChunk = (TextChunk) content;
if (textChunk.isWhiteSpaceChunk() || textChunk.isEmpty()) {
continue;
}
List<TextChunk> splitChunks = TextChunkUtils.splitTextChunkByWhiteSpaces(textChunk);
for (TextChunk splitChunk : splitChunks) {
SemanticTextNode semanticTextNode = new SemanticTextNode(splitChunk);
clusterTableConsumer.accept(new TableToken(splitChunk, semanticTextNode), semanticTextNode);
}
// } else if (content instanceof ImageChunk) {
// SemanticFigure semanticFigure = new SemanticFigure((ImageChunk) content);
// clusterTableConsumer.accept(new TableToken((ImageChunk) content, semanticFigure), semanticFigure);
}
}
clusterTableConsumer.processEnd();
List<TableBorder> result = new ArrayList<>();
for (Table table : clusterTableConsumer.getTables()) {
TableBorder tableBorder = table.createTableBorderFromTable();
if (tableBorder != null) {
setTableCellsSemanticTypes(tableBorder);
result.add(tableBorder);
}
}
return result;
}
private static void setTableCellsSemanticTypes(TableBorder table) {
for (int rowNumber = 0; rowNumber < table.getNumberOfRows(); rowNumber++) {
for (int colNumber = 0; colNumber < table.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = table.getCell(rowNumber, colNumber);
if (cell.getColNumber() == colNumber && cell.getRowNumber() == rowNumber) {
cell.setSemanticType(rowNumber == 0 ? SemanticType.TABLE_HEADER : SemanticType.TABLE_CELL);
}
}
}
}
// public static void findListAndTablesImageMethod(List<SemanticTextNode> nodes) {
// ClusterTableConsumer clusterTableConsumer = new ClusterTableConsumer();
// for (SemanticTextNode textNode : nodes) {
// ImageChunk imageChunk = new ImageChunk(textNode.getBoundingBox());
// SemanticFigure figure = new SemanticFigure(imageChunk);
// clusterTableConsumer.accept(new TableToken(imageChunk, figure), figure);
//// if (chunk instanceof TextChunk) {
//// SemanticTextNode semanticTextNode = new SemanticTextNode((TextChunk)chunk);
//// clusterTableConsumer.accept(new TableToken((TextChunk)chunk, semanticTextNode), semanticTextNode);
//// } else if (chunk instanceof ImageChunk) {
//// SemanticFigure semanticFigure = new SemanticFigure((ImageChunk) chunk);
//// clusterTableConsumer.accept(new TableToken((ImageChunk)chunk, semanticFigure), semanticFigure);
//// }
//
// }
// // if (recognitionArea.isValid()) {
//// List<INode> restNodes = new ArrayList<>(recognize());
//// init();
//// restNodes.add(root);
//// for (INode restNode : restNodes) {
//// accept(restNode);
//// }
//// }
// clusterTableConsumer.processEnd();
// System.out.println("test");
// }
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ContentFilterProcessor.java
+156
View File
@@ -0,0 +1,156 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.IChunk;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Processor for filtering and cleaning PDF content.
* Removes hidden text, out-of-page content, backgrounds, and other artifacts.
*/
public class ContentFilterProcessor {
private static final Logger LOGGER = Logger.getLogger(ContentFilterProcessor.class.getCanonicalName());
/**
* Filters and cleans page contents based on configuration.
*
* @param inputPdfName the path to the PDF file
* @param contents the raw page contents
* @param pageNumber the page number (0-indexed)
* @param config the configuration settings
* @return the filtered list of content objects
* @throws IOException if unable to process the content
*/
public static List<IObject> getFilteredContents(String inputPdfName, List<IChunk> contents, int pageNumber,
Config config) throws IOException {
List<IObject> pageContents = new ArrayList<>(contents);
TextProcessor.removeSameTextChunks(pageContents);
pageContents = DocumentProcessor.removeNullObjectsFromList(pageContents);
TextProcessor.removeTextDecorationImages(pageContents);
pageContents = DocumentProcessor.removeNullObjectsFromList(pageContents);
if (config.getFilterConfig().isFilterTinyText()) {
TextProcessor.filterTinyText(pageContents);
pageContents = DocumentProcessor.removeNullObjectsFromList(pageContents);
}
if (config.getFilterConfig().isFilterOutOfPage()) {
filterOutOfPageContents(pageNumber, pageContents);
pageContents = DocumentProcessor.removeNullObjectsFromList(pageContents);
}
TextProcessor.mergeCloseTextChunks(pageContents);
pageContents = DocumentProcessor.removeNullObjectsFromList(pageContents);
TextProcessor.trimTextChunksWhiteSpaces(pageContents);
filterConsecutiveSpaces(pageContents);
pageContents = splitTextChunksByWhiteSpacesInPageContents(pageContents);
// HiddenText detection moved to DocumentProcessor (sequential post-processing)
// to avoid ContrastRatioConsumer per-thread PDF rendering overhead
double replacementCharRatio = TextProcessor.measureReplacementCharRatio(pageContents);
StaticLayoutContainers.setReplacementCharRatio(pageNumber, replacementCharRatio);
if (replacementCharRatio >= 0.3) {
LOGGER.log(Level.WARNING,
"Page {0}: {1,number,#.#%} of characters are replacement characters (U+FFFD). "
+ "This PDF likely contains CID-keyed fonts without ToUnicode mappings. "
+ "Text extraction may be incomplete. Consider enabling hybrid OCR fallback with --hybrid docling-fast.",
new Object[]{pageNumber + 1, replacementCharRatio});
}
TextProcessor.replaceUndefinedCharacters(pageContents, config.getReplaceInvalidChars());
processBackgrounds(pageNumber, pageContents);
return pageContents;
}
/**
* Detects and removes background elements from page contents.
*
* @param pageNumber the page number (0-indexed)
* @param contents the page contents to process
*/
public static void processBackgrounds(int pageNumber, List<IObject> contents) {
BoundingBox pageBoundingBox = DocumentProcessor.getPageBoundingBox(pageNumber);
if (pageBoundingBox == null) {
return;
}
Set<LineArtChunk> backgrounds = new HashSet<>();
for (IObject content : contents) {
if (content instanceof LineArtChunk) {
if (isBackground(content, pageBoundingBox)) {
backgrounds.add((LineArtChunk) content);
}
}
}
if (!backgrounds.isEmpty()) {
LOGGER.log(Level.WARNING, "Detected background on page " + pageNumber);
contents.removeAll(backgrounds);
}
}
private static void filterConsecutiveSpaces(List<IObject> pageContents) {
for (IObject object : pageContents) {
if (object instanceof TextChunk) {
((TextChunk) object).compressSpaces();
}
}
}
private static boolean isBackground(IObject content, BoundingBox pageBoundingBox) {
return (content.getBoundingBox().getWidth() > 0.5 * pageBoundingBox.getWidth() &&
content.getBoundingBox().getHeight() > 0.1 * pageBoundingBox.getHeight()) ||
(content.getBoundingBox().getWidth() > 0.1 * pageBoundingBox.getWidth() &&
content.getBoundingBox().getHeight() > 0.5 * pageBoundingBox.getHeight());
}
private static void filterOutOfPageContents(int pageNumber, List<IObject> contents) {
BoundingBox pageBoundingBox = DocumentProcessor.getPageBoundingBox(pageNumber);
if (pageBoundingBox == null) {
return;
}
pageBoundingBox.move(-pageBoundingBox.getLeftX(), -pageBoundingBox.getBottomY());
for (int index = 0; index < contents.size(); index++) {
IObject object = contents.get(index);
if (object != null && pageBoundingBox.notOverlaps(object.getBoundingBox())) {
contents.set(index, null);
}
}
}
private static List<IObject> splitTextChunksByWhiteSpacesInPageContents(List<IObject> contents) {
List<IObject> newContents = new ArrayList<>();
for (IObject object : contents) {
if (object instanceof TextChunk) {
TextChunk textChunk = (TextChunk) object;
List<TextChunk> splitChunks = TextChunkUtils.splitTextChunkByWhiteSpaces(textChunk);
newContents.addAll(splitChunks);
} else {
newContents.add(object);
}
}
return newContents;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/DocumentProcessor.java
+882
View File
@@ -0,0 +1,882 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.hybrid.ElementMetadata;
import org.opendataloader.pdf.processors.readingorder.XYCutPlusPlusSorter;
import org.opendataloader.pdf.json.JsonWriter;
import org.opendataloader.pdf.markdown.MarkdownGenerator;
import org.opendataloader.pdf.markdown.MarkdownGeneratorFactory;
import org.opendataloader.pdf.markdown.MarkdownSyntax;
import org.opendataloader.pdf.html.HtmlGenerator;
import org.opendataloader.pdf.html.HtmlGeneratorFactory;
import org.opendataloader.pdf.pdf.PDFWriter;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.text.TextGenerator;
import org.opendataloader.pdf.utils.ContentSanitizer;
import org.opendataloader.pdf.utils.FileUtils;
import org.opendataloader.pdf.utils.ImagesUtils;
import org.opendataloader.pdf.utils.TextNodeUtils;
import org.verapdf.as.ASAtom;
import org.verapdf.containers.StaticCoreContainers;
import org.verapdf.cos.COSDictionary;
import org.verapdf.cos.COSObjType;
import org.verapdf.cos.COSObject;
import org.verapdf.cos.COSTrailer;
import org.verapdf.exceptions.InvalidPasswordException;
import org.verapdf.gf.model.impl.containers.StaticStorages;
import org.verapdf.gf.model.impl.cos.GFCosInfo;
import org.verapdf.gf.model.impl.sa.GFSAPDFDocument;
import org.verapdf.parser.PDFFlavour;
import org.verapdf.pd.PDDocument;
import org.verapdf.tools.StaticResources;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.TableBordersCollection;
import org.verapdf.wcag.algorithms.semanticalgorithms.consumers.LinesPreprocessingConsumer;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.xmp.containers.StaticXmpCoreContainers;
import org.opendataloader.pdf.exceptions.InvalidPdfFileException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.ForkJoinPool;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Main processor for PDF document analysis and output generation.
* Coordinates the extraction, processing, and generation of various output formats.
*/
public class DocumentProcessor {
private static final Logger LOGGER = Logger.getLogger(DocumentProcessor.class.getCanonicalName());
/**
* Releases PDF resources to prevent file locks and memory leaks.
* - Closes PDDocument to free OS file handles (required for file deletion)
* - Clears static containers to remove lingering references
* Should always be called in a finally block.
*/
private static void closePdfResources() {
clearCleanupStep("PDDocument", () -> {
PDDocument document = StaticResources.getDocument();
if (document != null) {
document.close();
}
});
clearCleanupStep("ImagesUtils", StaticContainers::closeImagesUtils);
clearCleanupStep("StaticResources", StaticResources::clear);
clearCleanupStep("StaticContainers", () -> StaticContainers.updateContainers(null));
clearCleanupStep(
"GFStaticContainers",
org.verapdf.gf.model.impl.containers.StaticContainers::clearAllContainers
);
clearCleanupStep("StaticLayoutContainers", StaticLayoutContainers::clearContainers);
clearCleanupStep("StaticStorages", StaticStorages::clearAllContainers);
clearCleanupStep("StaticCoreContainers", StaticCoreContainers::clearAllContainers);
clearCleanupStep("StaticXmpCoreContainers", StaticXmpCoreContainers::clearAllContainers);
}
/**
* Executes a cleanup step safely without interrupting subsequent steps.
*
* Each cleanup action is isolated so that a failure in one step
* does not prevent the remaining cleanup operations from running.
* Errors are logged for debugging purposes.
*/
private static void clearCleanupStep(String name, Runnable cleanup) {
try {
cleanup.run();
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Error clearing " + name, e);
}
}
/**
* Processes a PDF file and generates the configured outputs.
*
* @param inputPdfName the path to the input PDF file
* @param config the configuration settings
* @throws IOException if unable to process the file
*/
public static void processFile(String inputPdfName, Config config) throws IOException {
processFileWithResult(inputPdfName, config);
}
/**
* Processes a PDF file and returns a {@link ProcessingResult} containing
* metadata collected during processing (e.g., hybrid server timings).
*
* @param inputPdfName the path to the input PDF file
* @param config the configuration settings
* @return processing result with optional metadata
* @throws IOException if unable to process the file
*/
public static ProcessingResult processFileWithResult(String inputPdfName, Config config) throws IOException {
try {
// Phase 1: Extract
ExtractionResult extraction = extractContents(inputPdfName, config);
// Phase 2: Output (JSON/MD/HTML/PDF/Text)
long t0 = System.nanoTime();
generateOutputs(inputPdfName, extraction.getContents(), config, extraction.getElementMetadata());
long outputNs = System.nanoTime() - t0;
return new ProcessingResult(extraction.getHybridTimings(), extraction.getExtractionNs(), outputNs);
} finally {
// Always release resources, even if processing threw. closePdfResources
// logs and swallows per-step failures so cleanup cannot mask the original
// processing exception.
closePdfResources();
}
}
/**
* Run the extraction pipeline only (preprocessing + content extraction + sanitization).
* Does not generate any output files. The returned {@link ExtractionResult} can be
* passed to {@link org.opendataloader.pdf.api.AutoTagger} or used to generate
* specific output formats.
*
* <p>Structured processing (headings, lists, tables, captions) is always enabled
* because auto-tagging and all structured output formats depend on it.
*
* @param inputPdfName path to the input PDF file
* @param config configuration
* @return extraction result with contents and timing metadata
*/
public static ExtractionResult extractContents(String inputPdfName, Config config) throws IOException {
long t0 = System.nanoTime();
preprocessing(inputPdfName, config);
calculateDocumentInfo();
Set<Integer> pagesToProcess = getValidPageNumbers(config);
List<List<IObject>> contents;
if (StaticLayoutContainers.isUseStructTree()) {
contents = TaggedDocumentProcessor.processDocument(inputPdfName, config, pagesToProcess);
} else if (config.isHybridEnabled()) {
contents = HybridDocumentProcessor.processDocument(inputPdfName, config, pagesToProcess);
} else {
contents = processDocument(inputPdfName, config, pagesToProcess);
}
sortContents(contents, config);
ContentSanitizer contentSanitizer = new ContentSanitizer(config.getFilterConfig().getFilterRules(),
config.getFilterConfig().isFilterSensitiveData());
contentSanitizer.sanitizeContents(contents);
long extractionNs = System.nanoTime() - t0;
// Re-key metadata by actual IObject IDs in contents.
// After enrichment, IObject recognizedStructureIds may differ from transformer-assigned IDs.
// Match metadata to IObjects by bbox proximity.
Map<Long, ElementMetadata> rawMetadata = HybridDocumentProcessor.getLastElementMetadata();
Map<Long, ElementMetadata> remappedMetadata = remapMetadataToContents(rawMetadata, contents);
return new ExtractionResult(contents, extractionNs, HybridDocumentProcessor.getLastHybridTimings(),
remappedMetadata);
}
/**
* Validates and filters page numbers from config against actual document pages.
* Logs warnings for pages that don't exist in the document.
*
* @param config the configuration containing page selection
* @return Set of valid 0-indexed page numbers to process, or null for all pages
*/
private static Set<Integer> getValidPageNumbers(Config config) {
List<Integer> requestedPages = config.getPageNumbers();
if (requestedPages.isEmpty()) {
return null; // null means process all pages
}
int totalPages = StaticContainers.getDocument().getNumberOfPages();
Set<Integer> validPages = new LinkedHashSet<>();
List<Integer> invalidPages = new ArrayList<>();
for (Integer page : requestedPages) {
int zeroIndexed = page - 1; // Convert 1-based to 0-based
if (zeroIndexed >= 0 && zeroIndexed < totalPages) {
validPages.add(zeroIndexed);
} else {
invalidPages.add(page);
}
}
if (!invalidPages.isEmpty()) {
LOGGER.log(Level.WARNING,
"Requested pages {0} do not exist in document (total pages: {1}). Processing only existing pages: {2}",
new Object[]{invalidPages, totalPages,
validPages.stream().map(p -> p + 1).collect(Collectors.toList())});
}
if (validPages.isEmpty()) {
LOGGER.log(Level.WARNING,
"No valid pages to process. Document has {0} pages but requested: {1}",
new Object[]{totalPages, requestedPages});
}
return validPages;
}
@SuppressWarnings("unchecked")
private static List<List<IObject>> processDocument(String inputPdfName, Config config, Set<Integer> pagesToProcess) throws IOException {
int totalPages = StaticContainers.getDocument().getNumberOfPages();
List<List<IObject>> contents = new ArrayList<>(Collections.nCopies(totalPages, null));
// Capture ALL ThreadLocal state from main thread for propagation to workers
final var document = StaticContainers.getDocument();
final var pdDocument = StaticResources.getDocument();
final var tableBordersCollection = StaticContainers.getTableBordersCollection();
final var accumulatedNodeMapper = StaticContainers.getAccumulatedNodeMapper();
final var objectKeyMapper = StaticContainers.getObjectKeyMapper();
final var linesCollection = StaticContainers.getLinesCollection();
final boolean keepLineBreaks = StaticContainers.isKeepLineBreaks();
final boolean isDataLoader = StaticContainers.isDataLoader();
final var isIgnoreCharsWithoutUnicode = StaticContainers.getIsIgnoreCharactersWithoutUnicode();
// Capture StaticLayoutContainers state (shared mutable — synchronized list for headings)
final var headings = StaticLayoutContainers.getHeadings();
final long contentId = StaticLayoutContainers.getCurrentContentId();
final boolean useStructTree = StaticLayoutContainers.isUseStructTree();
final var embeddedImageBytesMap = StaticLayoutContainers.getEmbeddedImageBytesMap();
// Runnable that propagates ThreadLocal state to the current (worker) thread
final Runnable propagateState = () -> {
StaticResources.setDocument(pdDocument);
// veraPDF StaticContainers
StaticContainers.setDocument(document);
StaticContainers.setTableBordersCollection(tableBordersCollection);
StaticContainers.setAccumulatedNodeMapper(accumulatedNodeMapper);
StaticContainers.setObjectKeyMapper(objectKeyMapper);
StaticContainers.setLinesCollection(linesCollection);
StaticContainers.setKeepLineBreaks(keepLineBreaks);
StaticContainers.setIsDataLoader(isDataLoader);
StaticContainers.setIsIgnoreCharactersWithoutUnicode(isIgnoreCharsWithoutUnicode);
StaticContainers.setFileName(inputPdfName);
StaticContainers.setPassword(config.getPassword());
// Project StaticLayoutContainers — share the same headings list across workers
StaticLayoutContainers.setHeadings(headings);
StaticLayoutContainers.setCurrentContentId(contentId);
StaticLayoutContainers.setIsUseStructTree(useStructTree);
StaticLayoutContainers.setEmbeddedImageBytesMap(embeddedImageBytesMap);
};
// Pre-fetch all page artifacts on main thread (document access is ThreadLocal)
List<?>[] pageArtifacts = new List<?>[totalPages];
for (int i = 0; i < totalPages; i++) {
pageArtifacts[i] = document.getArtifacts(i);
}
int parallelism = config.getThreads();
ForkJoinPool pool = new ForkJoinPool(parallelism);
int pagesToProcessCount = (pagesToProcess != null) ? pagesToProcess.size() : totalPages;
LOGGER.log(Level.INFO, "Processing {0} pages with {1} threads", new Object[]{pagesToProcessCount, parallelism});
try {
// Loop 1: ContentFilter per-page (largest bottleneck)
pool.submit(() ->
IntStream.range(0, totalPages).parallel().forEach(pageNumber -> {
try {
propagateState.run();
if (shouldProcessPage(pageNumber, pagesToProcess)) {
List<IObject> pageContents = ContentFilterProcessor.getFilteredContents(inputPdfName,
(List) pageArtifacts[pageNumber], pageNumber, config);
contents.set(pageNumber, pageContents);
} else {
contents.set(pageNumber, new ArrayList<>());
}
} catch (IOException e) {
throw new UncheckedIOException(e);
}
})
).get();
// Hidden text detection: sequential post-processing (requires ContrastRatioConsumer
// which renders PDF pages — not safe to parallelize due to per-thread PDF file I/O)
if (config.getFilterConfig().isFilterHiddenText()) {
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
if (shouldProcessPage(pageNumber, pagesToProcess)) {
List<IObject> pageContents = HiddenTextProcessor.findHiddenText(contents.get(pageNumber), true);
contents.set(pageNumber, pageContents);
}
}
}
// Structured processing is always enabled — auto-tagging needs headings,
// lists, tables, and captions regardless of output format flags.
boolean structured = true;
// ClusterTableProcessor: whole-document (must be sequential)
if (structured && config.isClusterTableMethod()) {
new ClusterTableProcessor().processTables(contents);
}
// Loop 2: TableBorder + TextLine per-page
pool.submit(() ->
IntStream.range(0, totalPages).parallel().forEach(pageNumber -> {
if (!shouldProcessPage(pageNumber, pagesToProcess)) {
return;
}
propagateState.run();
List<IObject> pageContents = contents.get(pageNumber);
if (structured) {
TextDecorationProcessor.processStrikethroughAndUnderlinedText(pageContents, pageNumber, config.isDetectStrikethrough());
pageContents = TableBorderProcessor.processTableBorders(pageContents, pageNumber);
pageContents = pageContents.stream().filter(x -> !(x instanceof LineChunk)).collect(Collectors.toList());
pageContents = SpecialTableProcessor.detectSpecialTables(pageContents);
}
pageContents = TextLineProcessor.processTextLines(pageContents);
contents.set(pageNumber, pageContents);
})
).get();
if (structured) {
// Cross-page operations (must be sequential)
HeaderFooterProcessor.processHeadersAndFooters(contents, false);
new TableOfContentsProcessor().processTableOfContents(contents);
ListProcessor.processLists(contents, false);
}
// Loop 3: Paragraph + Heading per-page (always need ParagraphProcessor for text output)
pool.submit(() ->
IntStream.range(0, totalPages).parallel().forEach(pageNumber -> {
if (!shouldProcessPage(pageNumber, pagesToProcess)) {
return;
}
propagateState.run();
List<IObject> pageContents = contents.get(pageNumber);
pageContents = ParagraphProcessor.processParagraphs(pageContents);
if (structured) {
pageContents = ListProcessor.processListsFromTextNodes(pageContents);
HeadingProcessor.processHeadings(pageContents, false);
}
contents.set(pageNumber, pageContents);
})
).get();
// Sequential ID assignment (must be in page order, before CaptionProcessor)
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
if (shouldProcessPage(pageNumber, pagesToProcess)) {
setIDs(contents.get(pageNumber));
}
}
// Caption detection runs after setIDs so that recognizedStructureId is available
// for linking captions to figures/tables
if (structured) {
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
if (shouldProcessPage(pageNumber, pagesToProcess)) {
CaptionProcessor.processCaptions(contents.get(pageNumber));
}
}
}
if (structured) {
// Cross-page post-processing (must be sequential)
ListProcessor.checkNeighborLists(contents);
TableBorderProcessor.checkNeighborTables(contents);
HeadingProcessor.detectHeadingsLevels();
LevelProcessor.detectLevels(contents);
}
} catch (Exception e) {
throw new IOException("Parallel page processing failed", e);
} finally {
pool.shutdown();
}
return contents;
}
/**
* Checks if a page should be processed based on the filter.
*
* @param pageNumber 0-indexed page number
* @param pagesToProcess set of valid page numbers to process, or null for all pages
* @return true if the page should be processed
*/
/**
* Filters ElementMetadata down to entries whose transformer-assigned ID still
* matches an IObject in the post-enrichment contents. This is deliberately
* ID-based (not positional): sorting, filtering, and enrichment can reorder
* or drop IObjects, so positional matching would attach the wrong
* confidence/source label to an element. IObjects whose ID was rewritten
* during enrichment simply lose their metadata — preferable to a wrong one.
*/
private static Map<Long, ElementMetadata> remapMetadataToContents(
Map<Long, ElementMetadata> rawMetadata, List<List<IObject>> contents) {
if (rawMetadata == null || rawMetadata.isEmpty()) return Collections.emptyMap();
Map<Long, ElementMetadata> remapped = new LinkedHashMap<>();
for (List<IObject> pageContents : contents) {
for (IObject obj : pageContents) {
collectMetadata(obj, rawMetadata, remapped);
}
}
return remapped;
}
/**
* Walks an IObject tree and copies any metadata entry keyed by its
* recognized structure id into {@code remapped}. Containers like
* {@code ListItem} hold their own children via {@code getContents()}, so
* a shallow iteration over the top-level page list would miss nested
* images / pictures — their metadata (ai_score, source label, caption)
* would silently disappear from the JSON output. We recurse through the
* containers we actually emit at this level (lists, tables, headers,
* footers); leaf nodes terminate naturally.
*/
private static void collectMetadata(IObject obj,
Map<Long, ElementMetadata> rawMetadata,
Map<Long, ElementMetadata> remapped) {
if (obj == null) return;
Long id = obj.getRecognizedStructureId();
if (id != null && id != 0L) {
ElementMetadata meta = rawMetadata.get(id);
if (meta != null) {
remapped.put(id, meta);
}
}
// Recurse into every container the JSON serializers walk. This keeps
// the metadata visibility surface aligned with the serialized tree —
// any image / picture / heading that ends up in the JSON output also
// gets its ElementMetadata copied through. Add new container types
// here when their serializer descends into child IObjects.
if (obj instanceof org.verapdf.wcag.algorithms.entities.lists.ListItem) {
for (IObject child : ((org.verapdf.wcag.algorithms.entities.lists.ListItem) obj).getContents()) {
collectMetadata(child, rawMetadata, remapped);
}
} else if (obj instanceof org.verapdf.wcag.algorithms.entities.lists.PDFList) {
for (org.verapdf.wcag.algorithms.entities.lists.ListItem item :
((org.verapdf.wcag.algorithms.entities.lists.PDFList) obj).getListItems()) {
collectMetadata(item, rawMetadata, remapped);
}
} else if (obj instanceof org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder) {
org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder table =
(org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder) obj;
if (table.isTextBlock()) {
// Text-block tables serialize as a single anonymous cell. Recurse
// through the cell IObject itself so its own structureId metadata
// is captured alongside the children — going straight to
// getContents() would skip the cell-level entry.
org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell cell = table.getCell(0, 0);
if (cell != null) {
collectMetadata(cell, rawMetadata, remapped);
}
} else {
for (org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow row : table.getRows()) {
for (org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell cell : row.getCells()) {
collectMetadata(cell, rawMetadata, remapped);
}
}
}
} else if (obj instanceof org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell) {
for (IObject child : ((org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell) obj).getContents()) {
collectMetadata(child, rawMetadata, remapped);
}
} else if (obj instanceof org.verapdf.wcag.algorithms.entities.SemanticHeaderOrFooter) {
for (IObject child : ((org.verapdf.wcag.algorithms.entities.SemanticHeaderOrFooter) obj).getContents()) {
collectMetadata(child, rawMetadata, remapped);
}
}
}
private static boolean shouldProcessPage(int pageNumber, Set<Integer> pagesToProcess) {
return pagesToProcess == null || pagesToProcess.contains(pageNumber);
}
/**
* Writes the configured output files (JSON/MD/HTML/PDF/Text/images/tagged PDF)
* from already-extracted contents.
*
* <p><strong>Internal API. Do not call directly.</strong> This method is
* {@code public} only so the {@link org.opendataloader.pdf.api.OutputWriter}
* facade in the {@code api} package can delegate to it. The signature
* (notably the {@code List<List<IObject>>} and
* {@code Map<Long, ElementMetadata>} parameters) is an implementation
* detail and may change in any release. External callers must use
* {@link org.opendataloader.pdf.api.OutputWriter#writeOutputs}, which is
* the stable public API.
*/
public static void generateOutputs(String inputPdfName, List<List<IObject>> contents, Config config,
Map<Long, ElementMetadata> elementMetadata) throws IOException {
// Stdout mode: write primary format to stdout, skip file I/O
if (config.isOutputStdout()) {
java.io.Writer stdoutWriter = new java.io.BufferedWriter(
new java.io.OutputStreamWriter(System.out, java.nio.charset.StandardCharsets.UTF_8));
if (config.isGenerateText()) {
TextGenerator textGenerator = new TextGenerator(stdoutWriter, config);
textGenerator.writeToText(contents);
stdoutWriter.flush();
} else if (config.isGenerateMarkdown()) {
MarkdownGenerator markdownGenerator = new MarkdownGenerator(stdoutWriter, config);
markdownGenerator.writeToMarkdown(contents);
stdoutWriter.flush();
}
// JSON and HTML stdout not yet supported
return;
}
File inputPDF = new File(inputPdfName);
new File(config.getOutputFolder()).mkdirs();
if (!config.isImageOutputOff() && (config.isGenerateHtml() || config.isGenerateMarkdown() || config.isGenerateJSON())) {
String imagesDirectory;
if (config.getImageDir() != null && !config.getImageDir().isEmpty()) {
imagesDirectory = config.getImageDir();
} else {
String fileName = Paths.get(inputPdfName).getFileName().toString();
imagesDirectory = config.getOutputFolder() + File.separator + FileUtils.getBaseName(fileName) + MarkdownSyntax.IMAGES_DIRECTORY_SUFFIX;
}
StaticLayoutContainers.setImagesDirectory(imagesDirectory);
ImagesUtils imagesUtils = new ImagesUtils();
imagesUtils.write(contents);
}
if (config.isGenerateTaggedPDF()) {
AutoTaggingProcessor.createTaggedPDF(inputPDF, config.getOutputFolder(),
StaticResources.getDocument(), contents);
}
if (config.isGeneratePDF()) {
PDFWriter pdfWriter = new PDFWriter();
pdfWriter.updatePDF(inputPDF, config.getPassword(), config.getOutputFolder(), contents);
}
if (config.isGenerateJSON()) {
JsonWriter.writeToJson(inputPDF, config.getOutputFolder(), contents, elementMetadata,
null, config.isIncludeHeaderFooter());
}
if (config.isGenerateMarkdown()) {
try (MarkdownGenerator markdownGenerator = MarkdownGeneratorFactory.getMarkdownGenerator(inputPDF,
config)) {
markdownGenerator.writeToMarkdown(contents);
}
}
if (config.isGenerateHtml()) {
try (HtmlGenerator htmlGenerator = HtmlGeneratorFactory.getHtmlGenerator(inputPDF, config)) {
htmlGenerator.writeToHtml(contents);
}
}
if (config.isGenerateText()) {
try (TextGenerator textGenerator = new TextGenerator(inputPDF, config)) {
textGenerator.writeToText(contents);
}
}
}
/**
* Performs preprocessing on a PDF document.
* Initializes static containers and parses the document structure.
*
* @param pdfName the path to the PDF file
* @param config the configuration settings
* @throws IOException if unable to read the PDF file
*/
public static void preprocessing(String pdfName, Config config) throws IOException {
LOGGER.log(Level.INFO, () -> "File name: " + pdfName);
validatePdfMagicNumber(pdfName);
updateStaticContainers(config);
PDDocument pdDocument;
try {
pdDocument = new PDDocument(pdfName);
} catch (InvalidPasswordException pw) {
// Encrypted PDFs are not a content-validity failure — let the
// password-handling branch in callers (e.g. CLIMain) take over.
throw pw;
} catch (IOException cause) {
// Magic number was present, so the user expected a real PDF, but
// veraPDF could not parse the document (truncated download, body
// corruption, missing xref). Surface a friendly message instead
// of letting the raw veraPDF IOException leak as a stack trace.
throw new InvalidPdfFileException(
"'" + displayName(pdfName) + "' is not a valid PDF file (corrupted or truncated content).",
cause);
}
StaticResources.setDocument(pdDocument);
GFSAPDFDocument document = new GFSAPDFDocument(pdDocument);
// org.verapdf.gf.model.impl.containers.StaticContainers.setFlavour(Collections.singletonList(PDFAFlavour.WCAG_2_2));
StaticResources.setFlavour(Collections.singletonList(Objects.equals(pdDocument.getVersion(), 2.0F) ?
PDFFlavour.WCAG_2_2_PDF_2_0_HUMAN : PDFFlavour.WCAG_2_2_HUMAN));
StaticStorages.setIsFilterInvisibleLayers(config.getFilterConfig().isFilterHiddenOCG());
StaticContainers.setDocument(document);
if (config.isUseStructTree()) {
document.parseStructureTreeRoot();
if (document.getTree() != null) {
StaticLayoutContainers.setIsUseStructTree(true);
} else {
StaticLayoutContainers.setIsUseStructTree(false);
LOGGER.log(Level.WARNING, "The document has no structure tree. The 'use-struct-tree' option will be ignored.");
}
}
StaticContainers.setFileName(pdfName);
StaticContainers.setPassword(config.getPassword());
StaticContainers.setIsDataLoader(true);
StaticContainers.setIsIgnoreCharactersWithoutUnicode(false);
StaticResources.setIsFontProgramsParsing(true);
StaticStorages.setIsIgnoreMCIDs(!StaticLayoutContainers.isUseStructTree());
StaticStorages.setIsAddSpacesBetweenTextPieces(true);
document.parseChunks();
LinesPreprocessingConsumer linesPreprocessingConsumer = new LinesPreprocessingConsumer();
linesPreprocessingConsumer.findTableBorders();
StaticContainers.setTableBordersCollection(new TableBordersCollection(linesPreprocessingConsumer.getTableBorders()));
}
/**
* Verifies the input file contains the PDF magic number ({@code %PDF-})
* within its first 1024 bytes.
*
* <p>ISO 32000-1 §7.5.2 allows the {@code %PDF-} header to appear "near
* the beginning" of the file rather than strictly at byte 0; real-world
* PDFs sometimes have a leading UTF-8 BOM or whitespace. A 1024-byte
* search window matches that tolerance while still rejecting any
* JPG/PNG/HTML/empty file.
*
* @throws InvalidPdfFileException if the magic number is not present
* @throws IOException if the file cannot be opened or read
*/
private static void validatePdfMagicNumber(String pdfName) throws IOException {
Path path = Path.of(pdfName);
byte[] head;
try (InputStream in = Files.newInputStream(path)) {
head = in.readNBytes(1024);
}
byte[] marker = "%PDF-".getBytes(StandardCharsets.US_ASCII);
if (indexOfBytes(head, marker) < 0) {
throw new InvalidPdfFileException(
"'" + displayName(pdfName) + "' is not a valid PDF file (missing %PDF- header).");
}
}
/**
* Path.getFileName() returns null for filesystem roots (e.g. {@code C:\}).
* Fall back to the original input string in that case so the user-facing
* error message is never empty.
*/
private static String displayName(String pdfName) {
Path fileName = Path.of(pdfName).getFileName();
return fileName != null ? fileName.toString() : pdfName;
}
private static int indexOfBytes(byte[] haystack, byte[] needle) {
if (needle.length == 0 || haystack.length < needle.length) {
return -1;
}
int last = haystack.length - needle.length;
outer:
for (int i = 0; i <= last; i++) {
for (int j = 0; j < needle.length; j++) {
if (haystack[i + j] != needle[j]) {
continue outer;
}
}
return i;
}
return -1;
}
private static void updateStaticContainers(Config config) {
StaticResources.clear();
StaticContainers.updateContainers(null);
StaticLayoutContainers.clearContainers();
org.verapdf.gf.model.impl.containers.StaticContainers.clearAllContainers();
StaticCoreContainers.clearAllContainers();
StaticXmpCoreContainers.clearAllContainers();
StaticContainers.setKeepLineBreaks(config.isKeepLineBreaks());
StaticLayoutContainers.setCurrentContentId(1);
StaticLayoutContainers.setEmbedImages(config.isEmbedImages());
StaticLayoutContainers.setImageFormat(config.getImageFormat());
StaticResources.setPassword(config.getPassword());
}
/**
* Assigns unique IDs to each content object.
*
* @param contents the list of content objects
*/
public static void setIDs(List<IObject> contents) {
for (IObject object : contents) {
object.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
}
}
/**
* Sets index values for all content objects across all pages.
*
* @param contents the document contents organized by page
*/
public static void setIndexesForDocumentContents(List<List<IObject>> contents) {
for (List<IObject> pageContents : contents) {
setIndexesForContentsList(pageContents);
}
}
/**
* Sets index values for content objects in a list.
*
* @param contents the list of content objects
*/
public static void setIndexesForContentsList(List<IObject> contents) {
for (int index = 0; index < contents.size(); index++) {
contents.get(index).setIndex(index);
}
}
/**
* Creates a new list with null objects removed.
*
* @param contents the list that may contain null objects
* @return a new list without null objects
*/
public static List<IObject> removeNullObjectsFromList(List<IObject> contents) {
List<IObject> newContents = new ArrayList<>();
for (IObject content : contents) {
if (content != null) {
newContents.add(content);
}
}
return newContents;
}
private static void calculateDocumentInfo() {
PDDocument document = StaticResources.getDocument();
LOGGER.log(Level.INFO, () -> "Number of pages: " + document.getNumberOfPages());
COSTrailer trailer = document.getDocument().getTrailer();
GFCosInfo info = getInfo(trailer);
LOGGER.log(Level.INFO, () -> "Author: " + (info.getAuthor() != null ? info.getAuthor() : info.getXMPCreator()));
LOGGER.log(Level.INFO, () -> "Title: " + (info.getTitle() != null ? info.getTitle() : info.getXMPTitle()));
LOGGER.log(Level.INFO, () -> "Creation date: " + (info.getCreationDate() != null ? info.getCreationDate() : info.getXMPCreateDate()));
LOGGER.log(Level.INFO, () -> "Modification date: " + (info.getModDate() != null ? info.getModDate() : info.getXMPModifyDate()));
}
private static GFCosInfo getInfo(COSTrailer trailer) {
COSObject object = trailer.getKey(ASAtom.INFO);
return new GFCosInfo((COSDictionary) (object != null && object.getType() == COSObjType.COS_DICT ? object.getDirectBase() : COSDictionary.construct().get()));
}
/**
* Gets a debug string representation of a text node.
*
* @param textNode the text node to describe
* @return a string with font, size, color, and content information
*/
public static String getContentsValueForTextNode(SemanticTextNode textNode) {
return String.format("%s: font %s, text size %.2f, text color %s, text content \"%s\"",
textNode.getSemanticType().getValue(), textNode.getFontName(),
textNode.getFontSize(), Arrays.toString(TextNodeUtils.getTextColorOrDefault(textNode)),
textNode.getValue().length() > 15 ? textNode.getValue().substring(0, 15) + "..." : textNode.getValue());
}
/**
* Gets the bounding box for a page.
*
* @param pageNumber the page number (0-indexed)
* @return the page bounding box, or null if not available
*/
public static BoundingBox getPageBoundingBox(int pageNumber) {
PDDocument document = StaticResources.getDocument();
if (document == null) {
return null;
}
double[] cropBox = document.getPage(pageNumber).getCropBox();
if (cropBox == null) {
return null;
}
return new BoundingBox(pageNumber, cropBox);
}
/**
* Sorts page contents by their bounding box positions.
*
* @param contents the list of content objects to sort
* @return a new sorted list of content objects
*/
public static List<IObject> sortPageContents(List<IObject> contents) {
if (contents == null || contents.isEmpty()) {
return contents;
}
List<IObject> sortedContents = new ArrayList<>(contents);
sortedContents.sort((o1, o2) -> {
BoundingBox b1 = o1.getBoundingBox();
BoundingBox b2 = o2.getBoundingBox();
if (b1 == null && b2 == null) {
return 0;
}
if (b1 == null) {
return 1;
}
if (b2 == null) {
return -1;
}
if (!Objects.equals(b1.getPageNumber(), b2.getPageNumber())) {
return b1.getPageNumber() - b2.getPageNumber();
}
if (!Objects.equals(b1.getLastPageNumber(), b2.getLastPageNumber())) {
return b1.getLastPageNumber() - b2.getLastPageNumber();
}
if (!Objects.equals(b1.getTopY(), b2.getTopY())) {
return b2.getTopY() - b1.getTopY() > 0 ? 1 : -1;
}
if (!Objects.equals(b1.getLeftX(), b2.getLeftX())) {
return b1.getLeftX() - b2.getLeftX() > 0 ? 1 : -1;
}
if (!Objects.equals(b1.getBottomY(), b2.getBottomY())) {
return b1.getBottomY() - b2.getBottomY() > 0 ? 1 : -1;
}
if (!Objects.equals(b1.getRightX(), b2.getRightX())) {
return b1.getRightX() - b2.getRightX() > 0 ? 1 : -1;
}
return 0;
});
return sortedContents;
}
/**
* Sorts document contents according to the configured reading order.
*
* @param contents the document contents organized by page
* @param config the configuration containing reading order settings
*/
public static void sortContents(List<List<IObject>> contents, Config config) {
String readingOrder = config.getReadingOrder();
// xycut: XY-Cut++ sorting (per-page, stateless — safe to parallelize)
if (Config.READING_ORDER_XYCUT.equals(readingOrder)) {
int totalPages = StaticContainers.getDocument().getNumberOfPages();
IntStream pages = IntStream.range(0, totalPages);
if (config.getThreads() > 1) {
pages.parallel().forEach(pageNumber ->
contents.set(pageNumber, XYCutPlusPlusSorter.sort(contents.get(pageNumber))));
} else {
pages.forEach(pageNumber ->
contents.set(pageNumber, XYCutPlusPlusSorter.sort(contents.get(pageNumber))));
}
return;
}
// Log warning for unknown reading order values
if (!Config.READING_ORDER_OFF.equals(readingOrder)) {
LOGGER.log(Level.WARNING, "Unknown reading order value ''{0}'', using default ''off''", readingOrder);
}
// off: skip sorting (keep PDF COS object order)
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ExtractionResult.java
+64
View File
@@ -0,0 +1,64 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import com.fasterxml.jackson.databind.JsonNode;
import org.opendataloader.pdf.hybrid.ElementMetadata;
import org.verapdf.wcag.algorithms.entities.IObject;
import java.util.Collections;
import java.util.List;
import java.util.Map;
/**
* Internal result of the extraction pipeline (preprocessing + content extraction + sanitization).
* Carries the extracted contents and timing metadata for use by AutoTagger.
*/
public class ExtractionResult {
private final List<List<IObject>> contents;
private final long extractionNs;
private final JsonNode hybridTimings;
private final Map<Long, ElementMetadata> elementMetadata;
public ExtractionResult(List<List<IObject>> contents, long extractionNs, JsonNode hybridTimings,
Map<Long, ElementMetadata> elementMetadata) {
this.contents = contents;
this.extractionNs = extractionNs;
this.hybridTimings = hybridTimings;
this.elementMetadata = elementMetadata != null ? elementMetadata : Collections.emptyMap();
}
public ExtractionResult(List<List<IObject>> contents, long extractionNs, JsonNode hybridTimings) {
this(contents, extractionNs, hybridTimings, Collections.emptyMap());
}
public List<List<IObject>> getContents() {
return contents;
}
public long getExtractionNs() {
return extractionNs;
}
public JsonNode getHybridTimings() {
return hybridTimings;
}
public Map<Long, ElementMetadata> getElementMetadata() {
return elementMetadata;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/HeaderFooterProcessor.java
+365
View File
@@ -0,0 +1,365 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.verapdf.wcag.algorithms.entities.INode;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeaderOrFooter;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.lists.ListInterval;
import org.verapdf.wcag.algorithms.entities.lists.ListIntervalsCollection;
import org.verapdf.wcag.algorithms.entities.lists.info.ListItemInfo;
import org.verapdf.wcag.algorithms.entities.lists.info.ListItemTextInfo;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ListLabelsUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.listLabelsDetection.*;
import java.util.*;
import java.util.stream.Collectors;
/**
* Processor for detecting and extracting headers and footers from PDF documents.
* Identifies repeating content at the top and bottom of pages.
*/
public class HeaderFooterProcessor {
/**
* Processes document contents to detect headers and footers.
*
* @param contents the document contents organized by page
* @param isTagged whether the document is tagged
*/
public static void processHeadersAndFooters(List<List<IObject>> contents, boolean isTagged) {
DocumentProcessor.setIndexesForDocumentContents(contents);
List<List<IObject>> sortedContents = new ArrayList<>();
for (List<IObject> content : contents) {
sortedContents.add(DocumentProcessor.sortPageContents(content));
}
List<List<IObject>> filteredSortedContents = new ArrayList<>();
for (List<IObject> content : sortedContents) {
filteredSortedContents.add(content.stream().filter(c -> !(c instanceof LineChunk) && !(c instanceof LineArtChunk)).collect(Collectors.toList()));
}
List<SemanticHeaderOrFooter> footers = getHeadersOrFooters(filteredSortedContents, false);
List<SemanticHeaderOrFooter> headers = getHeadersOrFooters(filteredSortedContents, true);
for (int pageNumber = 0; pageNumber < contents.size(); pageNumber++) {
contents.set(pageNumber, updatePageContents(contents.get(pageNumber), headers.get(pageNumber), footers.get(pageNumber)));
}
if (!isTagged) {
processHeadersOrFootersContents(footers);
processHeadersOrFootersContents(headers);
}
}
private static void processHeadersOrFootersContents(List<SemanticHeaderOrFooter> headersOrFooters) {
for (SemanticHeaderOrFooter headerOrFooter : headersOrFooters) {
if (headerOrFooter != null) {
headerOrFooter.setContents(processHeaderOrFooterContent(headerOrFooter.getContents()));
}
}
}
private static List<IObject> updatePageContents(List<IObject> pageContents, SemanticHeaderOrFooter header, SemanticHeaderOrFooter footer) {
SortedSet<Integer> headerAndFooterIndexes = new TreeSet<>();
headerAndFooterIndexes.addAll(getHeaderOrFooterContentsIndexes(header));
headerAndFooterIndexes.addAll(getHeaderOrFooterContentsIndexes(footer));
if (headerAndFooterIndexes.isEmpty()) {
return pageContents;
}
List<IObject> result = new ArrayList<>();
if (header != null) {
result.add(header);
}
Iterator<Integer> iterator = headerAndFooterIndexes.iterator();
int nextHeaderOrFooterIndex = iterator.hasNext() ? iterator.next() : pageContents.size();
for (int index = 0; index < pageContents.size(); index++) {
if (index < nextHeaderOrFooterIndex) {
result.add(pageContents.get(index));
} else {
nextHeaderOrFooterIndex = iterator.hasNext() ? iterator.next() : pageContents.size();
}
}
if (footer != null) {
result.add(footer);
}
return result;
}
private static Set<Integer> getHeaderOrFooterContentsIndexes(SemanticHeaderOrFooter header) {
if (header == null) {
return Collections.emptySet();
}
SortedSet<Integer> set = new TreeSet<>();
for (IObject content : header.getContents()) {
set.add(content.getIndex());
}
return set;
}
private static List<SemanticHeaderOrFooter> getHeadersOrFooters(List<List<IObject>> sortedContents, boolean isHeaderDetection) {
List<SemanticHeaderOrFooter> headersOrFooters = new ArrayList<>(sortedContents.size());
List<Integer> numberOfHeaderOrFooterContentsForEachPage = getNumberOfHeaderOrFooterContentsForEachPage(sortedContents, isHeaderDetection);
for (int pageNumber = 0; pageNumber < sortedContents.size(); pageNumber++) {
Integer currentIndex = numberOfHeaderOrFooterContentsForEachPage.get(pageNumber);
if (currentIndex == 0) {
headersOrFooters.add(null);
continue;
}
List<IObject> pageContents = sortedContents.get(pageNumber);
List<IObject> headerContents = filterHeaderOrFooterContents(isHeaderDetection ? pageContents.subList(0, currentIndex) :
pageContents.subList(pageContents.size() - currentIndex, pageContents.size()), pageNumber, isHeaderDetection);
if (headerContents.isEmpty()) {
headersOrFooters.add(null);
continue;
}
SemanticHeaderOrFooter semanticHeaderOrFooter = new SemanticHeaderOrFooter(isHeaderDetection ? SemanticType.HEADER : SemanticType.FOOTER);
semanticHeaderOrFooter.addContents(headerContents);
semanticHeaderOrFooter.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
headersOrFooters.add(semanticHeaderOrFooter);
}
return headersOrFooters;
}
private static List<IObject> processHeaderOrFooterContent(List<IObject> contents) {
List<IObject> newContents = ParagraphProcessor.processParagraphs(contents);
newContents = ListProcessor.processListsFromTextNodes(newContents);
HeadingProcessor.processHeadings(newContents, false);
DocumentProcessor.setIDs(newContents);
CaptionProcessor.processCaptions(newContents);
return newContents;
}
/**
* Maximum vertical gap (in points) allowed between consecutive footer/header elements.
* If a candidate element is farther than this from the previously accepted element,
* it is not included in the header/footer region. This prevents body text that happens
* to repeat across pages from being absorbed into the footer.
*/
private static final double MAX_HEADER_FOOTER_GAP = 30.0;
private static List<Integer> getNumberOfHeaderOrFooterContentsForEachPage(List<List<IObject>> sortedContents, boolean isHeaderDetection) {
List<Integer> numberOfHeaderOrFooterContentsForEachPage = new ArrayList<>(sortedContents.size());
for (int pageNumber = 0; pageNumber < sortedContents.size(); pageNumber++) {
numberOfHeaderOrFooterContentsForEachPage.add(0);
}
int currentIndex = 0;
while (true) {
List<IObject> contents = new ArrayList<>(sortedContents.size());
for (int pageNumber = 0; pageNumber < sortedContents.size(); pageNumber++) {
if (numberOfHeaderOrFooterContentsForEachPage.get(pageNumber) != currentIndex) {
contents.add(null);
continue;
}
List<IObject> pageContents = sortedContents.get(pageNumber);
int index = isHeaderDetection ? currentIndex : pageContents.size() - 1 - currentIndex;
if (index >= 0 && index < pageContents.size()) {
IObject candidate = pageContents.get(index);
if (currentIndex > 0 && !isAdjacentToExistingHeaderOrFooter(
pageContents, currentIndex, isHeaderDetection, candidate)) {
contents.add(null);
} else {
contents.add(candidate);
}
} else {
contents.add(null);
}
}
Set<Integer> newIndexes = getIndexesOfHeaderOrFootersContents(contents);
if (newIndexes.isEmpty()) {
break;
}
for (Integer newIndex : newIndexes) {
numberOfHeaderOrFooterContentsForEachPage.set(newIndex, currentIndex + 1);
}
currentIndex++;
}
return numberOfHeaderOrFooterContentsForEachPage;
}
private static boolean isAdjacentToExistingHeaderOrFooter(
List<IObject> pageContents, int currentIndex, boolean isHeaderDetection, IObject candidate) {
int previousIndex = isHeaderDetection ? currentIndex - 1 : pageContents.size() - currentIndex;
if (previousIndex < 0 || previousIndex >= pageContents.size()) {
return true;
}
IObject previousElement = pageContents.get(previousIndex);
double gap;
if (isHeaderDetection) {
gap = previousElement.getBottomY() - candidate.getTopY();
} else {
gap = candidate.getBottomY() - previousElement.getTopY();
}
return gap <= MAX_HEADER_FOOTER_GAP;
}
private static Set<Integer> getIndexesOfHeaderOrFootersContents(List<IObject> contents) {
Set<Integer> result = new HashSet<>(contents.size());
for (int pageNumber = 0; pageNumber < contents.size() - 1; pageNumber++) {
IObject currentObject = contents.get(pageNumber);
IObject nextObject = contents.get(pageNumber + 1);
if (currentObject != null && nextObject != null) {
if (arePossibleHeadersOrFooters(currentObject, nextObject, 1)) {
result.add(pageNumber);
result.add(pageNumber + 1);
}
}
}
//2-page style
for (int pageNumber = 0; pageNumber < contents.size() - 2; pageNumber++) {
IObject currentObject = contents.get(pageNumber);
IObject nextObject = contents.get(pageNumber + 2);
if (currentObject != null && nextObject != null) {
if (arePossibleHeadersOrFooters(currentObject, nextObject, 2)) {
result.add(pageNumber);
result.add(pageNumber + 2);
}
}
}
return result;
}
/**
* Checks if a content object is a header or footer.
*
* @param content the content object to check
* @return true if the content is a header or footer, false otherwise
*/
public static boolean isHeaderOrFooter(IObject content) {
if (content instanceof INode) {
INode node = (INode) content;
if (node.getSemanticType() == SemanticType.HEADER || node.getSemanticType() == SemanticType.FOOTER) {
return true;
}
}
return false;
}
private static List<IObject> filterHeaderOrFooterContents(List<IObject> contents, int pageNumber, boolean isHeaderDetection) {
BoundingBox boundingBox = DocumentProcessor.getPageBoundingBox(pageNumber);
if (boundingBox == null) {
return contents;
}
List<IObject> result = new ArrayList<>();
for (IObject content : contents) {
if (isHeaderDetection) {
if (content.getBottomY() < boundingBox.getHeight() * 2 / 3) {
continue;
}
} else {
if (content.getTopY() > boundingBox.getHeight() / 3) {
continue;
}
}
result.add(content);
}
return result;
}
private static boolean arePossibleHeadersOrFooters(IObject object1, IObject object2, int increment) {
if (object1 instanceof SemanticTextNode && object2 instanceof SemanticTextNode) {
SemanticTextNode textNode1 = (SemanticTextNode) object1;
SemanticTextNode textNode2 = (SemanticTextNode) object2;
if (!BoundingBox.areOverlapsBoundingBoxesExcludingPages(object1.getBoundingBox(), object2.getBoundingBox())) {
return false;
}
if (!NodeUtils.areCloseNumbers(textNode1.getFontSize(), textNode2.getFontSize())) {
return false;
}
if (Objects.equals(textNode1.getValue(), textNode2.getValue())) {
return true;
}
List<SemanticTextNode> textNodes = new ArrayList<>(2);
textNodes.add(textNode1);
textNodes.add(textNode2);
if (getHeadersOrFootersIntervals(textNodes, increment).size() == 1) {
return true;
}
} else if (object1 instanceof TextLine && object2 instanceof TextLine) {
TextLine line1 = (TextLine) object1;
TextLine line2 = (TextLine) object2;
SemanticTextNode textNode1 = new SemanticTextNode();
textNode1.add(line1);
SemanticTextNode textNode2 = new SemanticTextNode();
textNode2.add(line2);
return arePossibleHeadersOrFooters(textNode1, textNode2, increment);
} else {
if (BoundingBox.areSameBoundingBoxesExcludingPages(object1.getBoundingBox(), object2.getBoundingBox())) {
return true;
}
}
return false;
}
private static Set<ListInterval> getHeadersOrFootersIntervals(List<SemanticTextNode> textNodes, int increment) {
List<ListItemTextInfo> textChildrenInfo = new ArrayList<>(textNodes.size());
for (int i = 0; i < textNodes.size(); i++) {
SemanticTextNode textNode = textNodes.get(i);
TextLine line = textNode.getFirstNonSpaceLine();
TextLine secondLine = textNode.getNonSpaceLine(1);
textChildrenInfo.add(new ListItemTextInfo(i, textNode.getSemanticType(),
line, line.getValue().trim(), secondLine == null));
}
Set<ListInterval> intervals = getHeadersOfFooterIntervals(textChildrenInfo, increment);
return intervals;
}
private static Set<ListInterval> getHeadersOfFooterIntervals(List<ListItemTextInfo> itemsInfo, int increment) {
ListIntervalsCollection listIntervals = new ListIntervalsCollection();
listIntervals.putAll((new AlfaLettersListLabelsDetectionAlgorithm1(increment)).getItemsIntervals(itemsInfo));
listIntervals.putAll((new AlfaLettersListLabelsDetectionAlgorithm2(increment)).getItemsIntervals(itemsInfo));
listIntervals.putAll((new KoreanLettersListLabelsDetectionAlgorithm(increment)).getItemsIntervals(itemsInfo));
listIntervals.putAll((new RomanNumbersListLabelsDetectionAlgorithm(increment)).getItemsIntervals(itemsInfo));
ArabicNumbersListLabelsDetectionAlgorithm arabicNumbersListLabelsDetectionAlgorithm = new ArabicNumbersListLabelsDetectionAlgorithm(increment);
arabicNumbersListLabelsDetectionAlgorithm.setHeaderOrFooterDetection(true);
listIntervals.putAll((arabicNumbersListLabelsDetectionAlgorithm).getItemsIntervals(itemsInfo));
ListIntervalsCollection correctIntervals = new ListIntervalsCollection(getEqualsItems(itemsInfo));
for (ListInterval listInterval : listIntervals.getSet()) {
List<String> labels = new LinkedList<>();
for (ListItemInfo info : listInterval.getListItemsInfos()) {
labels.add(((ListItemTextInfo) info).getListItem());
}
if (ListLabelsUtils.isListLabels(labels, increment)) {
correctIntervals.put(listInterval);
}
}
return correctIntervals.getSet();
}
private static Set<ListInterval> getEqualsItems(List<ListItemTextInfo> itemsInfo) {
Set<ListInterval> listIntervals = new HashSet<>();
String value = null;
ListInterval interval = new ListInterval();
for (ListItemTextInfo info : itemsInfo) {
if (!Objects.equals(info.getListItem(), value)) {
if (interval.getNumberOfListItems() > 1) {
listIntervals.add(interval);
}
value = info.getListItem();
interval = new ListInterval();
}
interval.getListItemsInfos().add(info);
}
if (interval.getNumberOfListItems() > 1) {
listIntervals.add(interval);
}
return listIntervals;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/HeadingProcessor.java
+244
View File
@@ -0,0 +1,244 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.utils.BulletedParagraphUtils;
import org.opendataloader.pdf.utils.TextNodeStatistics;
import org.opendataloader.pdf.utils.TextNodeUtils;
import org.verapdf.wcag.algorithms.entities.INode;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.TextBlock;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.text.TextStyle;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.util.*;
/**
* Processor for detecting and classifying headings in PDF content.
* Uses font size, weight, and position to identify potential headings.
*/
public class HeadingProcessor {
private static final double HEADING_PROBABILITY = 0.75;
private static final double BULLETED_HEADING_PROBABILITY = 0.1;
/**
* Processes content to identify and mark headings.
*
* @param contents the list of content objects to process
* @param isTableCell whether the content is inside a table cell
*/
public static void processHeadings(List<IObject> contents, boolean isTableCell) {
TextNodeStatistics textNodeStatistics = new TextNodeStatistics();
List<SemanticTextNode> textNodes = new LinkedList<>();
Map<SemanticTextNode, PDFList> textNodeToListMap = new HashMap<>();
for (IObject content : contents) {
processContent(textNodes, content, textNodeStatistics, textNodeToListMap);
}
int textNodesCount = textNodes.size();
if (isTableCell && textNodesCount < 2) {
return;
}
for (int index = 0; index < textNodesCount; index++) {
SemanticTextNode textNode = textNodes.get(index);
if (textNode.getSemanticType() == SemanticType.HEADING) {
continue;
}
SemanticTextNode prevNode = index != 0 ? textNodes.get(index - 1) : null;
SemanticTextNode nextNode = index + 1 < textNodesCount ? textNodes.get(index + 1) : null;
double probability = NodeUtils.headingProbability(textNode, prevNode, nextNode, textNode);
probability += textNodeStatistics.fontSizeRarityBoost(textNode);
probability += textNodeStatistics.fontWeightRarityBoost(textNode);
if (BulletedParagraphUtils.isBulletedParagraph(textNode)) {
probability += BULLETED_HEADING_PROBABILITY;
}
if (probability > HEADING_PROBABILITY && textNode.getSemanticType() != SemanticType.LIST) {
textNode.setSemanticType(SemanticType.HEADING);
}
if (textNode.getSemanticType() == SemanticType.HEADING && textNode.getInitialSemanticType() == SemanticType.LIST) {
PDFList list = textNodeToListMap.get(textNode);
if (isNotHeadings(list)) {
continue;
}
int listIndex = contents.indexOf(list);
contents.remove(listIndex);
contents.addAll(listIndex, disassemblePDFList(list));
}
}
setHeadings(contents);
}
private static List<IObject> disassemblePDFList(PDFList list) {
List<IObject> contents = new LinkedList<>();
for (ListItem item : list.getListItems()) {
SemanticTextNode node = convertListItemToSemanticTextNode(item);
node.setSemanticType(SemanticType.HEADING);
contents.add(node);
contents.addAll(item.getContents());
}
return contents;
}
private static SemanticTextNode convertListItemToSemanticTextNode(TextBlock textBlock) {
SemanticTextNode semanticTextNode = new SemanticTextNode(SemanticType.LIST);
for (TextLine line : textBlock.getLines()) {
semanticTextNode.add(line);
}
return semanticTextNode;
}
private static List<SemanticTextNode> getTextNodesFromContents(List<IObject> contents) {
List<SemanticTextNode> textNodes = new LinkedList<>();
for (IObject content : contents) {
if (content instanceof SemanticTextNode) {
textNodes.add((SemanticTextNode) content);
}
}
return textNodes;
}
private static void processContent(List<SemanticTextNode> textNodes, IObject content, TextNodeStatistics textNodeStatistics,
Map<SemanticTextNode, PDFList> possibleHeadingsInList) {
if (content instanceof SemanticTextNode) {
SemanticTextNode textNode = (SemanticTextNode) content;
if (!textNode.isSpaceNode()) {
textNodes.add(textNode);
textNodeStatistics.addTextNode(textNode);
}
} else if (content instanceof TableBorder && ((TableBorder) content).isTextBlock()) {
TableBorder textBlock = (TableBorder) content;
TableBorderCell cell = textBlock.getCell(0, 0);
List<SemanticTextNode> cellTextNodes = getTextNodesFromContents(cell.getContents());
if (cellTextNodes.size() == 1) {
processContent(textNodes, cellTextNodes.get(0), textNodeStatistics, possibleHeadingsInList);
}
} else if (content instanceof PDFList) {
PDFList list = (PDFList) content;
ListItem listItem = list.getFirstListItem();
SemanticTextNode textNode = convertListItemToSemanticTextNode(listItem);
textNodes.add(textNode);
textNodeStatistics.addTextNode(textNode);
possibleHeadingsInList.put(textNode, list);
}
}
private static boolean isNotHeadings(PDFList list) {
for (int i = 0; i < list.getListItems().size() - 1; i++) {
boolean onlyLineArtChunks = true;
List<ListItem> listItems = list.getListItems();
if (listItems.get(i).getContents().isEmpty()) {
return true;
}
for (IObject item : listItems.get(i).getContents()) {
if (!(item instanceof LineArtChunk)) {
onlyLineArtChunks = false;
break;
}
}
if (onlyLineArtChunks) {
return true;
}
}
return false;
}
private static void setHeadings(List<IObject> contents) {
for (int index = 0; index < contents.size(); index++) {
IObject content = contents.get(index);
if (content instanceof SemanticTextNode && ((INode) content).getSemanticType() == SemanticType.HEADING && !(content instanceof SemanticHeading)) {
SemanticHeading heading = new SemanticHeading((SemanticTextNode) content);
contents.set(index, heading);
StaticLayoutContainers.getHeadings().add(heading);
}
if (content instanceof TableBorder) {
TableBorder table = (TableBorder) content;
if (table.isTextBlock()) {
List<IObject> textBlockContents = table.getCell(0, 0).getContents();
setHeadings(textBlockContents);
}
}
}
}
/**
* Detects and assigns heading levels based on text style.
* Groups headings by text style and assigns levels from 1 upwards.
*/
public static void detectHeadingsLevels() {
SortedMap<TextStyle, Set<SemanticHeading>> map = new TreeMap<>();
List<SemanticHeading> headings = StaticLayoutContainers.getHeadings();
List<SemanticHeading> colorlessHeadings = new ArrayList<>();
for (SemanticHeading heading : headings) {
if (TextNodeUtils.getTextColorOrNull(heading) == null) {
colorlessHeadings.add(heading);
continue;
}
TextStyle textStyle = TextStyle.getTextStyle(heading);
map.computeIfAbsent(textStyle, k -> new HashSet<>()).add(heading);
}
int level = 1;
TextStyle previousTextStyle = null;
for (Map.Entry<TextStyle, Set<SemanticHeading>> entry : map.entrySet()) {
if (previousTextStyle != null && previousTextStyle.compareTo(entry.getKey()) != 0) {
level++;
}
previousTextStyle = entry.getKey();
for (SemanticHeading heading : entry.getValue()) {
heading.setHeadingLevel(level);
}
}
// Headings without color info get level based on font size relative to existing levels
for (SemanticHeading heading : colorlessHeadings) {
heading.setHeadingLevel(findClosestLevel(heading, map));
}
}
private static int findClosestLevel(SemanticHeading heading, SortedMap<TextStyle, Set<SemanticHeading>> map) {
if (map.isEmpty()) {
return 1;
}
double fontSize = heading.getFontSize();
int bestLevel = 1;
double bestDiff = Double.MAX_VALUE;
int level = 1;
TextStyle previousStyle = null;
for (Map.Entry<TextStyle, Set<SemanticHeading>> entry : map.entrySet()) {
if (previousStyle != null && previousStyle.compareTo(entry.getKey()) != 0) {
level++;
}
previousStyle = entry.getKey();
SemanticHeading representative = entry.getValue().iterator().next();
double diff = Math.abs(representative.getFontSize() - fontSize);
if (diff < bestDiff) {
bestDiff = diff;
bestLevel = level;
}
}
return bestLevel;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/HiddenTextProcessor.java
+71
View File
@@ -0,0 +1,71 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.semanticalgorithms.consumers.ContrastRatioConsumer;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.util.LinkedList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Processor for detecting hidden text in PDF documents.
* Identifies text with low contrast ratio against the background.
*/
public class HiddenTextProcessor {
private static final Logger LOGGER = Logger.getLogger(HiddenTextProcessor.class.getCanonicalName());
private static final double MIN_CONTRAST_RATIO = 1.2d;
/**
* Finds and marks or filters hidden text based on contrast ratio.
*
* @param contents the page contents to process
* @param isFilterHiddenText whether to filter out hidden text or just mark it
* @return the processed list of content objects
*/
public static List<IObject> findHiddenText(List<IObject> contents, boolean isFilterHiddenText) {
if (StaticContainers.getImagesUtils() == null) {
LOGGER.log(Level.WARNING, "Hidden-text filtering will be skipped for this document.");
return contents;
}
List<IObject> result = new LinkedList<>();
try {
ContrastRatioConsumer contrastRatioConsumer = new ContrastRatioConsumer();
for (IObject content : contents) {
if (content instanceof TextChunk) {
TextChunk textChunk = (TextChunk) content;
contrastRatioConsumer.calculateContrastRatio(textChunk);
if (textChunk.getContrastRatio() < MIN_CONTRAST_RATIO) {
if (!isFilterHiddenText) {
textChunk.setHiddenText(true);
} else {
continue;
}
}
}
result.add(content);
}
} finally {
StaticContainers.getImagesUtils().clearRenderedPages();
}
return result;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/HybridDocumentProcessor.java
+1529
View File
File diff suppressed because it is too large Load Diff
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/LevelProcessor.java
+148
View File
@@ -0,0 +1,148 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.utils.BulletedParagraphUtils;
import org.opendataloader.pdf.utils.levels.*;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticHeading;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.util.Collections;
import java.util.List;
import java.util.Stack;
import java.util.logging.Level;
import java.util.logging.Logger;
public class LevelProcessor {
private static final Logger LOGGER = Logger.getLogger(LevelProcessor.class.getCanonicalName());
private static boolean isDocTitleSet = false;
public static void detectLevels(List<List<IObject>> contents) {
setLevels(contents, new Stack<>());
}
private static void setLevels(List<List<IObject>> contents, Stack<LevelInfo> levelInfos) {
int levelInfosSize = levelInfos.size();
for (List<IObject> pageContents : contents) {
for (IObject content : pageContents) {
if (content instanceof SemanticHeading) {
setLevelForHeading((SemanticHeading) content);
continue;
}
LevelInfo levelInfo = null;
Integer index = null;
if (content instanceof PDFList) {
PDFList previousList = ((PDFList) content).getPreviousList();
if (previousList != null) {
if (previousList.getLevel() == null) {
LOGGER.log(Level.WARNING, "List without detected level");
} else {
index = Integer.parseInt(previousList.getLevel()) - 1;
}
}
if (index == null) {
levelInfo = new ListLevelInfo((PDFList) content);
}
} else if (content instanceof TableBorder) {
TableBorder previousTable = ((TableBorder) content).getPreviousTable();
if (previousTable != null) {
if (previousTable.getLevel() == null) {
LOGGER.log(Level.WARNING, "Table without detected level");
} else {
index = Integer.parseInt(previousTable.getLevel()) - 1;
}
}
if (index == null) {
TableBorder table = (TableBorder) content;
setLevelForTable(table);
if (!table.isTextBlock()) {
levelInfo = new TableLevelInfo(table);
}
}
} else if (content instanceof SemanticTextNode) {
if (BulletedParagraphUtils.isBulletedParagraph((SemanticTextNode) content)) {
if (BulletedParagraphUtils.isBulletedLineArtParagraph((SemanticTextNode) content)) {
levelInfo = new LineArtBulletParagraphLevelInfo((SemanticTextNode) content);
} else {
levelInfo = new TextBulletParagraphLevelInfo((SemanticTextNode) content);
}
}
}
if (levelInfo == null && index == null) {
continue;
}
if (index == null) {
index = getLevelInfoIndex(levelInfos, levelInfo);
}
if (index == null) {
content.setLevel(String.valueOf(levelInfos.size() + 1));
levelInfos.add(levelInfo);
} else {
content.setLevel(String.valueOf(index + 1));
for (int i = Math.max(index + 1, levelInfosSize); i < levelInfos.size(); i++) {
levelInfos.pop();
}
}
if (content instanceof PDFList) {
for (ListItem listItem : ((PDFList) content).getListItems()) {
setLevels(Collections.singletonList(listItem.getContents()), levelInfos);
}
}
}
}
isDocTitleSet = false;
}
private static void setLevelForHeading(SemanticHeading heading) {
if (heading.getHeadingLevel() == 1 && !isDocTitleSet) {
heading.setLevel("Doctitle");
isDocTitleSet = true;
} else {
heading.setLevel("Subtitle");
}
}
private static Integer getLevelInfoIndex(Stack<LevelInfo> levelInfos, LevelInfo levelInfo) {
for (int index = 0; index < levelInfos.size(); index++) {
LevelInfo currentLevelInfo = levelInfos.get(index);
if (LevelInfo.areSameLevelsInfos(currentLevelInfo, levelInfo)) {
return index;
}
}
return null;
}
private static void setLevelForTable(TableBorder tableBorder) {
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
TableBorderRow row = tableBorder.getRow(rowNumber);
for (int colNumber = 0; colNumber < tableBorder.getNumberOfColumns(); colNumber++) {
TableBorderCell tableBorderCell = row.getCell(colNumber);
if (tableBorderCell.getRowNumber() == rowNumber && tableBorderCell.getColNumber() == colNumber) {
setLevels(Collections.singletonList(tableBorderCell.getContents()), new Stack<>());
}
}
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ListProcessor.java
+573
View File
@@ -0,0 +1,573 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.utils.BulletedParagraphUtils;
import org.verapdf.as.ASAtom;
import org.verapdf.wcag.algorithms.entities.INode;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.*;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.enums.TextAlignment;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.lists.ListInterval;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.lists.TextListInterval;
import org.verapdf.wcag.algorithms.entities.lists.info.ListItemInfo;
import org.verapdf.wcag.algorithms.entities.lists.info.ListItemTextInfo;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ChunksMergeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ListLabelsUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ListUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.listLabelsDetection.NumberingStyleNames;
import java.util.*;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class ListProcessor {
private static final Logger LOGGER = Logger.getLogger(ListProcessor.class.getCanonicalName());
private static final double LIST_ITEM_PROBABILITY = 0.7;
private static final double LIST_ITEM_BASELINE_DIFFERENCE = 1.2;
private static final double LIST_ITEM_X_INTERVAL_RATIO = 0.3;
private static final Pattern ATTACHMENTS_PATTERN = Pattern.compile("^붙\\s*임\\s*(?=.)");
/**
* Maximum number of intervals to scan backward when matching a TextLine to an existing list.
* Prevents O(n²) scaling on large documents. A higher value is safer but slower.
*/
private static final int MAX_LIST_INTERVAL_LOOKBACK = 500;
private static final Map<String, ASAtom> listNumberingMap = new HashMap<>();
static {
listNumberingMap.put(NumberingStyleNames.ENGLISH_LETTERS, ASAtom.ORDERED);
listNumberingMap.put(NumberingStyleNames.ENGLISH_LETTERS_UPPER_CASE, ASAtom.UPPER_ALPHA);
listNumberingMap.put(NumberingStyleNames.ENGLISH_LETTERS_LOWER_CASE, ASAtom.LOWER_ALPHA);
listNumberingMap.put(NumberingStyleNames.ROMAN_NUMBERS_LOWER_CASE, ASAtom.LOWER_ROMAN);
listNumberingMap.put(NumberingStyleNames.ROMAN_NUMBERS, ASAtom.ORDERED);
listNumberingMap.put(NumberingStyleNames.ROMAN_NUMBERS_UPPER_CASE, ASAtom.UPPER_ROMAN);
listNumberingMap.put(NumberingStyleNames.KOREAN_LETTERS, ASAtom.ORDERED);
listNumberingMap.put(NumberingStyleNames.ARABIC_NUMBERS, ASAtom.DECIMAL);
listNumberingMap.put(NumberingStyleNames.CIRCLED_ARABIC_NUMBERS, ASAtom.ORDERED);
listNumberingMap.put(NumberingStyleNames.UNORDERED,ASAtom.UNORDERED);
listNumberingMap.put(NumberingStyleNames.UNKNOWN, ASAtom.NONE);
}
public static ASAtom getListNumbering(String numberingStyle) {
return listNumberingMap.get(numberingStyle);
}
public static void processLists(List<List<IObject>> contents, boolean isTableCell) {
List<TextListInterval> intervalsList = getTextLabelListIntervals(contents);
for (TextListInterval interval : intervalsList) {
for (ListItemTextInfo info : interval.getListItemsInfos()) {
info.getListItemValue().setListLine(true);
}
}
for (TextListInterval interval : intervalsList) {
// if (interval.getNumberOfColumns() > 1/*== interval.getNumberOfListItems()*/) {//to fix bounding box for multi-column lists
// continue;
// }
if (!isCorrectList(interval)) {//todo move to arabic number list recognition
continue;
}
Integer currentPageNumber = interval.getListItemsInfos().get(0).getPageNumber();
int index = 0;
PDFList previousList = null;
interval.setCommonSuffixLengthToAllInfos();
for (int i = 0; i < interval.getNumberOfListItems(); i++) {
ListItemInfo currentInfo = interval.getListItemsInfos().get(i);
if (!Objects.equals(currentInfo.getPageNumber(), currentPageNumber)) {
PDFList list = calculateList(interval, index, i - 1, contents.get(isTableCell ? 0 : currentPageNumber));
for (ListItem listItem : list.getListItems()) {
listItem.setContents(processListItemContent(listItem.getContents()));
}
if (previousList != null) {
PDFList.setListConnected(previousList, list);
}
currentPageNumber = currentInfo.getPageNumber();
index = i;
previousList = list;
}
}
PDFList list = calculateList(interval, index, interval.getNumberOfListItems() - 1, contents.get(isTableCell ? 0 : currentPageNumber));
for (ListItem listItem : list.getListItems()) {
listItem.setContents(processListItemContent(listItem.getContents()));
}
if (previousList != null) {
PDFList.setListConnected(previousList, list);
}
}
contents.replaceAll(DocumentProcessor::removeNullObjectsFromList);
}
private static List<IObject> processListItemContent(List<IObject> contents) {
List<IObject> newContents = ParagraphProcessor.processParagraphs(contents);
newContents = ListProcessor.processListsFromTextNodes(newContents);
DocumentProcessor.setIDs(newContents);
List<List<IObject>> contentsList = new ArrayList<>(1);
contentsList.add(newContents);
ListProcessor.checkNeighborLists(contentsList);
newContents = contentsList.get(0);
return newContents;
}
private static void processTextNodeListItemContent(List<IObject> contents) {
DocumentProcessor.setIDs(contents);
}
private static List<TextListInterval> getTextLabelListIntervals(List<List<IObject>> contents) {
List<TextListInterval> listIntervals = new ArrayList<>();
for (List<IObject> pageContents : contents) {
for (int i = 0; i < pageContents.size(); i++) {
IObject content = pageContents.get(i);
if (!(content instanceof TextLine)) {
continue;
}
TextLine line = (TextLine) content;
String value = line.getValue();
if (value.isEmpty() || line.isHiddenText()) {
continue;
}
ListItemTextInfo listItemTextInfo = createListItemTextInfo(i, line, value);
processListItem(listIntervals, listItemTextInfo);
}
}
LinkedHashSet<TextListInterval> intervalsList = new LinkedHashSet<>();
for (TextListInterval interval : listIntervals) {
if (interval != null && interval.getListItemsInfos().size() > 1) {
intervalsList.add(interval);
}
}
List<TextListInterval> result = new ArrayList<>(intervalsList);
Collections.reverse(result);
return result;
}
private static void processListItem(List<TextListInterval> listIntervals, ListItemTextInfo listItemTextInfo) {
double maxXGap = getMaxXGap(listItemTextInfo.getListItemValue().getFontSize());
boolean isSingle = true;
boolean shouldHaveSameLeft = false;
boolean shouldHaveSameLeftDifference = false;
boolean isUnordered = true;
Double previousLeftDifference = null;
int minIndex = Math.max(0, listIntervals.size() - MAX_LIST_INTERVAL_LOOKBACK);
for (int index = listIntervals.size() - 1; index >= minIndex; index--) {
TextListInterval interval = listIntervals.get(index);
ListItemTextInfo preivousListItemTextInfo = interval.getLastListItemInfo();
if (Objects.equals(listItemTextInfo.getPageNumber(), preivousListItemTextInfo.getPageNumber()) &&
listItemTextInfo.getListItemValue().getTopY() > preivousListItemTextInfo.getListItemValue().getTopY()) {
break;
}
double leftDifference = listItemTextInfo.getListItemValue().getLeftX() -
preivousListItemTextInfo.getListItemValue().getLeftX();
boolean haveSameLeft = NodeUtils.areCloseNumbers(leftDifference, 0, maxXGap);
try {
if (NodeUtils.areCloseNumbers(leftDifference, 0, 4 * maxXGap) &&
ListLabelsUtils.isTwoListItemsOfOneList(interval, listItemTextInfo,
!haveSameLeft, isUnordered)) {
listIntervals.add(interval);
isSingle = false;
break;
}
} catch (StringIndexOutOfBoundsException e) {
// Malformed label cannot be matched; treat as new list (isSingle remains true)
LOGGER.log(Level.WARNING, "Malformed list label, starting new list: " + listItemTextInfo.getListItemValue().getValue(), e);
break;
}
if (shouldHaveSameLeftDifference && !NodeUtils.areCloseNumbers(previousLeftDifference, leftDifference)) {
break;
}
if (leftDifference > maxXGap) {
isUnordered = false;
shouldHaveSameLeftDifference = true;
}
previousLeftDifference = leftDifference;
if (haveSameLeft) {
shouldHaveSameLeft = true;
} else if (shouldHaveSameLeft) {
isUnordered = false;
// break;
}
if (interval.getListItemsInfos().size() > 1 && haveSameLeft &&
!NumberingStyleNames.UNORDERED.equals(interval.getNumberingStyle())) {
isUnordered = false;
}
}
if (isSingle) {
TextListInterval listInterval = new TextListInterval();
listInterval.getListItemsInfos().add(listItemTextInfo);
listIntervals.add(listInterval);
}
}
private static ListItemTextInfo createListItemTextInfo(int i, TextLine line, String value) {
Matcher matcher = ATTACHMENTS_PATTERN.matcher(value);
if (matcher.find()) {
int length = matcher.group().length();
line = new TextLine(line);
line.getBoundingBox().setLeftX(line.getSymbolStartCoordinate(length));
value = value.substring(length);
}
return new ListItemTextInfo(i, SemanticType.PARAGRAPH,
line, value, true);
}
private static PDFList calculateList(TextListInterval interval, int startIndex, int endIndex, List<IObject> pageContents) {
PDFList list = new PDFList();
list.setNumberingStyle(interval.getNumberingStyle());
list.setCommonPrefix(interval.getCommonPrefix());
boolean isListSet = false;
for (int index = startIndex; index <= endIndex; index++) {
ListItemTextInfo currentInfo = interval.getListItemsInfos().get(index);
int nextIndex = index != endIndex ? interval.getListItemsInfos().get(index + 1).getIndex() : pageContents.size();
ListItem listItem = new ListItem(new BoundingBox(), null);
IObject object = pageContents.get(currentInfo.getIndex());
if (object == null || object instanceof PDFList) {
LOGGER.log(Level.INFO, "List item is connected with different lists");
continue;
}
pageContents.set(currentInfo.getIndex(), isListSet ? null : list);
isListSet = true;
if (object instanceof SemanticTextNode) {
SemanticTextNode textNode = (SemanticTextNode) object;
for (TextLine textLine : textNode.getFirstColumn().getLines()) {
listItem.add(textLine);
}
} else {
TextLine textLine = (TextLine) object;
listItem.add(textLine);
}
if (index != endIndex) {
addContentToListItem(nextIndex, currentInfo, pageContents, listItem);
} else {
addContentToLastPageListItem(nextIndex, currentInfo, pageContents, listItem);
}
listItem.setLabelLength(currentInfo.getLabelLength());
list.add(listItem);
}
if (list.getListItems().isEmpty()) {
LOGGER.log(Level.WARNING, "List is not added to contents");
}
return list;
}
private static void addContentToListItem(int nextIndex, ListItemInfo currentInfo, List<IObject> pageContents,
ListItem listItem) {
boolean isListItem = true;
TextLine previousTextLine = null;
for (int index = currentInfo.getIndex() + 1; index < nextIndex; index++) {
IObject content = pageContents.get(index);
if (content instanceof TextLine) {
TextLine currentTextLine = (TextLine) content;
if (previousTextLine != null) {
if (isListItem && isListItemLine(listItem, previousTextLine, currentTextLine)) {
listItem.add(previousTextLine);
} else {
isListItem = false;
listItem.getContents().add(previousTextLine);
}
}
previousTextLine = currentTextLine;
} else if (content != null) {
if (previousTextLine != null) {
if (isListItem && isListItemLine(listItem, previousTextLine, null)) {
listItem.add(previousTextLine);
} else {
isListItem = false;
listItem.getContents().add(previousTextLine);
}
previousTextLine = null;
}
listItem.getContents().add(content);
}
pageContents.set(index, null);
}
if (previousTextLine != null) {
if (isListItem && isListItemLine(listItem, previousTextLine, null)) {
listItem.add(previousTextLine);
} else {
listItem.getContents().add(previousTextLine);
}
}
}
private static void addContentToLastPageListItem(int nextIndex, ListItemInfo currentInfo, List<IObject> pageContents,
ListItem listItem) {
TextLine previousTextLine = null;
Integer previousIndex = null;
for (int index = currentInfo.getIndex() + 1; index < nextIndex; index++) {
IObject content = pageContents.get(index);
if (!(content instanceof TextLine)) {
continue;
}
TextLine nextLine = (TextLine) content;
if (previousTextLine != null) {
if (isListItemLine(listItem, previousTextLine, nextLine)) {
listItem.add(previousTextLine);
pageContents.set(previousIndex, null);
} else {
previousTextLine = null;
break;
}
}
previousTextLine = nextLine;
previousIndex = index;
}
if (previousTextLine != null) {
if (isListItemLine(listItem, previousTextLine, null)) {
listItem.add(previousTextLine);
pageContents.set(previousIndex, null);
}
}
}
private static boolean isListItemLine(ListItem listItem, TextLine currentLine, TextLine nextLine) {
TextLine listLine = listItem.getLastLine();
if (ChunksMergeUtils.mergeLeadingProbability(listLine, currentLine) < LIST_ITEM_PROBABILITY) {
return false;
}
if (nextLine != null) {
if (Math.abs(listLine.getBaseLine() - currentLine.getBaseLine()) >
LIST_ITEM_BASELINE_DIFFERENCE * Math.abs(currentLine.getBaseLine() - nextLine.getBaseLine())) {
return false;
}
}
if (listItem.getLinesNumber() > 1) {
TextAlignment alignment = ChunksMergeUtils.getAlignment(listLine, currentLine);
if (alignment != TextAlignment.JUSTIFY && alignment != TextAlignment.LEFT) {
return false;
}
} else {
double maxXGap = getMaxXGap(listLine.getFontSize());
if (currentLine.getLeftX() < listLine.getLeftX() - maxXGap) {
return false;
}
}
if (BulletedParagraphUtils.isLabeledLine(currentLine)) {
return false;
}
if (currentLine.isListLine()) {
return false;
}
return true;
}
private static double getMaxXGap(double fontSize) {
return fontSize * LIST_ITEM_X_INTERVAL_RATIO;
}
public static List<IObject> processListsFromTextNodes(List<IObject> contents) {
List<SemanticTextNode> textNodes = new ArrayList<>();
List<Integer> textNodesIndexes = new ArrayList<>();
for (int index = 0; index < contents.size(); index++) {
IObject content = contents.get(index);
if (content instanceof SemanticTextNode) {
textNodes.add((SemanticTextNode) content);
textNodesIndexes.add(index);
}
}
List<ListItemTextInfo> textChildrenInfo = calculateTextChildrenInfo(textNodes);
List<INode> nodes = new LinkedList<>(textNodes);
Set<ListInterval> intervals = ListUtils.getChildrenListIntervals(ListLabelsUtils.getListItemsIntervals(textChildrenInfo), nodes);
for (ListInterval interval : intervals) {
updateListInterval(interval, textNodesIndexes);
TextListInterval textListInterval = new TextListInterval(interval);
if (!isCorrectList(textListInterval)) {
continue;
}
textListInterval.setCommonSuffixLengthToAllInfos();
PDFList list = calculateList(textListInterval, 0, interval.getNumberOfListItems() - 1, contents);
for (ListItem listItem : list.getListItems()) {
processTextNodeListItemContent(listItem.getContents());
}
}
return DocumentProcessor.removeNullObjectsFromList(contents);
}
private static List<ListItemTextInfo> calculateTextChildrenInfo(List<SemanticTextNode> textNodes) {
List<ListItemTextInfo> textChildrenInfo = new ArrayList<>(textNodes.size());
for (int i = 0; i < textNodes.size(); i++) {
SemanticTextNode textNode = textNodes.get(i);
if (textNode.isSpaceNode() || textNode.isEmpty()) {
continue;
}
TextLine line = textNode.getFirstNonSpaceLine();
TextLine secondLine = textNode.getNonSpaceLine(1);
textChildrenInfo.add(new ListItemTextInfo(i, textNode.getSemanticType(),
line, line.getValue(), secondLine == null));
}
return textChildrenInfo;
}
private static void updateListInterval(ListInterval interval, List<Integer> textNodesIndexes) {
for (ListItemInfo itemInfo : interval.getListItemsInfos()) {
itemInfo.setIndex(textNodesIndexes.get(itemInfo.getIndex()));
}
}
private static boolean isCorrectList(TextListInterval interval) {//move inside arabic numeration detection
return !isDoubles(interval);
}
private static boolean isDoubles(TextListInterval interval) {
for (ListItemTextInfo listItemTextInfo : interval.getListItemsInfos()) {
if (listItemTextInfo != null) {
if (!listItemTextInfo.getListItemValue().getValue().matches("^\\d+\\.\\d+$")) {
return false;
}
} else {
return false;
}
}
return true;
}
public static void checkNeighborLists(List<List<IObject>> contents) {
PDFList previousList = null;
SemanticTextNode middleContent = null;
for (List<IObject> pageContents : contents) {
DocumentProcessor.setIndexesForContentsList(pageContents);
for (IObject content : pageContents) {
if (content instanceof PDFList) {
PDFList currentList = (PDFList) content;
if (previousList != null) {
if (previousList.getNextList() == null && currentList.getPreviousList() == null) {
if (isNeighborLists(previousList, currentList, middleContent)) {
if (middleContent != null) {
addMiddleContentToList(previousList, currentList, middleContent, pageContents);
}
if (Objects.equals(previousList.getPageNumber(), currentList.getPageNumber()) &&
BoundingBox.areHorizontalOverlapping(previousList.getBoundingBox(), currentList.getBoundingBox())) {
previousList.add(currentList);
pageContents.set(currentList.getIndex(), null);
currentList = null;
} else {
PDFList.setListConnected(previousList, currentList);
}
}
} else if (Objects.equals(previousList.getNextListId(), currentList.getRecognizedStructureId())) {
if (middleContent != null && isMiddleContentPartOfList(previousList, middleContent, currentList)) {
addMiddleContentToList(previousList, currentList, middleContent, pageContents);
}
}
}
if (currentList != null) {
previousList = currentList;
}
middleContent = null;
} else {
if (!HeaderFooterProcessor.isHeaderOrFooter(content) &&
!(content instanceof LineChunk) && !(content instanceof LineArtChunk) &&
!(content instanceof ImageChunk)) {
if (middleContent == null && content instanceof SemanticTextNode) {
middleContent = (SemanticTextNode) content;
} else {
middleContent = null;
previousList = null;
}
}
}
}
}
contents.replaceAll(DocumentProcessor::removeNullObjectsFromList);
}
private static void addMiddleContentToList(PDFList previousList, PDFList currentList, SemanticTextNode middleContent, List<IObject> pageContents) {
ListItem lastListItem = previousList.getLastListItem();
if (Objects.equals(lastListItem.getPageNumber(), middleContent.getPageNumber()) &&
BoundingBox.areHorizontalOverlapping(lastListItem.getBoundingBox(), middleContent.getBoundingBox())) {
for (TextColumn textColumn : middleContent.getColumns()) {
lastListItem.add(textColumn.getLines());
}
previousList.getBoundingBox().union(middleContent.getBoundingBox());
pageContents.set(middleContent.getIndex(), null);
} else {
if (middleContent.getTopY() > currentList.getTopY()) {
addFirstLBodyToList(currentList, middleContent);
pageContents.set(middleContent.getIndex(), null);
}
}
}
private static void addFirstLBodyToList(PDFList currentList, SemanticTextNode middleContent) {
ListItem listItem = new ListItem(new BoundingBox(), middleContent.getRecognizedStructureId());
for (TextColumn textColumn : middleContent.getColumns()) {
listItem.add(textColumn.getLines());
}
currentList.add(0, listItem);
}
public static boolean isNeighborLists(PDFList previousList, PDFList currentList, SemanticTextNode middleContent) {
List<ListItemTextInfo> textChildrenInfo = getTextChildrenInfosForNeighborLists(previousList, currentList);
Set<ListInterval> listIntervals = ListLabelsUtils.getListItemsIntervals(textChildrenInfo);
if (listIntervals.size() != 1) {
return false;
}
ListInterval interval = listIntervals.iterator().next();
if (interval.getNumberOfListItems() != textChildrenInfo.size()) {
return false;
}
if (middleContent != null && !isMiddleContentPartOfList(previousList, middleContent, currentList)) {
return false;
}
return true;
}
private static boolean isMiddleContentPartOfList(PDFList previousList, SemanticTextNode middleContent, PDFList currentList) {
if (middleContent.getLeftX() < currentList.getLeftX()) {
return false;
}
if (!Objects.equals(middleContent.getPageNumber(), currentList.getPageNumber())) {
return false;
}
for (ListItem listItem : currentList.getListItems()) {
if (listItem.getLinesNumber() > 1) {
double xInterval = getMaxXGap(Math.max(listItem.getFontSize(), middleContent.getFontSize()));
if (!NodeUtils.areCloseNumbers(listItem.getSecondLine().getLeftX(), middleContent.getLeftX(), xInterval)) {
return false;
}
break;
}
}
return true;
}
private static List<ListItemTextInfo> getTextChildrenInfosForNeighborLists(PDFList previousList, PDFList currentList) {
List<ListItemTextInfo> textChildrenInfo = new ArrayList<>(4);
if (previousList.getNumberOfListItems() > 1) {
textChildrenInfo.add(createListItemTextInfoFromListItem(0, previousList.getPenultListItem()));
}
textChildrenInfo.add(createListItemTextInfoFromListItem(1, previousList.getLastListItem()));
textChildrenInfo.add(createListItemTextInfoFromListItem(2, currentList.getFirstListItem()));
if (currentList.getNumberOfListItems() > 1) {
textChildrenInfo.add(createListItemTextInfoFromListItem(3, currentList.getSecondListItem()));
}
return textChildrenInfo;
}
private static ListItemTextInfo createListItemTextInfoFromListItem(int index, ListItem listItem) {
TextLine line = listItem.getFirstLine();
return new ListItemTextInfo(index, SemanticType.LIST_ITEM, line, line.getValue(), listItem.getLinesNumber() == 1);
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/PDFStreamWriter.java
+102
View File
@@ -0,0 +1,102 @@
package org.opendataloader.pdf.processors;
import org.apache.pdfbox.io.IOUtils;
import org.verapdf.as.ASAtom;
import org.verapdf.cos.*;
import org.verapdf.operator.InlineImageOperator;
import org.verapdf.operator.Operator;
import org.verapdf.parser.Operators;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.util.List;
import java.util.Map;
public class PDFStreamWriter {
public PDFStreamWriter() {
}
public byte[] write(List<Object> tokens) throws IOException {
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
PrintWriter printWriter = new PrintWriter(out)) {
for (Object rawToken : tokens) {
write(rawToken, printWriter, out);
}
return out.toByteArray();
}
}
private void write(Object rawToken, PrintWriter printWriter, OutputStream out) throws IOException {
if (rawToken instanceof COSArray) {
out.write("[".getBytes());
for (COSObject item : (COSArray) rawToken) {
write(item.getDirectBase(), printWriter, out);
}
out.write("]".getBytes());
out.write(" ".getBytes());
} else if (rawToken instanceof COSDictionary) {
out.write("<<".getBytes());
for (Map.Entry<ASAtom, COSObject> item : ((COSDictionary)rawToken).getEntrySet()) {
out.write(item.getKey().toString().getBytes());
out.write(" ".getBytes());
write(item.getValue(), printWriter, out);
}
out.write(">>".getBytes());
out.write(" ".getBytes());
} else if (rawToken instanceof COSString) {
COSString string = (COSString) rawToken;
if (string.isHexadecimal()) {
out.write("<".getBytes());
out.write(string.getHexString().getBytes());
out.write(">".getBytes());
out.write(" ".getBytes());
} else {
out.write("(".getBytes());
byte[] bytes = ((COSString) rawToken).get();
for (byte currentByte : bytes) {
if (currentByte == 40 || currentByte == 41 || currentByte == 92) {
out.write('\\');
}
if (currentByte == 13) {
out.write('\\');
out.write('r');
} else {
out.write(currentByte);
}
}
out.write(")".getBytes());
out.write(" ".getBytes());
}
} else if (rawToken instanceof COSBoolean) {
out.write(((COSBoolean) rawToken).getBoolean().toString().getBytes());
out.write(" ".getBytes());
} else if (rawToken instanceof COSBase) {
out.write(rawToken.toString().getBytes());
out.write(" ".getBytes());
} else if (rawToken instanceof COSObject) {
write(((COSObject) rawToken).getDirectBase(), printWriter, out);
} else if (rawToken instanceof Operator) {
Operator operator = (Operator) rawToken;
out.write(((Operator) rawToken).getOperator().getBytes());
out.write("\n".getBytes());
if (Operators.BI.equals(operator.getOperator())) {
InlineImageOperator inlineImageOperator = (InlineImageOperator) operator;
for (Map.Entry<ASAtom, COSObject> item : inlineImageOperator.getImageParameters().getEntrySet()) {
out.write(item.getKey().toString().getBytes());
out.write(" ".getBytes());
write(item.getValue(), printWriter, out);
}
out.write(Operators.ID.getBytes());
out.write("\n".getBytes());
IOUtils.copy(inlineImageOperator.getImageData(), out);
out.write("\n".getBytes());
out.write(Operators.EI.getBytes());
out.write("\n".getBytes());
}
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ParagraphProcessor.java
+542
View File
@@ -0,0 +1,542 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.utils.BulletedParagraphUtils;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticParagraph;
import org.verapdf.wcag.algorithms.entities.content.TextBlock;
import org.verapdf.wcag.algorithms.entities.content.TextColumn;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.TextAlignment;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.CaptionUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ChunksMergeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.util.*;
public class ParagraphProcessor {
public static final double DIFFERENT_LINES_PROBABILITY = 0.75;
public static List<IObject> processParagraphs(List<IObject> contents) {
DocumentProcessor.setIndexesForContentsList(contents);
List<TextBlock> blocks = new ArrayList<>();
for (IObject content : contents) {
if (content instanceof TextLine) {
blocks.add(new TextBlock((TextLine) content));
}
}
blocks = detectParagraphsWithJustifyAlignments(blocks);
blocks = detectFirstAndLastLinesOfParagraphsWithJustifyAlignments(blocks);
blocks = detectParagraphsWithLeftAlignments(blocks, true);
blocks = detectParagraphsWithLeftAlignments(blocks, false);
blocks = detectFirstLinesOfParagraphWithLeftAlignments(blocks);
blocks = detectParagraphsWithCenterAlignments(blocks);
blocks = detectParagraphsWithRightAlignments(blocks);
blocks = detectTwoLinesParagraphs(blocks);
blocks = processOtherLines(blocks);
return getContentsWithDetectedParagraphs(contents, blocks);
}
private static List<IObject> getContentsWithDetectedParagraphs(List<IObject> contents, List<TextBlock> blocks) {
List<IObject> newContents = new ArrayList<>();
Iterator<TextBlock> iterator = blocks.iterator();
TextBlock currentBlock = iterator.hasNext() ? iterator.next() : null;
Integer currentIndex = currentBlock != null ? currentBlock.getFirstLine().getIndex() : null;
for (int index = 0; index < contents.size(); index++) {
IObject content = contents.get(index);
if (!(content instanceof TextLine)) {
newContents.add(content);
} else if (Objects.equals(currentIndex, index)) {
newContents.add(createParagraphFromTextBlock(currentBlock));
currentBlock = iterator.hasNext() ? iterator.next() : null;
currentIndex = currentBlock != null ? currentBlock.getFirstLine().getIndex() : null;
}
}
return newContents;
}
private static List<TextBlock> detectParagraphsWithJustifyAlignments(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
TextAlignment textAlignment = ChunksMergeUtils.getAlignment(previousBlock.getLastLine(), nextBlock.getFirstLine());
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (textAlignment == TextAlignment.JUSTIFY && probability > DIFFERENT_LINES_PROBABILITY &&
areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.JUSTIFY);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static List<TextBlock> detectParagraphsWithCenterAlignments(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (areLinesOfParagraphsWithCenterAlignments(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.CENTER);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static boolean areLinesOfParagraphsWithCenterAlignments(TextBlock previousBlock, TextBlock nextBlock) {
TextAlignment textAlignment = ChunksMergeUtils.getAlignment(previousBlock.getLastLine(), nextBlock.getFirstLine());
if (textAlignment != TextAlignment.CENTER) {
return false;
}
double probability = getDifferentLinesProbability(previousBlock, nextBlock, true, false);
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
return true;
}
private static List<TextBlock> detectFirstAndLastLinesOfParagraphsWithJustifyAlignments(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
TextAlignment textAlignment = ChunksMergeUtils.getAlignment(previousBlock.getLastLine(), nextBlock.getFirstLine());
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (isFirstLineOfBlock(previousBlock, nextBlock, textAlignment, probability)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.JUSTIFY);
previousBlock.setHasStartLine(true);
previousBlock.setHasEndLine(nextBlock.isHasEndLine());
} else if (isLastLineOfBlock(previousBlock, nextBlock, textAlignment, probability)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setHasEndLine(true);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static List<TextBlock> detectParagraphsWithLeftAlignments(List<TextBlock> textBlocks, boolean checkStyle) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (areLinesOfParagraphsWithLeftAlignments(previousBlock, nextBlock, checkStyle)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.LEFT);
previousBlock.setHasEndLine(false);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static boolean areLinesOfParagraphsWithRightAlignments(TextBlock previousBlock, TextBlock nextBlock) {
TextAlignment textAlignment = ChunksMergeUtils.getAlignment(previousBlock.getLastLine(), nextBlock.getFirstLine());
if (textAlignment != TextAlignment.RIGHT) {
return false;
}
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
if (previousBlock.getLinesNumber() != 1 && previousBlock.getTextAlignment() != TextAlignment.RIGHT) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (nextBlock.getLinesNumber() != 1 && nextBlock.getTextAlignment() != TextAlignment.RIGHT) {
return false;
}
return true;
}
private static boolean areLinesOfParagraphsWithLeftAlignments(TextBlock previousBlock, TextBlock nextBlock, boolean checkStyle) {
TextAlignment textAlignment = ChunksMergeUtils.getAlignment(previousBlock.getLastLine(), nextBlock.getFirstLine());
if (textAlignment != TextAlignment.LEFT) {
return false;
}
boolean haveSameStyle = TextChunkUtils.areTextChunksHaveSameStyle(previousBlock.getLastLine().getFirstTextChunk(),
nextBlock.getFirstLine().getFirstTextChunk());
if (checkStyle && !haveSameStyle) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (BulletedParagraphUtils.isLabeledLine(nextBlock.getFirstLine())) {
return false;
}
boolean areShouldBeCloseLines = false;
if (previousBlock.getLinesNumber() != 1) {
if (previousBlock.getTextAlignment() == TextAlignment.JUSTIFY) {
if (!haveSameStyle) {
return false;
}
areShouldBeCloseLines = true;
} else if (previousBlock.getTextAlignment() != TextAlignment.LEFT) {
return false;
}
}
if (nextBlock.getLinesNumber() != 1) {
if (nextBlock.getTextAlignment() == TextAlignment.JUSTIFY) {
if (!haveSameStyle) {
return false;
}
areShouldBeCloseLines = true;
} else if (nextBlock.getTextAlignment() != TextAlignment.LEFT) {
return false;
}
}
double probability = getDifferentLinesProbability(previousBlock, nextBlock, true, areShouldBeCloseLines);
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
return true;
}
private static List<TextBlock> detectFirstLinesOfParagraphWithLeftAlignments(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (isFirstLineOfParagraphWithLeftAlignment(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.LEFT);
previousBlock.setHasStartLine(true);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static boolean isFirstLineOfParagraphWithLeftAlignment(TextBlock previousBlock, TextBlock nextBlock) {
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (previousBlock.getLinesNumber() != 1) {
return false;
}
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (BulletedParagraphUtils.isLabeledLine(nextBlock.getFirstLine())) {
return false;
}
if (nextBlock.isHasStartLine()) {
return false;
}
if (nextBlock.getTextAlignment() != TextAlignment.LEFT) {
return false;
}
if (!CaptionUtils.areOverlapping(previousBlock.getLastLine(), nextBlock.getFirstLine().getBoundingBox())) {
return false;
}
return true;
}
private static List<TextBlock> detectTwoLinesParagraphs(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (isTwoLinesParagraph(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.LEFT);
previousBlock.setHasStartLine(true);
previousBlock.setHasEndLine(true);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static boolean isTwoLinesParagraph(TextBlock previousBlock, TextBlock nextBlock) {
if (previousBlock.getLinesNumber() != 1 || nextBlock.getLinesNumber() != 1) {
return false;
}
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (BulletedParagraphUtils.isLabeledLine(nextBlock.getFirstLine())) {
return false;
}
if (previousBlock.getLastLine().getLeftX() < nextBlock.getFirstLine().getLeftX() ||
previousBlock.getLastLine().getRightX() < nextBlock.getFirstLine().getRightX()) {
return false;
}
return true;
}
private static boolean isFirstLineOfBulletedParagraphWithLeftAlignment(TextBlock previousBlock, TextBlock nextBlock) {
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
if (previousBlock.getLinesNumber() != 1) {
return false;
}
if (nextBlock.isHasStartLine()) {
return false;
}
if (BulletedParagraphUtils.isLabeledLine(nextBlock.getFirstLine())) {
return false;
}
if (!BulletedParagraphUtils.isLabeledLine(previousBlock.getFirstLine())) {
return false;
}
if (previousBlock.getLastLine().getLeftX() > nextBlock.getFirstLine().getLeftX()) {
return false;
}
if (nextBlock.getTextAlignment() != TextAlignment.LEFT && nextBlock.getLinesNumber() != 1) {
return false;
}
if (!CaptionUtils.areOverlapping(previousBlock.getLastLine(), nextBlock.getFirstLine().getBoundingBox())) {
return false;
}
return true;
}
private static List<TextBlock> detectParagraphsWithRightAlignments(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (areLinesOfParagraphsWithRightAlignments(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.RIGHT);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static List<TextBlock> detectBulletedParagraphsWithLeftAlignments(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (isFirstLineOfBulletedParagraphWithLeftAlignment(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
previousBlock.setTextAlignment(TextAlignment.LEFT);
previousBlock.setHasStartLine(true);
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static List<TextBlock> processOtherLines(List<TextBlock> textBlocks) {
List<TextBlock> newBlocks = new ArrayList<>();
if (!textBlocks.isEmpty()) {
newBlocks.add(textBlocks.get(0));
}
if (textBlocks.size() > 1) {
for (int i = 1; i < textBlocks.size(); i++) {
TextBlock previousBlock = newBlocks.get(newBlocks.size() - 1);
TextBlock nextBlock = textBlocks.get(i);
if (isOneParagraph(previousBlock, nextBlock)) {
previousBlock.add(nextBlock.getLines());
} else {
newBlocks.add(nextBlock);
}
}
}
return newBlocks;
}
private static boolean isOneParagraph(TextBlock previousBlock, TextBlock nextBlock) {
if (!areCloseStyle(previousBlock, nextBlock)) {
return false;
}
double probability = getDifferentLinesProbability(previousBlock, nextBlock, false, false);
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (BulletedParagraphUtils.isLabeledLine(nextBlock.getFirstLine())) {
return false;
}
if (!CaptionUtils.areOverlapping(previousBlock.getLastLine(), nextBlock.getFirstLine().getBoundingBox())) {
return false;
}
if (previousBlock.getLinesNumber() != 1 && previousBlock.getTextAlignment() != null) {
return false;
}
if (nextBlock.getLinesNumber() != 1 && nextBlock.getTextAlignment() != null) {
return false;
}
return true;
}
private static boolean isFirstLineOfBlock(TextBlock previousBlock, TextBlock nextBlock, TextAlignment textAlignment,
double probability) {
if (previousBlock.getLinesNumber() != 1) {
return false;
}
if (textAlignment != TextAlignment.RIGHT) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (nextBlock.getTextAlignment() != TextAlignment.JUSTIFY) {
return false;
}
if (nextBlock.isHasStartLine()) {
return false;
}
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
return true;
}
private static boolean isLastLineOfBlock(TextBlock previousBlock, TextBlock nextBlock, TextAlignment textAlignment,
double probability) {
if (nextBlock.getLinesNumber() != 1) {
return false;
}
if (textAlignment != TextAlignment.LEFT) {
return false;
}
if (!areTextBlocksHaveSameTextSize(previousBlock, nextBlock)) {
return false;
}
if (previousBlock.getTextAlignment() != TextAlignment.JUSTIFY) {
return false;
}
if (previousBlock.isHasEndLine()) {
return false;
}
if (probability < DIFFERENT_LINES_PROBABILITY) {
return false;
}
return true;
}
public static SemanticParagraph createParagraphFromTextBlock(TextBlock textBlock) {
SemanticParagraph textParagraph = new SemanticParagraph();
textParagraph.getColumns().add(new TextColumn());
textParagraph.getLastColumn().getBlocks().add(textBlock);
textParagraph.setBoundingBox(textBlock.getBoundingBox());
textParagraph.setCorrectSemanticScore(1.0);
textParagraph.setHiddenText(textBlock.isHiddenText());
return textParagraph;
}
private static double getDifferentLinesProbability(TextBlock previousBlock, TextBlock nextBlock,
boolean areSupportNotSingleLines, boolean areShouldBeCloseLines) {
if (previousBlock.isHiddenText() != nextBlock.isHiddenText()) {
return 0;
}
if (previousBlock.getLinesNumber() == 1 && nextBlock.getLinesNumber() == 1) {
return ChunksMergeUtils.mergeLeadingProbability(previousBlock.getLastLine(), nextBlock.getFirstLine());
}
if (previousBlock.getLinesNumber() == 1) {
return ChunksMergeUtils.mergeLeadingProbability(previousBlock.getLastLine(), nextBlock, areShouldBeCloseLines);
}
if (nextBlock.getLinesNumber() == 1) {
return ChunksMergeUtils.mergeLeadingProbability(previousBlock, nextBlock.getFirstLine(), areShouldBeCloseLines);
}
if (areSupportNotSingleLines) {
return ChunksMergeUtils.mergeLeadingProbability(previousBlock, nextBlock);
}
return 0;
}
private static boolean areCloseStyle(TextBlock previousBlock, TextBlock nextBlock) {
return NodeUtils.areCloseNumbers(previousBlock.getFontSize(), nextBlock.getFontSize(), 1e-1) &&
NodeUtils.areCloseNumbers(previousBlock.getFirstLine().getFirstTextChunk().getFontWeight(),
nextBlock.getFirstLine().getFirstTextChunk().getFontWeight(), 1e-1);
}
private static boolean areTextBlocksHaveSameTextSize(TextBlock firstBlock, TextBlock secondBlock) {
for (Double textSize1 : firstBlock.getTextSizes()) {
for (Double textSize2 : secondBlock.getTextSizes()) {
if (NodeUtils.areCloseNumbers(textSize1, textSize2)) {
return true;
}
}
}
return false;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/ProcessingResult.java
+80
View File
@@ -0,0 +1,80 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import com.fasterxml.jackson.databind.JsonNode;
/**
* Result of {@link DocumentProcessor#processFile} containing optional metadata
* collected during processing.
*
* <p>Currently carries hybrid-server pipeline timings when hybrid mode is used.
* The timings JSON has the structure:
* <pre>{
* "layout": {"total_s": 1.2, "avg_s": 0.12, "count": 10},
* "ocr": {"total_s": 2.1, "avg_s": 0.21, "count": 10},
* "table_structure": {"total_s": 0.8, "avg_s": 0.40, "count": 2},
* "doc_enrich": {"total_s": 5.3, "avg_s": 5.30, "count": 1},
* ...
* }</pre>
*/
public class ProcessingResult {
private static final ProcessingResult EMPTY = new ProcessingResult(null, 0, 0);
private final JsonNode hybridTimings;
private final long extractionNs;
private final long outputNs;
public ProcessingResult(JsonNode hybridTimings, long extractionNs, long outputNs) {
this.hybridTimings = hybridTimings;
this.extractionNs = extractionNs;
this.outputNs = outputNs;
}
/** Returns an empty result. */
public static ProcessingResult empty() {
return EMPTY;
}
/**
* Per-step pipeline timings from the hybrid server, or {@code null} if
* hybrid mode was not used or the server did not return timings.
*/
public JsonNode getHybridTimings() {
return hybridTimings;
}
/** Time spent on data extraction (parsing + layout analysis + content extraction), in nanoseconds. */
public long getExtractionNs() {
return extractionNs;
}
/** Time spent on output generation (auto-tagging + JSON/MD/HTML export), in nanoseconds. */
public long getOutputNs() {
return outputNs;
}
/** Extraction time in milliseconds. */
public long getExtractionMs() {
return extractionNs / 1_000_000;
}
/** Output generation time in milliseconds. */
public long getOutputMs() {
return outputNs / 1_000_000;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/SpecialTableProcessor.java
+100
View File
@@ -0,0 +1,100 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.util.ArrayList;
import java.util.List;
public class SpecialTableProcessor {
private static final String KOREAN_TABLE_REGEX = "\\(?(수신|경유|제목)\\)?.*";
public static List<IObject> detectSpecialTables(List<IObject> contents) {
detectSpecialKoreanTables(contents);
return DocumentProcessor.removeNullObjectsFromList(contents);
}
private static void detectSpecialKoreanTables(List<IObject> contents) {
List<TextLine> lines = new ArrayList<>();
Integer index = null;
for (int currentIndex = 0; currentIndex < contents.size(); currentIndex++) {
IObject content = contents.get(currentIndex);
if (content instanceof TextLine) {
TextLine line = ((TextLine) content);
if (line.getValue().matches(KOREAN_TABLE_REGEX)) {
lines.add(line);
contents.set(currentIndex, null);
if (index == null) {
index = currentIndex;
}
} else if (!lines.isEmpty()) {
contents.set(index, detectSpecialKoreanTable(lines));
lines.clear();
}
} else if (!lines.isEmpty()) {
contents.set(index, detectSpecialKoreanTable(lines));
lines.clear();
}
}
if (!lines.isEmpty()) {
contents.set(index, detectSpecialKoreanTable(lines));
}
}
private static TableBorder detectSpecialKoreanTable(List<TextLine> lines) {
TableBorder table = new TableBorder(lines.size(), 2);
for (int rowNumber = 0; rowNumber < lines.size(); rowNumber++) {
TextLine line = lines.get(rowNumber);
BoundingBox box = line.getBoundingBox();
int index = line.getValue().indexOf(":");
boolean isOneCellRow = index == -1;
TableBorderRow tableBorderRow = new TableBorderRow(rowNumber, 2, null);
table.getRows()[rowNumber] = tableBorderRow;
if (isOneCellRow) {
TableBorderCell tableBorderCell = new TableBorderCell(rowNumber, 0, 1, 2, null);
tableBorderCell.setSemanticType(SemanticType.TABLE_CELL);
tableBorderCell.addContentObject(line);
tableBorderCell.setBoundingBox(box);
tableBorderRow.getCells()[0] = tableBorderCell;
tableBorderRow.getCells()[1] = tableBorderCell;
} else {
TableBorderCell cell1 = new TableBorderCell(rowNumber, 0, 1, 1, null);
cell1.setSemanticType(SemanticType.TABLE_CELL);
TextLine line1 = new TextLine(line, 0, index - 1);
cell1.addContentObject(line1);
cell1.setBoundingBox(line1.getBoundingBox());
tableBorderRow.getCells()[0] = cell1;
TableBorderCell cell2 = new TableBorderCell(rowNumber, 1, 1, 1, null);
cell2.setSemanticType(SemanticType.TABLE_CELL);
TextLine line2 = new TextLine(line, index + 1, line.getValue().length());
cell2.addContentObject(line2);
cell2.setBoundingBox(line2.getBoundingBox());
tableBorderRow.getCells()[1] = cell2;
}
tableBorderRow.setBoundingBox(box);
table.getBoundingBox().union(box);
}
return TableBorderProcessor.normalizeAndProcessTableBorder(new ArrayList<>(lines), table, table.getPageNumber());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TableBorderProcessor.java
+253
View File
@@ -0,0 +1,253 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class TableBorderProcessor {
private static final double LINE_ART_PERCENT = 0.9;
private static final double NEIGHBOUR_TABLE_EPSILON = 0.2;
/**
* Maximum depth for nested table processing.
* Real-world PDFs rarely have tables nested more than 2-3 levels.
* This limit prevents stack overflow from malicious or malformed PDFs.
*/
private static final int MAX_NESTED_TABLE_DEPTH = 10;
/**
* Thread-local counter for tracking current nesting depth.
*/
private static final ThreadLocal<Integer> currentDepth = ThreadLocal.withInitial(() -> 0);
public static List<IObject> processTableBorders(List<IObject> contents, int pageNumber) {
// Check if TableBordersCollection exists (may be null if no borders detected during preprocessing)
if (StaticContainers.getTableBordersCollection() == null) {
return new ArrayList<>(contents);
}
// Check depth limit to prevent stack overflow from deeply nested tables
int depth = currentDepth.get();
if (depth >= MAX_NESTED_TABLE_DEPTH) {
// Exceeded maximum nesting depth - return contents without further table processing
return new ArrayList<>(contents);
}
try {
currentDepth.set(depth + 1);
List<IObject> newContents = new ArrayList<>();
Set<TableBorder> processedTableBorders = new LinkedHashSet<>();
for (IObject content : contents) {
TableBorder tableBorder = addContentToTableBorder(content);
if (tableBorder != null) {
if (content instanceof LineChunk && tableBorder.isOneCellTable()) {
continue;
}
if (!processedTableBorders.contains(tableBorder)) {
processedTableBorders.add(tableBorder);
newContents.add(tableBorder);
}
if (content instanceof TextChunk) {
TextChunk textChunk = (TextChunk) content;
TextChunk textChunkPart = getTextChunkPartBeforeTable(textChunk, tableBorder);
if (textChunkPart != null && !textChunkPart.isEmpty() && !textChunkPart.isWhiteSpaceChunk()) {
newContents.add(textChunkPart);
}
textChunkPart = getTextChunkPartAfterTable(textChunk, tableBorder);
if (textChunkPart != null && !textChunkPart.isEmpty() && !textChunkPart.isWhiteSpaceChunk()) {
newContents.add(textChunkPart);
}
}
} else {
newContents.add(content);
}
}
Map<TableBorder, TableBorder> normalizedTables = new HashMap<>();
for (TableBorder border : processedTableBorders) {
StaticContainers.getTableBordersCollection().removeTableBorder(border, pageNumber);
TableBorder normalizedTable = normalizeAndProcessTableBorder(contents, border, pageNumber);
normalizedTables.put(border, normalizedTable);
// Remove the outer table while processing its contents, then restore the page index
// with the final instance so later lookups still see the normalized table.
// StaticContainers.getTableBordersCollection().getTableBorders(pageNumber).add(normalizedTable);
}
for (int index = 0; index < newContents.size(); index++) {
IObject content = newContents.get(index);
if (content instanceof TableBorder && normalizedTables.containsKey(content)) {
newContents.set(index, normalizedTables.get(content));
}
}
return newContents;
} finally {
// Reset depth when exiting this level (clean up ThreadLocal)
if (depth == 0) {
currentDepth.remove();
} else {
currentDepth.set(depth);
}
}
}
private static TableBorder addContentToTableBorder(IObject content) {
if (StaticContainers.getTableBordersCollection() == null) {
return null;
}
TableBorder tableBorder = StaticContainers.getTableBordersCollection().getTableBorder(content.getBoundingBox());
if (tableBorder != null) {
if (content instanceof LineChunk) {
return tableBorder;
}
if (content instanceof LineArtChunk && BoundingBox.areSameBoundingBoxes(tableBorder.getBoundingBox(), content.getBoundingBox())) {
return tableBorder;
}
Set<TableBorderCell> tableBorderCells = tableBorder.getTableBorderCells(content);
if (!tableBorderCells.isEmpty()) {
if (tableBorderCells.size() > 1 && content instanceof TextChunk) {
TextChunk textChunk = (TextChunk) content;
for (TableBorderCell tableBorderCell : tableBorderCells) {
TextChunk currentTextChunk = getTextChunkPartForTableCell(textChunk, tableBorderCell);
if (currentTextChunk != null && !currentTextChunk.isEmpty()) {
tableBorderCell.addContentObject(currentTextChunk);
}
}
} else {
for (TableBorderCell tableBorderCell : tableBorderCells) {
if (content instanceof LineArtChunk &&
tableBorderCell.getBoundingBox().getIntersectionPercent(content.getBoundingBox()) > LINE_ART_PERCENT) {
return tableBorder;
}
tableBorderCell.addContentObject(content);
break;
}
}
return tableBorder;
}
if (content instanceof LineArtChunk) {
return tableBorder;
}
}
return null;
}
public static void processTableBorder(TableBorder tableBorder, int pageNumber) {
processTableBorderContents(tableBorder, pageNumber);
}
static TableBorder normalizeAndProcessTableBorder(List<IObject> rawPageContents, TableBorder tableBorder, int pageNumber) {
TableBorder normalizedTable = TableStructureNormalizer.normalize(rawPageContents, tableBorder);
processTableBorderContents(normalizedTable, pageNumber);
return normalizedTable;
}
private static void processTableBorderContents(TableBorder tableBorder, int pageNumber) {
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
TableBorderRow row = tableBorder.getRow(rowNumber);
for (int colNumber = 0; colNumber < tableBorder.getNumberOfColumns(); colNumber++) {
TableBorderCell tableBorderCell = row.getCell(colNumber);
if (tableBorderCell.getRowNumber() == rowNumber && tableBorderCell.getColNumber() == colNumber) {
tableBorderCell.setContents(processTableCellContent(tableBorderCell.getContents(), pageNumber));
}
}
}
}
private static List<IObject> processTableCellContent(List<IObject> contents, int pageNumber) {
List<IObject> newContents = TableBorderProcessor.processTableBorders(contents, pageNumber);
newContents = TextLineProcessor.processTextLines(newContents);
List<List<IObject>> contentsList = new ArrayList<>(1);
contentsList.add(newContents);
ListProcessor.processLists(contentsList, true);
newContents = contentsList.get(0);
newContents = ParagraphProcessor.processParagraphs(newContents);
newContents = ListProcessor.processListsFromTextNodes(newContents);
HeadingProcessor.processHeadings(newContents, true);
DocumentProcessor.setIDs(newContents);
CaptionProcessor.processCaptions(newContents);
contentsList.set(0, newContents);
ListProcessor.checkNeighborLists(contentsList);
newContents = contentsList.get(0);
return newContents;
}
public static void checkNeighborTables(List<List<IObject>> contents) {
TableBorder previousTable = null;
for (List<IObject> iObjects : contents) {
for (IObject content : iObjects) {
if (content instanceof TableBorder && !((TableBorder) content).isTextBlock()) {
TableBorder currentTable = (TableBorder) content;
if (previousTable != null) {
checkNeighborTables(previousTable, currentTable);
}
previousTable = currentTable;
} else {
if (!HeaderFooterProcessor.isHeaderOrFooter(content) &&
!(content instanceof LineChunk) && !(content instanceof LineArtChunk)) {
previousTable = null;
}
}
}
}
}
private static void checkNeighborTables(TableBorder previousTable, TableBorder currentTable) {
if (currentTable.getNumberOfColumns() != previousTable.getNumberOfColumns()) {
return;
}
if (!NodeUtils.areCloseNumbers(currentTable.getWidth(), previousTable.getWidth(), NEIGHBOUR_TABLE_EPSILON)) {
return;
}
for (int columnNumber = 0; columnNumber < previousTable.getNumberOfColumns(); columnNumber++) {
TableBorderCell cell1 = previousTable.getCell(0, columnNumber);
TableBorderCell cell2 = currentTable.getCell(0, columnNumber);
if (!NodeUtils.areCloseNumbers(cell1.getWidth(), cell2.getWidth(), NEIGHBOUR_TABLE_EPSILON)) {
return;
}
}
previousTable.setNextTable(currentTable);
currentTable.setPreviousTable(previousTable);
}
private static TextChunk getTextChunkPartForTableCell(TextChunk textChunk, TableBorderCell cell) {
return TextChunkUtils.getTextChunkPartForRange(textChunk, cell.getLeftX(), cell.getRightX(), true);
}
public static TextChunk getTextChunkPartBeforeTable(TextChunk textChunk, TableBorder table) {
return TextChunkUtils.getTextChunkPartBeforeBoundingBox(textChunk, table.getBoundingBox());
}
public static TextChunk getTextChunkPartAfterTable(TextChunk textChunk, TableBorder table) {
return TextChunkUtils.getTextChunkPartAfterBoundingBox(textChunk, table.getBoundingBox());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TableOfContentsProcessor.java
+221
View File
@@ -0,0 +1,221 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.*;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.semanticalgorithms.tocs.TOCIInfo;
import org.verapdf.wcag.algorithms.semanticalgorithms.tocs.TOCInterval;
import org.verapdf.wcag.algorithms.semanticalgorithms.consumers.TOCDetectionConsumer;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.util.*;
public class TableOfContentsProcessor {
private static final double TOC_ITEM_X_INTERVAL_RATIO = 0.3;
private final Set<String> pageLabels = new HashSet<>();
public void processTableOfContents(List<List<IObject>> contents) {
pageLabels.clear();
calculatePageLabels();
List<TOCInterval> intervalsList = getTocIntervals(contents);
for (TOCInterval interval : intervalsList) {
for (TOCIInfo info : interval.getTOCItemsInfos()) {
info.getTOCItemValue().setListLine(true);
}
}
for (TOCInterval interval : intervalsList) {
Integer currentPageNumber = interval.getTOCItemsInfos().get(0).getPageNumber();
int index = 0;
SemanticTOC previousTOC = null;
for (int i = 0; i < interval.getNumberOfTOCItems(); i++) {
TOCIInfo currentInfo = interval.getTOCItemsInfos().get(i);
if (!Objects.equals(currentInfo.getPageNumber(), currentPageNumber)) {
SemanticTOC toc = calculateTableOfContents(interval, index, i - 1,
contents.get(currentPageNumber));
processTOC(previousTOC, toc);
currentPageNumber = currentInfo.getPageNumber();
index = i;
previousTOC = toc;
}
}
SemanticTOC toc = calculateTableOfContents(interval, index, interval.getNumberOfTOCItems() - 1,
contents.get(currentPageNumber));
processTOC(previousTOC, toc);
}
contents.replaceAll(DocumentProcessor::removeNullObjectsFromList);
}
private static void processTOC(SemanticTOC previousTOC, SemanticTOC toc) {
for (IObject tocItem : toc.getTOCItems()) {
if (tocItem instanceof SemanticTOCI) {
SemanticTOCI toci = (SemanticTOCI) tocItem;
toci.setContents(processTOCItemContent(toci.getContents()));
} else {
//
}
}
if (previousTOC != null) {
SemanticTOC.setTOCConnected(previousTOC, toc);
}
}
private void calculatePageLabels() {
for (int pageNumber = 0; pageNumber < StaticContainers.getDocument().getNumberOfPages(); pageNumber++) {
String pageLabel = StaticContainers.getDocument().getPage(pageNumber).getPageLabel();
if (pageLabel != null) {
pageLabels.add(pageLabel);
}
}
}
private static List<IObject> processTOCItemContent(List<IObject> contents) {
List<IObject> newContents = ParagraphProcessor.processParagraphs(contents);
DocumentProcessor.setIDs(newContents);
return newContents;
}
private List<TOCInterval> getTocIntervals(List<List<IObject>> contents) {
List<TOCInterval> tocIntervals = new ArrayList<>();
for (List<IObject> pageContents : contents) {
for (int i = 0; i < pageContents.size(); i++) {
IObject content = pageContents.get(i);
if (!(content instanceof TextLine)) {
continue;
}
TextLine line = (TextLine) content;
String value = line.getValue();
if (value.isEmpty() || line.isHiddenText()) {
continue;
}
TOCIInfo tocItemTextInfo = createTOCIInfo(i, line, value);
processTOCItem(tocIntervals, tocItemTextInfo);
}
}
LinkedHashSet<TOCInterval> intervalsList = new LinkedHashSet<>();
for (TOCInterval interval : tocIntervals) {
if (interval != null && interval.getTOCItemsInfos().size() > 2) {
intervalsList.add(interval);
}
}
List<TOCInterval> result = new ArrayList<>(intervalsList);
Collections.reverse(result);
return result;
}
private void processTOCItem(List<TOCInterval> tocIntervals, TOCIInfo tocItemTextInfo) {
if (!hasPageNumber(tocItemTextInfo)) {
return;
}
boolean isSingle = true;
if (!tocIntervals.isEmpty()) {
TOCInterval interval = tocIntervals.get(tocIntervals.size() - 1);
if (isTwoTOCItemsOfOneTOC(interval, tocItemTextInfo)) {
tocIntervals.add(interval);
isSingle = false;
}
}
if (isSingle) {
TOCInterval tocInterval = new TOCInterval();
tocInterval.getTOCItemsInfos().add(tocItemTextInfo);
tocIntervals.add(tocInterval);
}
}
private static boolean isTwoTOCItemsOfOneTOC(TOCInterval interval, TOCIInfo currentItem) {
double maxXGap = getMaxXGap(currentItem.getTOCItemValue().getFontSize());
TOCIInfo previousItem = interval.getLastTOCItemInfo();
if (previousItem == null) {
return false;
}
if (Objects.equals(currentItem.getPageNumber(), previousItem.getPageNumber()) &&
currentItem.getTOCItemValue().getTopY() > previousItem.getTOCItemValue().getTopY()) {
return false;
}
if (!NodeUtils.areCloseNumbers(previousItem.getTOCItemValue().getRightX(),
currentItem.getTOCItemValue().getRightX(), maxXGap)) {
return false;
}
interval.getTOCItemsInfos().add(currentItem);
return true;
}
private boolean hasPageNumber(TOCIInfo tocItem) {
String textValue = tocItem.getText();
if (textValue.matches(".+[" + TOCDetectionConsumer.SPACES + "\\.]\\d+") && !textValue.matches(".*\\d+\\.\\d+")) {
return true;
}
for (String pageLabel : pageLabels) {
if (textValue.length() > pageLabel.length() && textValue.endsWith(pageLabel)) {
return true;
}
}
return false;
}
private static TOCIInfo createTOCIInfo(int i, TextLine line, String value) {
TOCIInfo tociInfo = new TOCIInfo();
tociInfo.setIndex(i);
tociInfo.setTOCItemValue(line);
tociInfo.setText(value);
return tociInfo;
}
private static SemanticTOC calculateTableOfContents(TOCInterval interval, int startIndex, int endIndex,
List<IObject> pageContents) {
SemanticTOC toc = new SemanticTOC();
for (int index = startIndex; index <= endIndex; index++) {
TOCIInfo currentInfo = interval.getTOCItemsInfos().get(index);
SemanticTOCI tocItem = new SemanticTOCI(new BoundingBox(), null);
TextLine textLine = (TextLine) pageContents.get(currentInfo.getIndex());
if (index != startIndex) {
int nextIndex = interval.getTOCItemsInfos().get(index - 1).getIndex();
addContentToTOCItem(nextIndex, currentInfo, pageContents, tocItem);
} else {
//
}
pageContents.set(interval.getTOCItemsInfos().get(index).getIndex(), index == startIndex ? toc : null);
tocItem.add(textLine);
toc.add(tocItem);
}
return toc;
}
private static void addContentToTOCItem(int nextIndex, TOCIInfo currentInfo, List<IObject> pageContents,
SemanticTOCI tocItem) {
List<TextLine> lines = new LinkedList<>();
for (int index = currentInfo.getIndex() - 1; index > nextIndex; index--) {
IObject content = pageContents.get(index);
if (content instanceof TextLine) {
TextLine currentTextLine = (TextLine) content;
lines.add(0, currentTextLine);
} else if (content != null) {
tocItem.getContents().add(0, content);
}
pageContents.set(index, null);
}
tocItem.add(lines);
}
private static double getMaxXGap(double fontSize) {
return fontSize * TOC_ITEM_X_INTERVAL_RATIO;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TableStructureNormalizer.java
+510
View File
@@ -0,0 +1,510 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
class TableStructureNormalizer {
private static final int MAX_UNDERSEGMENTED_ROWS = 2;
private static final int MIN_UNDERSEGMENTED_COLUMNS = 3;
private static final int MIN_UNDERSEGMENTED_TEXT_LINES = 8;
private static final int MIN_ROW_BAND_MISMATCH = 2;
private static final int OVERSIZED_CELL_LINE_COUNT = 4;
private static final double MIN_ROW_BAND_EPSILON = 3.0;
private static final double ROW_BAND_EPSILON_RATIO = 0.6;
private static final double ROW_BAND_ASSIGNMENT_EPSILON = 6.0;
private static final double ROW_ORDER_EPSILON = 1.5;
private static final Comparator<IObject> CONTENT_COMPARATOR =
Comparator.comparingDouble(IObject::getCenterY).reversed()
.thenComparingDouble(IObject::getLeftX);
private static final Comparator<TextLine> TEXT_LINE_COMPARATOR =
Comparator.comparingDouble(TextLine::getCenterY).reversed()
.thenComparingDouble(TextLine::getLeftX);
static TableBorder normalize(List<IObject> rawPageContents, TableBorder tableBorder) {
if (rawPageContents == null || rawPageContents.isEmpty()) {
return tableBorder;
}
if (tableBorder.isTextBlock()) {
return tableBorder;
}
if (tableBorder.getNumberOfRows() > MAX_UNDERSEGMENTED_ROWS ||
tableBorder.getNumberOfColumns() < MIN_UNDERSEGMENTED_COLUMNS) {
return tableBorder;
}
List<ColumnSnapshot> columnSnapshots = collectColumnSnapshots(rawPageContents, tableBorder);
int denseColumns = countDenseColumns(columnSnapshots);
if (denseColumns < 2) {
return tableBorder;
}
List<RowBand> rowBands = collectRowBands(tableBorder, columnSnapshots);
if (rowBands.size() < tableBorder.getNumberOfRows() + MIN_ROW_BAND_MISMATCH) {
return tableBorder;
}
TableBorder rebuiltTable = rebuildTable(tableBorder, rowBands);
if (!isReplacementQualityBetter(tableBorder, rebuiltTable)) {
return tableBorder;
}
return rebuiltTable;
}
private static List<ColumnSnapshot> collectColumnSnapshots(List<IObject> rawPageContents, TableBorder tableBorder) {
List<ColumnSnapshot> columnSnapshots = new ArrayList<>(tableBorder.getNumberOfColumns());
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
columnSnapshots.add(new ColumnSnapshot());
}
for (IObject content : rawPageContents) {
if (content == null || !isInsideTableBounds(content, tableBorder)) {
continue;
}
if (content instanceof TextChunk) {
addTextChunkToColumns((TextChunk) content, tableBorder, columnSnapshots);
} else if (!(content instanceof LineChunk) && !(content instanceof LineArtChunk)) {
int columnNumber = findBestColumn(content, tableBorder);
if (columnNumber >= 0) {
columnSnapshots.get(columnNumber).addContent(content);
}
}
}
for (ColumnSnapshot columnSnapshot : columnSnapshots) {
columnSnapshot.finalizeSnapshot();
}
return columnSnapshots;
}
private static void addTextChunkToColumns(TextChunk textChunk, TableBorder tableBorder,
List<ColumnSnapshot> columnSnapshots) {
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
TextChunk columnTextChunk = TextChunkUtils.getTextChunkPartForRange(textChunk,
tableBorder.getLeftX(columnNumber), tableBorder.getRightX(columnNumber), true);
if (columnTextChunk != null && !columnTextChunk.isEmpty() && !columnTextChunk.isWhiteSpaceChunk()) {
columnSnapshots.get(columnNumber).addContent(columnTextChunk);
}
}
}
private static int findBestColumn(IObject content, TableBorder tableBorder) {
double centerX = content.getCenterX();
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
if (centerX >= tableBorder.getLeftX(columnNumber) && centerX <= tableBorder.getRightX(columnNumber)) {
return columnNumber;
}
}
int closestColumn = -1;
double closestDistance = Double.MAX_VALUE;
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
double columnCenter = (tableBorder.getLeftX(columnNumber) + tableBorder.getRightX(columnNumber)) / 2;
double distance = Math.abs(centerX - columnCenter);
if (distance < closestDistance) {
closestDistance = distance;
closestColumn = columnNumber;
}
}
return closestColumn;
}
private static boolean isInsideTableBounds(IObject content, TableBorder tableBorder) {
return content.getCenterX() >= tableBorder.getLeftX() && content.getCenterX() <= tableBorder.getRightX() &&
content.getCenterY() >= tableBorder.getBottomY() && content.getCenterY() <= tableBorder.getTopY();
}
private static int countDenseColumns(List<ColumnSnapshot> columnSnapshots) {
int denseColumns = 0;
for (ColumnSnapshot columnSnapshot : columnSnapshots) {
if (columnSnapshot.meaningfulLineCount >= MIN_UNDERSEGMENTED_TEXT_LINES) {
denseColumns++;
}
}
return denseColumns;
}
private static List<RowBand> collectRowBands(TableBorder tableBorder, List<ColumnSnapshot> columnSnapshots) {
List<TextLine> textLines = new ArrayList<>();
for (ColumnSnapshot columnSnapshot : columnSnapshots) {
textLines.addAll(columnSnapshot.textLines);
}
textLines.sort(TEXT_LINE_COMPARATOR);
List<RowBand> rowBands = new ArrayList<>();
for (TextLine textLine : textLines) {
RowBand matchingBand = findMatchingRowBand(rowBands, textLine);
if (matchingBand == null) {
matchingBand = new RowBand(tableBorder.getNumberOfColumns());
rowBands.add(matchingBand);
}
matchingBand.addLine(textLine);
}
for (int columnNumber = 0; columnNumber < columnSnapshots.size(); columnNumber++) {
for (IObject content : columnSnapshots.get(columnNumber).contents) {
RowBand matchingBand = findBestRowBand(rowBands, content);
if (matchingBand != null) {
matchingBand.addContent(columnNumber, content);
}
}
}
rowBands.removeIf(rowBand -> rowBand.isEmpty());
rowBands.sort(Comparator.comparingDouble(RowBand::getCenterY).reversed());
rowBands.forEach(RowBand::sortContents);
return rowBands;
}
private static RowBand findMatchingRowBand(List<RowBand> rowBands, TextLine textLine) {
for (RowBand rowBand : rowBands) {
double epsilon = Math.max(MIN_ROW_BAND_EPSILON,
Math.min(rowBand.getAverageHeight(), textLine.getHeight()) * ROW_BAND_EPSILON_RATIO);
if (Math.abs(rowBand.getCenterY() - textLine.getCenterY()) <= epsilon ||
rowBand.hasVerticalOverlap(textLine.getTopY(), textLine.getBottomY())) {
return rowBand;
}
}
return null;
}
private static RowBand findBestRowBand(List<RowBand> rowBands, IObject content) {
RowBand bestBand = null;
double bestDistance = Double.MAX_VALUE;
for (RowBand rowBand : rowBands) {
if (rowBand.hasVerticalOverlap(content.getTopY(), content.getBottomY())) {
double distance = Math.abs(rowBand.getCenterY() - content.getCenterY());
if (distance < bestDistance) {
bestDistance = distance;
bestBand = rowBand;
}
}
}
if (bestBand != null) {
return bestBand;
}
for (RowBand rowBand : rowBands) {
double distance = Math.abs(rowBand.getCenterY() - content.getCenterY());
if (distance < bestDistance && distance <= ROW_BAND_ASSIGNMENT_EPSILON + rowBand.getAverageHeight()) {
bestDistance = distance;
bestBand = rowBand;
}
}
return bestBand;
}
private static TableBorder rebuildTable(TableBorder originalTable, List<RowBand> rowBands) {
TableBorder rebuiltTable = new TableBorder(rowBands.size(), originalTable.getNumberOfColumns());
rebuiltTable.setRecognizedStructureId(originalTable.getRecognizedStructureId());
rebuiltTable.setBoundingBox(new BoundingBox(originalTable.getBoundingBox()));
rebuiltTable.setNode(originalTable.getNode());
rebuiltTable.setIndex(originalTable.getIndex());
rebuiltTable.setLevel(originalTable.getLevel());
rebuiltTable.setPreviousTable(originalTable.getPreviousTable());
rebuiltTable.setNextTable(originalTable.getNextTable());
for (int rowNumber = 0; rowNumber < rowBands.size(); rowNumber++) {
RowBand rowBand = rowBands.get(rowNumber);
TableBorderRow rebuiltRow = new TableBorderRow(rowNumber, originalTable.getNumberOfColumns(),
originalTable.getRecognizedStructureId());
rebuiltRow.setBoundingBox(rowBand.createRowBoundingBox(originalTable));
rebuiltTable.getRows()[rowNumber] = rebuiltRow;
for (int columnNumber = 0; columnNumber < originalTable.getNumberOfColumns(); columnNumber++) {
TableBorderCell rebuiltCell = new TableBorderCell(rowNumber, columnNumber, 1, 1,
originalTable.getRecognizedStructureId());
rebuiltCell.setSemanticType(rowNumber == 0 ? SemanticType.TABLE_HEADER : SemanticType.TABLE_CELL);
rebuiltCell.setContents(rowBand.getContents(columnNumber));
rebuiltCell.setBoundingBox(rowBand.createCellBoundingBox(originalTable, columnNumber));
rebuiltRow.getCells()[columnNumber] = rebuiltCell;
}
}
rebuiltTable.calculateCoordinatesUsingBoundingBoxesOfRowsAndColumns();
return rebuiltTable;
}
private static boolean isReplacementQualityBetter(TableBorder originalTable, TableBorder rebuiltTable) {
int originalNonEmptyRows = countNonEmptyRows(originalTable);
int rebuiltNonEmptyRows = countNonEmptyRows(rebuiltTable);
if (rebuiltNonEmptyRows <= originalNonEmptyRows) {
return false;
}
int originalNonEmptyColumns = countNonEmptyColumns(originalTable);
int rebuiltNonEmptyColumns = countNonEmptyColumns(rebuiltTable);
if (rebuiltNonEmptyColumns < originalNonEmptyColumns) {
return false;
}
if (!hasMonotonicRowOrder(rebuiltTable)) {
return false;
}
TableLineStats originalLineStats = collectTableLineStats(originalTable);
TableLineStats rebuiltLineStats = collectTableLineStats(rebuiltTable);
return rebuiltLineStats.oversizedCellCount < originalLineStats.oversizedCellCount ||
rebuiltLineStats.maxMeaningfulTextLines < originalLineStats.maxMeaningfulTextLines;
}
private static int countNonEmptyRows(TableBorder tableBorder) {
int count = 0;
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
boolean hasContent = false;
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
TableBorderCell cell = tableBorder.getRow(rowNumber).getCell(columnNumber);
if (cell != null && cell.getRowNumber() == rowNumber && cell.getColNumber() == columnNumber &&
hasMeaningfulContent(cell.getContents())) {
hasContent = true;
break;
}
}
if (hasContent) {
count++;
}
}
return count;
}
private static int countNonEmptyColumns(TableBorder tableBorder) {
int count = 0;
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
boolean hasContent = false;
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
TableBorderCell cell = tableBorder.getRow(rowNumber).getCell(columnNumber);
if (cell != null && cell.getRowNumber() == rowNumber && cell.getColNumber() == columnNumber &&
hasMeaningfulContent(cell.getContents())) {
hasContent = true;
break;
}
}
if (hasContent) {
count++;
}
}
return count;
}
private static boolean hasMeaningfulContent(List<IObject> contents) {
if (contents == null) {
return false;
}
for (IObject content : contents) {
if (content instanceof TextChunk) {
if (!((TextChunk) content).isWhiteSpaceChunk() && !((TextChunk) content).isEmpty()) {
return true;
}
} else if (content instanceof TextLine) {
if (!((TextLine) content).isSpaceLine() && !((TextLine) content).isEmpty()) {
return true;
}
} else if (!(content instanceof LineChunk) && !(content instanceof LineArtChunk)) {
return true;
}
}
return false;
}
private static boolean hasMonotonicRowOrder(TableBorder tableBorder) {
double previousCenterY = Double.POSITIVE_INFINITY;
double previousBottomY = Double.POSITIVE_INFINITY;
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
TableBorderRow row = tableBorder.getRow(rowNumber);
double currentCenterY = row.getBoundingBox().getCenterY();
if (currentCenterY >= previousCenterY) {
return false;
}
if (row.getTopY() > previousBottomY + ROW_ORDER_EPSILON) {
return false;
}
previousCenterY = currentCenterY;
previousBottomY = row.getBottomY();
}
return true;
}
private static TableLineStats collectTableLineStats(TableBorder tableBorder) {
int oversizedCellCount = 0;
int maxMeaningfulTextLines = 0;
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
TableBorderCell cell = tableBorder.getRow(rowNumber).getCell(columnNumber);
if (cell != null && cell.getRowNumber() == rowNumber && cell.getColNumber() == columnNumber) {
int meaningfulTextLines = countMeaningfulTextLines(cell.getContents());
if (meaningfulTextLines >= OVERSIZED_CELL_LINE_COUNT) {
oversizedCellCount++;
}
maxMeaningfulTextLines = Math.max(maxMeaningfulTextLines, meaningfulTextLines);
}
}
}
return new TableLineStats(oversizedCellCount, maxMeaningfulTextLines);
}
private static int countMeaningfulTextLines(List<IObject> contents) {
if (contents == null || contents.isEmpty()) {
return 0;
}
List<IObject> orderedContents = new ArrayList<>(contents);
orderedContents.sort(CONTENT_COMPARATOR);
int count = 0;
for (IObject content : TextLineProcessor.processTextLines(orderedContents)) {
if (content instanceof TextLine) {
TextLine textLine = (TextLine) content;
if (!textLine.isEmpty() && !textLine.isSpaceLine()) {
count++;
}
}
}
return count;
}
private static final class ColumnSnapshot {
private final List<IObject> contents = new ArrayList<>();
private final List<TextLine> textLines = new ArrayList<>();
private int meaningfulLineCount;
private void addContent(IObject content) {
contents.add(content);
}
private void finalizeSnapshot() {
contents.sort(CONTENT_COMPARATOR);
List<IObject> textCandidates = new ArrayList<>();
for (IObject content : contents) {
if (content instanceof TextChunk || content instanceof TextLine) {
textCandidates.add(content);
}
}
for (IObject content : TextLineProcessor.processTextLines(textCandidates)) {
if (content instanceof TextLine) {
TextLine textLine = (TextLine) content;
if (!textLine.isEmpty() && !textLine.isSpaceLine()) {
textLines.add(textLine);
meaningfulLineCount++;
}
}
}
}
}
private static final class TableLineStats {
private final int oversizedCellCount;
private final int maxMeaningfulTextLines;
private TableLineStats(int oversizedCellCount, int maxMeaningfulTextLines) {
this.oversizedCellCount = oversizedCellCount;
this.maxMeaningfulTextLines = maxMeaningfulTextLines;
}
}
private static final class RowBand {
private final List<List<IObject>> contentsByColumn;
private double topY = Double.NEGATIVE_INFINITY;
private double bottomY = Double.POSITIVE_INFINITY;
private double centerY;
private double averageHeight;
private int lineCount;
private RowBand(int columnCount) {
this.contentsByColumn = new ArrayList<>(columnCount);
for (int columnNumber = 0; columnNumber < columnCount; columnNumber++) {
this.contentsByColumn.add(new ArrayList<>());
}
}
private void addLine(TextLine textLine) {
updateBounds(textLine.getTopY(), textLine.getBottomY(), textLine.getCenterY(), textLine.getHeight());
}
private void addContent(int columnNumber, IObject content) {
contentsByColumn.get(columnNumber).add(content);
updateBounds(content.getTopY(), content.getBottomY(), content.getCenterY(), content.getHeight());
}
private void updateBounds(double contentTopY, double contentBottomY, double contentCenterY, double height) {
topY = Math.max(topY, contentTopY);
bottomY = Math.min(bottomY, contentBottomY);
centerY = ((centerY * lineCount) + contentCenterY) / (lineCount + 1);
averageHeight = ((averageHeight * lineCount) + height) / (lineCount + 1);
lineCount++;
}
private boolean hasVerticalOverlap(double contentTopY, double contentBottomY) {
return contentBottomY <= topY + ROW_ORDER_EPSILON && contentTopY >= bottomY - ROW_ORDER_EPSILON;
}
private boolean isEmpty() {
for (List<IObject> contents : contentsByColumn) {
if (!contents.isEmpty()) {
return false;
}
}
return true;
}
private void sortContents() {
for (List<IObject> contents : contentsByColumn) {
contents.sort(CONTENT_COMPARATOR);
}
}
private List<IObject> getContents(int columnNumber) {
return new ArrayList<>(contentsByColumn.get(columnNumber));
}
private BoundingBox createRowBoundingBox(TableBorder tableBorder) {
return new BoundingBox(tableBorder.getPageNumber(), tableBorder.getLeftX(), bottomY,
tableBorder.getRightX(), topY);
}
private BoundingBox createCellBoundingBox(TableBorder tableBorder, int columnNumber) {
return new BoundingBox(tableBorder.getPageNumber(), tableBorder.getLeftX(columnNumber), bottomY,
tableBorder.getRightX(columnNumber), topY);
}
private double getCenterY() {
return centerY;
}
private double getAverageHeight() {
return averageHeight;
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TaggedDocumentProcessor.java
+497
View File
@@ -0,0 +1,497 @@
package org.opendataloader.pdf.processors;
import org.opendataloader.pdf.api.Config;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import org.opendataloader.pdf.entities.EnrichedImageChunk;
import org.opendataloader.pdf.entities.SemanticFootnote;
import org.verapdf.gf.model.impl.sa.GFSANode;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
import org.verapdf.wcag.algorithms.entities.content.TextBlock;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.enums.SemanticType;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.geometry.MultiBoundingBox;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderCell;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import org.verapdf.wcag.algorithms.semanticalgorithms.consumers.TableChecker;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import java.util.*;
public class TaggedDocumentProcessor {
private static List<List<IObject>> contents;
private static Stack<List<IObject>> contentsStack = new Stack<>();
private static Set<Integer> pagesToProcess;
public static List<List<IObject>> processDocument(String inputPdfName, Config config, Set<Integer> pages) {
pagesToProcess = pages;
contentsStack.clear();
contents = new ArrayList<>();
int totalPages = StaticContainers.getDocument().getNumberOfPages();
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
contents.add(new ArrayList<>());
}
ITree tree = StaticContainers.getDocument().getTree();
processStructElem(tree.getRoot(), null);
List<List<IObject>> artifacts = collectArtifacts(totalPages);
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
if (!shouldProcessPage(pageNumber)) {
continue;
}
artifacts.set(pageNumber, TextLineProcessor.processTextLines(artifacts.get(pageNumber)));
}
HeaderFooterProcessor.processHeadersAndFooters(artifacts, true);
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
if (!shouldProcessPage(pageNumber)) {
continue;
}
DocumentProcessor.setIDs(artifacts.get(pageNumber));
contents.get(pageNumber).addAll(artifacts.get(pageNumber));
}
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
if (!shouldProcessPage(pageNumber)) {
continue;
}
List<IObject> pageContents = TextLineProcessor.processTextLines(contents.get(pageNumber));
contents.set(pageNumber, ParagraphProcessor.processParagraphs(pageContents));
}
return contents;
}
private static List<List<IObject>> collectArtifacts(int totalPages) {
List<List<IObject>> artifacts = new ArrayList<>();
for (int pageNumber = 0; pageNumber < totalPages; pageNumber++) {
artifacts.add(new ArrayList<>());
if (!shouldProcessPage(pageNumber)) {
continue;
}
for (IObject content : StaticContainers.getDocument().getArtifacts(pageNumber)) {
if (content instanceof ImageChunk) {
artifacts.get(pageNumber).add(content);
} else if (content instanceof TextChunk) {
TextChunk textChunk = (TextChunk) content;
if (!textChunk.isWhiteSpaceChunk() && !textChunk.isEmpty()) {
artifacts.get(pageNumber).add(content);
}
}
}
}
return artifacts;
}
/**
* Checks if a page should be processed based on the filter.
*
* @param pageNumber 0-indexed page number
* @return true if the page should be processed
*/
private static boolean shouldProcessPage(int pageNumber) {
return pagesToProcess == null || pagesToProcess.contains(pageNumber);
}
private static void processStructElem(INode node, INode parent) {
if (node instanceof SemanticFigure) {
processImage((SemanticFigure) node, parent);
return;
}
if (node instanceof SemanticSpan) {
processTextChunk((SemanticSpan) node);
}
if (node.getInitialSemanticType() == null) {
for (INode child : node.getChildren()) {
processStructElem(child, node);
}
return;
}
switch (node.getInitialSemanticType()) {
case CAPTION:
processCaption(node);
break;
case HEADING:
case TITLE:
processHeading(node);
break;
case LIST:
processList(node);
break;
case NUMBER_HEADING:
processNumberedHeading(node);
break;
case PARAGRAPH:
processParagraph(node);
break;
case TABLE:
processTable(node);
break;
case TABLE_OF_CONTENT:
addObjectToContent(processTOC(node));
break;
case NOTE:
case FENOTE:
processFootnote(node);
break;
default:
for (INode child : node.getChildren()) {
processStructElem(child, node);
}
}
}
private static void addObjectToContent(IObject object) {
Integer pageNumber = object.getPageNumber();
if (pageNumber != null && shouldProcessPage(pageNumber)) {
if (contentsStack.isEmpty()) {
contents.get(pageNumber).add(object);
} else {
contentsStack.peek().add(object);
}
}
object.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
}
private static void processParagraph(INode paragraph) {
addObjectToContent(createParagraph(paragraph));
}
private static SemanticParagraph createParagraph(INode paragraph) {
List<IObject> contents = new ArrayList<>();
processChildContents(paragraph, contents);
contents = TextLineProcessor.processTextLines(contents);
TextBlock textBlock = new TextBlock(new MultiBoundingBox());
for (IObject content : contents) {
if (content instanceof TextLine) {
textBlock.add((TextLine)content);
} else {
addObjectToContent(content);
}
}
return ParagraphProcessor.createParagraphFromTextBlock(textBlock);
}
private static void processHeading(INode node) {
SemanticHeading heading = new SemanticHeading(createParagraph(node));
heading.setHeadingLevel(1);//update
addObjectToContent(heading);
}
private static void processFootnote(INode node) {
SemanticFootnote footnote = new SemanticFootnote(createParagraph(node));
addObjectToContent(footnote);
}
private static void processNumberedHeading(INode node) {
SemanticHeading heading = new SemanticHeading(createParagraph(node));
GFSANode gfsaNode = (GFSANode) node;
String headingLevel = gfsaNode.getStructElem().getstandardType();
heading.setHeadingLevel(Integer.parseInt(headingLevel.substring(1)));
addObjectToContent(heading);
}
private static void processList(INode node) {
PDFList list = new PDFList();
list.setBoundingBox(new MultiBoundingBox());
for (INode child : node.getChildren()) {
if (child.getInitialSemanticType() == SemanticType.LIST) {
processList(child);
} else if (child.getInitialSemanticType() == SemanticType.LIST_ITEM) {
ListItem listItem = processListItem(child);
if (listItem.getPageNumber() != null) {
list.add(listItem);
}
} else {
processStructElem(child, node);
}
}
addObjectToContent(list);
}
private static ListItem processListItem(INode node) {
ListItem listItem = new ListItem(new MultiBoundingBox(), null);
List<IObject> contents = new ArrayList<>();
listItem.setLabelLength(calculateLabelLength(node));
processChildContents(node, contents);
contents = TextLineProcessor.processTextLines(contents);
for (IObject content : contents) {
if (content instanceof TextLine) {
listItem.add((TextLine)content);
} else {
listItem.getContents().add(content);
}
}
listItem.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return listItem;
}
private static int calculateLabelLength(INode listItem) {
List<INode> children = listItem.getChildren();
if (!children.isEmpty()) {
INode firstChild = children.get(0);
if (firstChild.getInitialSemanticType() == SemanticType.LIST_LABEL) {
return getTextChunksLength(firstChild);
}
}
return 0;
}
private static int getTextChunksLength(INode node) {
int result = 0;
for (INode child : node.getChildren()) {
if (child instanceof SemanticSpan) {
result += (((SemanticSpan)child).getColumns().get(0).getFirstLine().getFirstTextChunk().getValue().length());
} else {
result += getTextChunksLength(child);
}
}
return result;
}
private static void processTable(INode tableNode) {
List<INode> tableRows = processTableRows(tableNode);
if (tableRows.isEmpty()) {
return;
}
int numberOfRows = tableRows.size();
int numberOfColumns = TableChecker.getNumberOfColumns(tableRows.get(0));
List<List<TableBorderCell>> table = new ArrayList<>(numberOfRows);
for (int rowNumber = 0; rowNumber < numberOfRows; rowNumber++) {
addTableRow(numberOfColumns, table);
}
BoundingBox tableBoundingBox = new MultiBoundingBox();
for (int rowNumber = 0; rowNumber < tableRows.size(); rowNumber++) {
int columnNumber = 0;
for (INode elem : tableRows.get(rowNumber).getChildren()) {
SemanticType type = elem.getInitialSemanticType();
if (SemanticType.TABLE_CELL != type && SemanticType.TABLE_HEADER != type) {
continue;
}
while (columnNumber < numberOfColumns && table.get(rowNumber).get(columnNumber) != null) {
++columnNumber;
}
TableBorderCell cell = new TableBorderCell(elem, rowNumber, columnNumber);
cell.setSemanticType(type);
processTableCell(cell, elem);
tableBoundingBox.union(cell.getBoundingBox());
for (int i = 0; i < cell.getRowSpan(); i++) {
if (rowNumber + i >= numberOfRows) {
numberOfRows++;
addTableRow(numberOfColumns, table);
}
for (int j = 0; j < cell.getColSpan(); j++) {
if (columnNumber + j >= numberOfColumns) {
addTableColumn(table);
numberOfColumns++;
}
table.get(rowNumber + i).set(columnNumber + j, cell);
}
}
columnNumber += cell.getColSpan();
}
}
if (tableBoundingBox.isEmpty()) {
//empty table
return;
}
TableBorder tableBorder = new TableBorder(tableBoundingBox, createRowsForTable(table, numberOfRows, numberOfColumns),
numberOfRows, numberOfColumns);
setBoundingBoxesForTableRowsAndTableCells(tableBorder);
addObjectToContent(tableBorder);
}
private static List<INode> processTableRows(INode table) {
List<INode> listTR = new LinkedList<>();
for (INode elem : table.getChildren()) {
SemanticType type = elem.getInitialSemanticType();
if (SemanticType.TABLE_ROW == type) {
listTR.add(elem);
processTableRowsChildren(elem);
} else if (SemanticType.TABLE_FOOTER == type || SemanticType.TABLE_BODY == type ||
SemanticType.TABLE_HEADERS == type) {
for (INode child : elem.getChildren()) {
if (SemanticType.TABLE_ROW == child.getInitialSemanticType()) {
listTR.add(child);
processTableRowsChildren(child);
} else {
processStructElem(child, elem);
}
}
} else {
processStructElem(elem, table);
}
}
return listTR;
}
private static void processTableRowsChildren(INode tableRow) {
for (INode tableCell : tableRow.getChildren()) {
SemanticType tableCellType = tableCell.getInitialSemanticType();
if (SemanticType.TABLE_CELL != tableCellType && SemanticType.TABLE_HEADER != tableCellType) {
processStructElem(tableCell, tableRow);
}
}
}
private static void addTableRow(int numberOfColumns, List<List<TableBorderCell>> table) {
List<TableBorderCell> row = new ArrayList<>(numberOfColumns);
table.add(row);
for (int columnNumber = 0; columnNumber < numberOfColumns; columnNumber++) {
row.add(null);
}
}
private static void addTableColumn(List<List<TableBorderCell>> table) {
for (List<TableBorderCell> tableBorderCells : table) {
tableBorderCells.add(null);
}
}
private static void processTableCell(TableBorderCell cell, INode elem) {
List<IObject> rawContents = new ArrayList<>();
processChildContents(elem, rawContents);
List<IObject> processed = TextLineProcessor.processTextLines(rawContents);
TextBlock textBlock = new TextBlock(new MultiBoundingBox());
for (IObject content : processed) {
if (content instanceof TextLine) {
textBlock.add((TextLine) content);
} else {
cell.getContents().add(content);
}
}
if (!textBlock.isEmpty()) {
cell.getContents().add(ParagraphProcessor.createParagraphFromTextBlock(textBlock));
}
BoundingBox cellBoundingBox = new MultiBoundingBox();
for (IObject content : cell.getContents()) {
cellBoundingBox.union(content.getBoundingBox());
}
cell.setBoundingBox(cellBoundingBox);
cell.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
}
private static void processChildContents(INode elem, List<IObject> contents) {
contentsStack.add(contents);
for (INode childChild : elem.getChildren()) {
processStructElem(childChild, elem);
}
contentsStack.pop();
}
private static TableBorderRow[] createRowsForTable(List<List<TableBorderCell>> table, int numberOfRows, int numberOfColumns) {
TableBorderRow[] rows = new TableBorderRow[numberOfRows];
for (int rowNumber = 0; rowNumber < numberOfRows; rowNumber++) {
rows[rowNumber] = new TableBorderRow(rowNumber, numberOfColumns, null);
}
for (int rowNumber = 0; rowNumber < numberOfRows; rowNumber++) {
for (int colNumber = 0; colNumber < numberOfColumns; colNumber++) {
rows[rowNumber].getCells()[colNumber] = table.get(rowNumber).get(colNumber);
if (rows[rowNumber].getCell(colNumber) == null) {
TableBorderCell cell = new TableBorderCell(rowNumber, colNumber, 1, 1, 0L);
cell.setSemanticType(SemanticType.TABLE_CELL);
cell.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
rows[rowNumber].getCells()[colNumber] = cell;
}
}
}
return rows;
}
private static void setBoundingBoxesForTableRowsAndTableCells(TableBorder tableBorder) {
BoundingBox boundingBox = new BoundingBox(tableBorder.getPageNumber(),
tableBorder.getTopY(), tableBorder.getLeftX(), tableBorder.getTopY(), tableBorder.getLeftX());
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
BoundingBox rowBoundingBox = new MultiBoundingBox();
for (int colNumber = 0; colNumber < tableBorder.getNumberOfColumns(); colNumber++) {
TableBorderCell cell = tableBorder.getCell(rowNumber, colNumber);
if (cell.getColNumber() == colNumber && cell.getRowNumber() == rowNumber) {
if (cell.getBoundingBox().isEmpty()) {
cell.setBoundingBox(boundingBox);
} else {
rowBoundingBox.union(tableBorder.getCell(rowNumber, colNumber).getBoundingBox());
}
}
}
tableBorder.getRow(rowNumber).setBoundingBox(rowBoundingBox.isEmpty() ? boundingBox : rowBoundingBox);
}
for (int rowNumber = 0; rowNumber < tableBorder.getNumberOfRows(); rowNumber++) {
for (int columnNumber = 0; columnNumber < tableBorder.getNumberOfColumns(); columnNumber++) {
TableBorderCell cell = tableBorder.getCell(rowNumber, columnNumber);
if (cell.getRowNumber() == rowNumber && cell.getColNumber() == columnNumber && cell.getBoundingBox().isEmpty()) {
cell.setBoundingBox(boundingBox);
}
}
}
}
private static void processCaption(INode node) {
SemanticCaption caption = new SemanticCaption(createParagraph(node));
addObjectToContent(caption);
}
private static SemanticTOC processTOC(INode node) {
SemanticTOC toc = new SemanticTOC();
toc.setBoundingBox(new MultiBoundingBox());
for (INode child : node.getChildren()) {
if (child.getInitialSemanticType() == SemanticType.TABLE_OF_CONTENT) {
toc.add(processTOC(child));
} else if (child.getInitialSemanticType() == SemanticType.TABLE_OF_CONTENT_ITEM) {
SemanticTOCI tocItem = processTOCItem(child);
if (tocItem.getPageNumber() != null) {
toc.add(tocItem);
}
} else {
processStructElem(child, node);
}
}
return toc;
}
private static SemanticTOCI processTOCItem(INode node) {
SemanticTOCI tocItem = new SemanticTOCI(new MultiBoundingBox(), null);
List<IObject> contents = new ArrayList<>();
processChildContents(node, contents);
contents = TextLineProcessor.processTextLines(contents);
for (IObject content : contents) {
if (content instanceof TextLine) {
tocItem.add((TextLine)content);
} else {
tocItem.getContents().add(content);
}
}
tocItem.setRecognizedStructureId(StaticLayoutContainers.incrementContentId());
return tocItem;
}
private static void processImage(SemanticFigure image, INode parent) {
GFSANode parentNode = (GFSANode) parent;
List<ImageChunk> images = image.getImages();
if (!images.isEmpty()) {
String alt = parentNode.getStructElem().getStructElemDictionary().getAlternateDescription();
ImageChunk imageChunk = images.get(0);
addObjectToContent(alt == null ? imageChunk : new EnrichedImageChunk(imageChunk, alt));
}
}
private static void processTextChunk(SemanticSpan semanticSpan) {
addObjectToContent(semanticSpan.getColumns().get(0).getFirstLine().getFirstTextChunk());
}
private static List<IObject> getContents(INode node) {
List<IObject> result = new ArrayList<>();
for (INode child : node.getChildren()) {
if (child instanceof SemanticSpan) {
result.add(((SemanticSpan)child).getColumns().get(0).getFirstLine().getFirstTextChunk());
} else if (child instanceof SemanticFigure) {
processImage((SemanticFigure)child, node);
} else {
result.addAll(getContents(child));
}
}
return result;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TextDecorationProcessor.java
+301
View File
@@ -0,0 +1,301 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.LineChunk;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.semanticalgorithms.containers.StaticContainers;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import java.util.*;
/**
* Detects strikethrough and underlined text by finding horizontal line or line-art rules that
* pass through the vertical center or baseline of text chunks. Marks affected TextChunks by
* setting their isStrikethroughText or isUnderlinedText fields to true.
*
* Filters to avoid false positives:
* 1. Rule-to-text-height ratio (rejects thick background fills/borders)
* 2. Rule-to-text width ratio (rejects rules wider than text)
* 3. Vertical center or baseline alignment
* 4. Horizontal overlap requirement
* 5. Multi-chunk matching using the combined text width
*/
public class TextDecorationProcessor {
private static final double VERTICAL_CENTER_TOLERANCE = 0.2;
private static final double MIN_HORIZONTAL_OVERLAP_RATIO = 0.8;
private static final double MAX_LINE_TO_TEXT_WIDTH_RATIO = 1.5;
// Real strikethrough marks are thin rules. Text-height rectangles commonly
// come from glyph bounds, filled artifacts, backgrounds, or table artwork.
private static final double MAX_RULE_THICKNESS = 2.0;
private static final double MAX_RULE_TO_TEXT_HEIGHT_RATIO = 0.25;
/**
* Detects strikethrough and underlined lines among page contents and sets affected
* TextChunk isStrikethroughText or isUnderlinedText field to true.
*
* @param pageContents the list of content objects for a page
* @param pageNumber the number of a page
* @param detectStrikethrough wether strikethrough text should be detected
*/
public static void processStrikethroughAndUnderlinedText(List<IObject> pageContents, int pageNumber, boolean detectStrikethrough) {
if (pageContents == null) {
return;
}
SortedSet<LineChunk> horizontalLines = StaticContainers.getLinesCollection() != null ?
StaticContainers.getLinesCollection().getHorizontalLines(pageNumber) : Collections.emptySortedSet();
List<HorizontalRuleCandidate> horizontalRules = new ArrayList<>();
for (LineChunk lineChunk : horizontalLines) {
HorizontalRuleCandidate rule = HorizontalRuleCandidate.fromLineChunk(lineChunk);
if (rule != null) {
horizontalRules.add(rule);
}
}
List<TextChunk> textChunks = new ArrayList<>();
for (IObject content : pageContents) {
// if (content instanceof LineArtChunk) {
// HorizontalRuleCandidate rule = HorizontalRuleCandidate.fromLineArtChunk((LineArtChunk) content);
// if (rule != null) {
// horizontalRules.add(rule);
// }
// } else
if (content instanceof TextChunk) {
textChunks.add((TextChunk) content);
}
}
if (horizontalRules.isEmpty() || textChunks.isEmpty()) {
return;
}
for (HorizontalRuleCandidate rule : horizontalRules) {
List<TextChunk> strikethroughMatches = new ArrayList<>();
List<TextChunk> underlineMatches = new ArrayList<>();
for (TextChunk textChunk : textChunks) {
if (textChunk.isWhiteSpaceChunk() || textChunk.isEmpty()) {
continue;
}
if (detectStrikethrough && isStrikethroughRule(rule, textChunk)) {
strikethroughMatches.add(textChunk);
}
if (isUnderlineRule(rule, textChunk)) {
underlineMatches.add(textChunk);
}
}
if (detectStrikethrough && isValidMatch(rule, strikethroughMatches)) {
for (TextChunk chunk : strikethroughMatches) {
if (!chunk.getIsStrikethroughText()) {
chunk.setIsStrikethroughText();
}
}
}
if (isValidMatch(rule, underlineMatches)) {
for (TextChunk chunk : underlineMatches) {
if (!chunk.getIsUnderlinedText()) {
chunk.setIsUnderlinedText();
}
}
}
}
}
/**
* Determines whether a horizontal line is a strikethrough for the given text chunk.
*/
static boolean isStrikethroughLine(LineChunk line, TextChunk textChunk) {
HorizontalRuleCandidate rule = HorizontalRuleCandidate.fromLineChunk(line);
return rule != null && isStrikethroughRule(rule, textChunk) && isValidMatch(rule, Collections.singletonList(textChunk));
}
/**
* Determines whether a line-art rectangle is a strikethrough for the given text chunk.
*/
// static boolean isStrikethroughLineArt(LineArtChunk lineArt, TextChunk textChunk) {
// HorizontalRuleCandidate rule = HorizontalRuleCandidate.fromLineArtChunk(lineArt);
// // Line-art rectangles are also used for table/background artwork, so
// // apply the width-ratio guard even for direct single-chunk checks.
// return rule != null && isStrikethroughRule(rule, textChunk) && isValidMatch(rule, List.of(textChunk));
// }
/**
* Determines whether a horizontal line is an underlined for the given text chunk.
*/
public static boolean isUnderlineLine(LineChunk line, TextChunk textChunk) {
HorizontalRuleCandidate rule = HorizontalRuleCandidate.fromLineChunk(line);
return rule != null && isUnderlineRule(rule, textChunk) && isValidMatch(rule, Collections.singletonList(textChunk));
}
/**
* Determines whether a line-art rectangle is an underlined for the given text chunk.
*/
// public static boolean isUnderlineLineArt(LineArtChunk lineArt, TextChunk textChunk) {
// HorizontalRuleCandidate rule = HorizontalRuleCandidate.fromLineArtChunk(lineArt);
// return rule != null && isUnderlineRule(rule, textChunk) && isValidMatch(rule, List.of(textChunk));
// }
// private static boolean isHorizontalLineArt(LineArtChunk lineArt) {
// BoundingBox bbox = lineArt.getBoundingBox();
// return bbox != null && bbox.getWidth() > bbox.getHeight();
// }
private static boolean isStrikethroughRule(HorizontalRuleCandidate rule, TextChunk textChunk) {
if (rule == null || textChunk == null) {
return false;
}
double textHeight = textChunk.getHeight();
if (textHeight <= 0) {
return false;
}
// LineChunk thickness is stroke width; LineArtChunk thickness is rectangle
// height. In both cases, strikethrough marks should stay thin relative to text.
double maxRuleThickness = Math.min(MAX_RULE_THICKNESS, textHeight * MAX_RULE_TO_TEXT_HEIGHT_RATIO);
if (rule.thickness > maxRuleThickness) {
return false;
}
// Check vertical position: the line's Y should be near the vertical center of the text
double textCenterY = textChunk.getCenterY();
double lineY = rule.centerY;
double tolerance = textHeight * VERTICAL_CENTER_TOLERANCE;
if (Math.abs(lineY - textCenterY) > tolerance) {
return false;
}
// Check horizontal overlap
double textLeftX = textChunk.getLeftX();
double textRightX = textChunk.getRightX();
double lineLeftX = rule.leftX;
double lineRightX = rule.rightX;
double overlapLeft = Math.max(textLeftX, lineLeftX);
double overlapRight = Math.min(textRightX, lineRightX);
double overlapWidth = overlapRight - overlapLeft;
if (overlapWidth <= 0) {
return false;
}
double textWidth = textChunk.getWidth();
if (textWidth <= 0 || (overlapWidth / textWidth) < MIN_HORIZONTAL_OVERLAP_RATIO) {
return false;
}
return true;
}
private static boolean isUnderlineRule(HorizontalRuleCandidate rule, TextChunk textChunk) {
if (rule == null || textChunk == null) {
return false;
}
double textHeight = textChunk.getHeight();
if (textHeight <= 0) return false;
// Thickness: line width must be less than eps[2] * textHeight
// (eps[2] = 0.3, i.e. 30% of text height)
if (rule.thickness >= NodeUtils.UNDERLINED_TEXT_EPSILONS[2] * textHeight) {
return false;
}
// Vertical position: line must be between the baseline and
// (baseline - eps[1] * textHeight)
double baseline = textChunk.getBaseLine();
double lineY = rule.centerY;
double lowerBound = baseline - NodeUtils.UNDERLINED_TEXT_EPSILONS[1] * textHeight;
// Assuming Y increases upward, baseline is above the descender zone.
// lineY should be <= baseline and >= lowerBound.
if (lineY > baseline || lineY < lowerBound) {
return false;
}
// Horizontal overlap: at least eps[0] * textWidth overlap
double textLeftX = textChunk.getLeftX();
double textRightX = textChunk.getRightX();
double overlap = Math.min(rule.rightX - textLeftX, textRightX - rule.leftX);
if (overlap <= NodeUtils.UNDERLINED_TEXT_EPSILONS[0] * textChunk.getWidth()) {
return false;
}
return true;
}
private static boolean isValidMatch(HorizontalRuleCandidate rule, List<TextChunk> matchingChunks) {
if (matchingChunks.isEmpty()) {
return false;
}
// Validate against the combined span of all matched chunks. This accepts
// split text on one visual line while still rejecting table separators
// and background rules that extend far past the text group.
double textGroupWidth = 0.0;
for (TextChunk chunk : matchingChunks) {
textGroupWidth += chunk.getWidth();
}
return textGroupWidth > 0 && rule.width / textGroupWidth <= MAX_LINE_TO_TEXT_WIDTH_RATIO;
}
/**
* Normalized geometry for either LineChunk or LineArtChunk.
*/
private static class HorizontalRuleCandidate {
private final double leftX;
private final double rightX;
private final double centerY;
private final double width;
private final double thickness;
private HorizontalRuleCandidate(double leftX, double rightX, double centerY, double width, double thickness) {
this.leftX = leftX;
this.rightX = rightX;
this.centerY = centerY;
this.width = width;
this.thickness = thickness;
}
private static HorizontalRuleCandidate fromLineChunk(LineChunk line) {
if (line == null || line.getBoundingBox() == null || !line.isHorizontalLine()) {
return null;
}
BoundingBox bbox = line.getBoundingBox();
return new HorizontalRuleCandidate(line.getLeftX(), line.getRightX(), line.getCenterY(),
bbox.getWidth(), line.getWidth());
}
// private static HorizontalRuleCandidate fromLineArtChunk(LineArtChunk lineArt) {
// if (lineArt == null || !isHorizontalLineArt(lineArt)) {
// return null;
// }
// BoundingBox bbox = lineArt.getBoundingBox();
// return new HorizontalRuleCandidate(lineArt.getLeftX(), lineArt.getRightX(), lineArt.getCenterY(),
// bbox.getWidth(), bbox.getHeight());
// }
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TextLineProcessor.java
+150
View File
@@ -0,0 +1,150 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.SemanticTextNode;
import org.verapdf.wcag.algorithms.entities.content.LineArtChunk;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.entities.content.TextLine;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ChunksMergeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ListUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Set;
public class TextLineProcessor {
private static final double ONE_LINE_PROBABILITY = 0.75;
private static final Comparator<TextChunk> TEXT_CHUNK_COMPARATOR =
Comparator.comparingDouble(o -> o.getBoundingBox().getLeftX());
public static List<IObject> processTextLines(List<IObject> contents) {
List<IObject> newContents = new ArrayList<>();
// Track which TextChunk immediately follows a whitespace chunk in stream order,
// using reference identity so lookups are immune to TextChunk.equals() semantics.
// Stream order may differ from visual (leftX) order in rare PDFs, but whitespace
// chunks originate from the same PDF text operator as their adjacent text chunks,
// so stream-order adjacency is reliable for this signal.
Set<TextChunk> chunksAfterWhitespace = Collections.newSetFromMap(new IdentityHashMap<>());
TextLine previousLine = new TextLine(new TextChunk(""));
boolean isSeparateLine = false;
boolean pendingWhitespace = false;
for (IObject content : contents) {
if (content instanceof TextChunk) {
TextChunk textChunk = (TextChunk) content;
if (textChunk.isWhiteSpaceChunk() || textChunk.isEmpty()) {
if (textChunk.isWhiteSpaceChunk()) {
pendingWhitespace = true;
}
continue;
}
if (pendingWhitespace) {
chunksAfterWhitespace.add(textChunk);
pendingWhitespace = false;
}
TextLine currentLine = new TextLine(textChunk);
double oneLineProbability = ChunksMergeUtils.countOneLineProbability(new SemanticTextNode(), previousLine, currentLine);
isSeparateLine |= (oneLineProbability < ONE_LINE_PROBABILITY) || previousLine.isHiddenText() != currentLine.isHiddenText();
if (isSeparateLine) {
previousLine.setBoundingBox(new BoundingBox(previousLine.getBoundingBox()));
previousLine = currentLine;
newContents.add(previousLine);
} else {
previousLine.add(currentLine);
}
isSeparateLine = false;
} else {
if (content instanceof TableBorder) {
isSeparateLine = true;
}
newContents.add(content);
pendingWhitespace = false;
}
}
for (int i = 0; i < newContents.size(); i++) {
IObject content = newContents.get(i);
if (content instanceof TextLine) {
TextLine textLine = (TextLine) content;
textLine.getTextChunks().sort(TEXT_CHUNK_COMPARATOR);
double threshold = textLine.getFontSize() * TextChunkUtils.TEXT_LINE_SPACE_RATIO;
newContents.set(i, getTextLineWithSpaces(textLine, threshold, chunksAfterWhitespace));
}
}
linkTextLinesWithConnectedLineArtBullet(newContents);
return newContents;
}
private static TextLine getTextLineWithSpaces(TextLine textLine, double threshold,
Set<TextChunk> chunksAfterWhitespace) {
List<TextChunk> textChunks = textLine.getTextChunks();
TextChunk currentTextChunk = textChunks.get(0);
double previousEnd = currentTextChunk.getTextEnd();
TextLine newLine = new TextLine();
newLine.add(currentTextChunk);
for (int i = 1; i < textChunks.size(); i++) {
currentTextChunk = textChunks.get(i);
double currentStart = currentTextChunk.getTextStart();
boolean hasGap = currentStart - previousEnd > threshold;
boolean hadWhitespace = chunksAfterWhitespace.contains(currentTextChunk);
if (hasGap || hadWhitespace) {
TextChunk spaceChunk = new TextChunk(new BoundingBox(currentTextChunk.getBoundingBox()), " ", currentTextChunk.getFontName(),
currentTextChunk.getFontSize(), currentTextChunk.getFontWeight(), currentTextChunk.getItalicAngle(), currentTextChunk.getBaseLine(),
currentTextChunk.getFontColor(), null, currentTextChunk.getSlantDegree());
spaceChunk.setTextStart(previousEnd);
spaceChunk.setTextEnd(currentStart);
spaceChunk.adjustSymbolEndsToBoundingBox(null);
newLine.add(spaceChunk);
}
previousEnd = currentTextChunk.getTextEnd();
newLine.add(currentTextChunk);
}
return newLine;
}
private static void linkTextLinesWithConnectedLineArtBullet(List<IObject> contents) {
LineArtChunk lineArtChunk = null;
for (IObject content : contents) {
if (content instanceof LineArtChunk) {
lineArtChunk = (LineArtChunk) content;
continue;
}
if (content instanceof TableBorder) {
lineArtChunk = null;
}
if (content instanceof TextLine && lineArtChunk != null) {
TextLine textLine = (TextLine) content;
if (isLineConnectedWithLineArt(textLine, lineArtChunk)) {
textLine.setConnectedLineArtLabel(lineArtChunk);
}
lineArtChunk = null;
}
}
}
private static boolean isLineConnectedWithLineArt(TextLine textLine, LineArtChunk lineArt) {
return lineArt.getRightX() <= textLine.getLeftX() && lineArt.getBoundingBox().getHeight() <
ListUtils.LIST_LABEL_HEIGHT_EPSILON * textLine.getBoundingBox().getHeight();
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/TextProcessor.java
+161
View File
@@ -0,0 +1,161 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors;
import org.verapdf.gf.model.factory.chunks.ChunkParser;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.content.ImageChunk;
import org.verapdf.wcag.algorithms.entities.content.TextChunk;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.ChunksMergeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.NodeUtils;
import org.verapdf.wcag.algorithms.semanticalgorithms.utils.TextChunkUtils;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class TextProcessor {
private static final double MIN_TEXT_INTERSECTION_PERCENT = 0.5;
private static final double MAX_TOP_DECORATION_IMAGE_EPSILON = 0.3;
private static final double MAX_BOTTOM_DECORATION_IMAGE_EPSILON = 0.1;
private static final double MAX_LEFT_DECORATION_IMAGE_EPSILON = 0.1;
private static final double MAX_RIGHT_DECORATION_IMAGE_EPSILON = 1.5;
private static final double NEIGHBORS_TEXT_CHUNKS_EPSILON = 0.1;
private static final double TEXT_MIN_HEIGHT = 1;
public static void replaceUndefinedCharacters(List<IObject> contents, String replacementCharacterString) {
if (ChunkParser.REPLACEMENT_CHARACTER_STRING.equals(replacementCharacterString)) {
return;
}
for (IObject object : contents) {
if (object instanceof TextChunk) {
TextChunk textChunk = ((TextChunk) object);
if (textChunk.getValue().contains(ChunkParser.REPLACEMENT_CHARACTER_STRING)) {
textChunk.setValue(textChunk.getValue().replace(ChunkParser.REPLACEMENT_CHARACTER_STRING, replacementCharacterString));
}
}
}
}
public static double measureReplacementCharRatio(List<IObject> contents) {
char replacementChar = ChunkParser.REPLACEMENT_CHARACTER_STRING.charAt(0);
int totalChars = 0;
int replacementChars = 0;
for (IObject object : contents) {
if (object instanceof TextChunk) {
String value = ((TextChunk) object).getValue();
totalChars += value.length();
for (int i = 0; i < value.length(); i++) {
if (value.charAt(i) == replacementChar) {
replacementChars++;
}
}
}
}
if (totalChars == 0) {
return 0.0;
}
return (double) replacementChars / totalChars;
}
public static void filterTinyText(List<IObject> contents) {
for (int i = 0; i < contents.size(); i++) {
IObject object = contents.get(i);
if (object instanceof TextChunk) {
TextChunk textChunk = ((TextChunk) object);
if (textChunk.getBoundingBox().getHeight() <= TEXT_MIN_HEIGHT) {
contents.set(i, null);
}
}
}
}
public static void trimTextChunksWhiteSpaces(List<IObject> contents) {
for (int i = 0; i < contents.size(); i++) {
IObject object = contents.get(i);
if (object instanceof TextChunk) {
contents.set(i, ChunksMergeUtils.getTrimTextChunk((TextChunk) object));
}
}
}
public static void mergeCloseTextChunks(List<IObject> contents) {
for (int i = 0; i < contents.size() - 1; i++) {
IObject object = contents.get(i);
IObject nextObject = contents.get(i + 1);
if (object instanceof TextChunk && nextObject instanceof TextChunk) {
TextChunk textChunk = (TextChunk) object;
TextChunk nextTextChunk = (TextChunk) nextObject;
if (TextChunkUtils.areTextChunksHaveSameStyle(textChunk, nextTextChunk) &&
TextChunkUtils.areTextChunksHaveSameBaseLine(textChunk, nextTextChunk) &&
areNeighborsTextChunks(textChunk, nextTextChunk)) {
contents.set(i, null);
contents.set(i + 1, TextChunkUtils.unionTextChunks(textChunk, nextTextChunk));
}
}
}
}
public static void removeSameTextChunks(List<IObject> contents) {
DocumentProcessor.setIndexesForContentsList(contents);
List<IObject> sortedTextChunks = contents.stream().filter(c -> c instanceof TextChunk).sorted(
Comparator.comparing(x -> ((TextChunk) x).getValue())).collect(Collectors.toList());
TextChunk lastTextChunk = null;
for (IObject object : sortedTextChunks) {
if (object instanceof TextChunk) {
TextChunk currentTextChunk = (TextChunk) object;
if (lastTextChunk != null && areSameTextChunks(lastTextChunk, currentTextChunk)) {
contents.set(lastTextChunk.getIndex(), null);
}
lastTextChunk = currentTextChunk;
}
}
}
public static boolean areSameTextChunks(TextChunk firstTextChunk, TextChunk secondTextChunk) {
return Objects.equals(firstTextChunk.getValue(), secondTextChunk.getValue()) &&
NodeUtils.areCloseNumbers(firstTextChunk.getWidth(), secondTextChunk.getWidth()) &&
NodeUtils.areCloseNumbers(firstTextChunk.getHeight(), secondTextChunk.getHeight()) &&
firstTextChunk.getBoundingBox().getIntersectionPercent(secondTextChunk.getBoundingBox()) > MIN_TEXT_INTERSECTION_PERCENT;
}
public static void removeTextDecorationImages(List<IObject> contents) {
TextChunk lastTextChunk = null;
for (int index = 0; index < contents.size(); index++) {
IObject object = contents.get(index);
if (object instanceof TextChunk) {
lastTextChunk = (TextChunk) object;
} else if (object instanceof ImageChunk && lastTextChunk != null &&
isTextChunkDecorationImage((ImageChunk) object, lastTextChunk)) {
contents.set(index, null);
}
}
}
public static boolean isTextChunkDecorationImage(ImageChunk imageChunk, TextChunk textChunk) {
return NodeUtils.areCloseNumbers(imageChunk.getTopY(), textChunk.getTopY(), MAX_TOP_DECORATION_IMAGE_EPSILON * textChunk.getHeight()) &&
NodeUtils.areCloseNumbers(imageChunk.getBottomY(), textChunk.getBottomY(), MAX_BOTTOM_DECORATION_IMAGE_EPSILON * textChunk.getHeight()) &&
(NodeUtils.areCloseNumbers(imageChunk.getLeftX(), textChunk.getLeftX(), MAX_LEFT_DECORATION_IMAGE_EPSILON * textChunk.getHeight()) || imageChunk.getLeftX() > textChunk.getLeftX()) &&
(NodeUtils.areCloseNumbers(imageChunk.getRightX(), textChunk.getRightX(), MAX_RIGHT_DECORATION_IMAGE_EPSILON * textChunk.getHeight()) || imageChunk.getRightX() < textChunk.getRightX());
}
private static boolean areNeighborsTextChunks(TextChunk firstTextChunk, TextChunk secondTextChunk) {
return NodeUtils.areCloseNumbers(firstTextChunk.getTextEnd(), secondTextChunk.getTextStart(),
NEIGHBORS_TEXT_CHUNKS_EPSILON * firstTextChunk.getBoundingBox().getHeight());
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/processors/readingorder/XYCutPlusPlusSorter.java
+651
View File
@@ -0,0 +1,651 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.processors.readingorder;
import org.verapdf.wcag.algorithms.entities.IObject;
import org.verapdf.wcag.algorithms.entities.geometry.BoundingBox;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
/**
* XY-Cut++ algorithm for reading order detection based on arXiv:2504.10258.
* <p>
* An enhanced XY-Cut implementation that handles:
* <ul>
* <li>Cross-layout elements (headers, footers spanning multiple columns)</li>
* <li>Adaptive axis selection based on density ratios</li>
* <li>L-shaped region handling</li>
* </ul>
* <p>
* This is a simplified geometric implementation without semantic type priorities.
* <p>
* Algorithm overview:
* <ol>
* <li>Pre-mask: Identify cross-layout elements (width > beta * maxWidth, overlaps >= 2)</li>
* <li>Compute density ratio to determine split direction preference</li>
* <li>Recursive segmentation with adaptive XY/YX-Cut</li>
* <li>Merge cross-layout elements at appropriate positions</li>
* </ol>
*/
public class XYCutPlusPlusSorter {
/** Default beta multiplier for cross-layout detection threshold.
* Higher value = fewer elements detected as cross-layout.
* 2.0 means element must be 2x wider than maxWidth to be considered cross-layout (effectively disabled). */
static final double DEFAULT_BETA = 2.0;
/** Default density threshold for adaptive axis selection. */
static final double DEFAULT_DENSITY_THRESHOLD = 0.9;
/** Minimum horizontal overlap ratio to count as overlapping. */
static final double OVERLAP_THRESHOLD = 0.1;
/** Minimum number of overlaps required for cross-layout classification. */
static final int MIN_OVERLAP_COUNT = 2;
/** Minimum gap size (in points) required to perform a cut.
* Prevents splitting on insignificant gaps (e.g., 1-pixel gaps). */
static final double MIN_GAP_THRESHOLD = 5.0;
/** Width ratio threshold for narrow outlier filtering.
* Elements narrower than this fraction of the region width are considered
* potential outliers that may bridge column gaps (e.g., page numbers, footnote markers). */
static final double NARROW_ELEMENT_WIDTH_RATIO = 0.1;
private XYCutPlusPlusSorter() {
// Utility class - prevent instantiation
}
// ========== PUBLIC API ==========
/**
* Sort objects using XY-Cut++ algorithm with default parameters.
*
* @param objects List of objects to sort
* @return Sorted list of objects in reading order
*/
public static List<IObject> sort(List<IObject> objects) {
return sort(objects, DEFAULT_BETA, DEFAULT_DENSITY_THRESHOLD);
}
/**
* Sort objects using XY-Cut++ algorithm with custom parameters.
*
* @param objects List of objects to sort
* @param beta Cross-layout detection threshold multiplier
* @param densityThreshold Density ratio threshold for axis selection
* @return Sorted list of objects in reading order
*/
public static List<IObject> sort(List<IObject> objects, double beta, double densityThreshold) {
if (objects == null || objects.size() <= 1) {
return objects;
}
// Filter out objects with null bounding boxes
List<IObject> validObjects = new ArrayList<>();
for (IObject obj : objects) {
if (obj != null && obj.getBoundingBox() != null) {
validObjects.add(obj);
}
}
if (validObjects.size() <= 1) {
return validObjects;
}
// Phase 1: Pre-mask cross-layout elements
List<IObject> crossLayoutElements = identifyCrossLayoutElements(validObjects, beta);
List<IObject> remainingObjects = new ArrayList<>(validObjects);
remainingObjects.removeAll(crossLayoutElements);
if (remainingObjects.isEmpty()) {
// All objects are cross-layout, just sort by Y
return sortByYThenX(validObjects);
}
// Phase 2: Compute density ratio for adaptive axis selection
double densityRatio = computeDensityRatio(remainingObjects);
boolean preferHorizontalFirst = densityRatio > densityThreshold;
// Phase 3: Recursive segmentation with adaptive axis
List<IObject> sortedMain = recursiveSegment(remainingObjects, preferHorizontalFirst);
// Phase 4: Merge cross-layout elements back at appropriate positions
return mergeCrossLayoutElements(sortedMain, crossLayoutElements);
}
// ========== PHASE 1: CROSS-LAYOUT DETECTION ==========
/**
* Identify cross-layout elements that span multiple regions.
* An element is cross-layout if:
* 1. Its width exceeds beta * maxWidth (where maxWidth is the widest element)
* 2. It horizontally overlaps with at least MIN_OVERLAP_COUNT other elements
*
* Using maxWidth instead of median ensures only truly wide elements
* (like titles spanning the full page) are detected as cross-layout.
*
* @param objects List of objects to analyze
* @param beta Threshold multiplier for width comparison (e.g., 0.7 = 70% of max width)
* @return List of cross-layout elements
*/
static List<IObject> identifyCrossLayoutElements(List<IObject> objects, double beta) {
List<IObject> crossLayoutElements = new ArrayList<>();
if (objects.size() < 3) {
// Need at least 3 objects for meaningful cross-layout detection
return crossLayoutElements;
}
// Calculate max width among all objects
double maxWidth = 0;
for (IObject obj : objects) {
BoundingBox bbox = obj.getBoundingBox();
if (bbox != null) {
double width = bbox.getWidth();
maxWidth = Math.max(maxWidth, width);
}
}
// Threshold: element must be at least beta * maxWidth to be cross-layout
// With beta=0.7, element must be at least 70% as wide as the widest element
double threshold = beta * maxWidth;
for (IObject obj : objects) {
BoundingBox bbox = obj.getBoundingBox();
if (bbox == null) {
continue;
}
double width = bbox.getWidth();
// Criterion 1: Width exceeds threshold (close to max width)
if (width >= threshold) {
// Criterion 2: Overlaps with at least MIN_OVERLAP_COUNT other elements
if (hasMinimumOverlaps(obj, objects, MIN_OVERLAP_COUNT)) {
crossLayoutElements.add(obj);
}
}
}
return crossLayoutElements;
}
/**
* Check if an element horizontally overlaps with at least minCount other elements.
*
* @param element The element to check
* @param objects All objects including the element
* @param minCount Minimum number of overlaps required
* @return true if the element overlaps with at least minCount other elements
*/
static boolean hasMinimumOverlaps(IObject element, List<IObject> objects, int minCount) {
BoundingBox elementBox = element.getBoundingBox();
if (elementBox == null) {
return false;
}
int overlapCount = 0;
for (IObject other : objects) {
if (other == element) {
continue;
}
BoundingBox otherBox = other.getBoundingBox();
if (otherBox == null) {
continue;
}
double overlapRatio = calculateHorizontalOverlapRatio(elementBox, otherBox);
if (overlapRatio >= OVERLAP_THRESHOLD) {
overlapCount++;
if (overlapCount >= minCount) {
return true;
}
}
}
return false;
}
/**
* Calculate the horizontal overlap ratio between two bounding boxes.
* The ratio is relative to the smaller box's width.
*
* @param box1 First bounding box
* @param box2 Second bounding box
* @return Overlap ratio (0.0 to 1.0)
*/
static double calculateHorizontalOverlapRatio(BoundingBox box1, BoundingBox box2) {
double overlapLeft = Math.max(box1.getLeftX(), box2.getLeftX());
double overlapRight = Math.min(box1.getRightX(), box2.getRightX());
double overlapWidth = Math.max(0, overlapRight - overlapLeft);
if (overlapWidth <= 0) {
return 0;
}
double width1 = box1.getWidth();
double width2 = box2.getWidth();
double smallerWidth = Math.min(width1, width2);
return smallerWidth > 0 ? overlapWidth / smallerWidth : 0;
}
// ========== PHASE 2: DENSITY RATIO COMPUTATION ==========
/**
* Compute the density ratio to determine split direction preference.
* Density = total content area / bounding region area.
* Higher density suggests content-dense layouts (newspapers) -> prefer horizontal splits.
* Lower density suggests sparse layouts -> prefer vertical splits.
*
* @param objects List of objects
* @return Density ratio (0.0 to 1.0)
*/
static double computeDensityRatio(List<IObject> objects) {
if (objects == null || objects.isEmpty()) {
return 1.0; // Default to XY-Cut
}
BoundingBox regionBounds = calculateBoundingRegion(objects);
if (regionBounds == null) {
return 1.0;
}
double regionArea = regionBounds.getArea();
if (regionArea <= 0) {
return 1.0;
}
double contentArea = calculateTotalArea(objects);
return Math.min(1.0, contentArea / regionArea);
}
/**
* Calculate the bounding box that encompasses all objects.
*
* @param objects List of objects
* @return Bounding box encompassing all objects, or null if no valid objects
*/
static BoundingBox calculateBoundingRegion(List<IObject> objects) {
BoundingBox boundingBox = new BoundingBox();
for (IObject obj : objects) {
BoundingBox bbox = obj.getBoundingBox();
boundingBox.union(bbox);
}
return boundingBox.isEmpty() ? null : boundingBox;
}
/**
* Calculate the total area covered by all objects.
*
* @param objects List of objects
* @return Total area
*/
static double calculateTotalArea(List<IObject> objects) {
double totalArea = 0;
for (IObject obj : objects) {
BoundingBox bbox = obj.getBoundingBox();
if (bbox != null) {
totalArea += bbox.getArea();
}
}
return totalArea;
}
// ========== PHASE 3: RECURSIVE SEGMENTATION ==========
/**
* Recursively segment and sort objects using adaptive XY/YX-Cut.
* <p>
* The algorithm uses projection-based gap detection to find clean cuts.
* For two-column academic paper layouts:
* 1. First try horizontal cut to separate header from body
* 2. Then try vertical cut to separate columns
* <p>
* The algorithm prefers horizontal cuts first (Y-axis split) when there's a significant
* horizontal gap, which properly handles layouts with wide headers followed by columns.
*
* @param objects List of objects to segment
* @param preferHorizontalFirst Initial preference (used as tiebreaker)
* @return Sorted list of objects
*/
static List<IObject> recursiveSegment(List<IObject> objects, boolean preferHorizontalFirst) {
if (objects == null || objects.size() <= 1) {
return objects != null ? new ArrayList<>(objects) : new ArrayList<>();
}
// Find best cuts in both directions using projection-based detection
CutInfo horizontalCut = findBestHorizontalCutWithProjection(objects);
CutInfo verticalCut = findBestVerticalCutWithProjection(objects);
// Choose cut direction based on gap sizes
// Apply minimum gap threshold to avoid splitting on insignificant gaps
boolean hasValidHorizontalCut = horizontalCut.gap >= MIN_GAP_THRESHOLD;
boolean hasValidVerticalCut = verticalCut.gap >= MIN_GAP_THRESHOLD;
boolean useHorizontalCut;
if (hasValidHorizontalCut && hasValidVerticalCut) {
// Both cuts available - prefer larger gap
useHorizontalCut = horizontalCut.gap > verticalCut.gap;
} else if (hasValidHorizontalCut) {
useHorizontalCut = true;
} else if (hasValidVerticalCut) {
useHorizontalCut = false;
} else {
// No valid cuts found - sort by Y then X (reading order)
return sortByYThenX(objects);
}
if (useHorizontalCut) {
List<List<IObject>> groups = splitByHorizontalCut(objects, horizontalCut.position);
// Safety: if split produced only one group, fall back to prevent infinite recursion
if (groups.size() <= 1) {
return sortByYThenX(objects);
}
return flatMapRecursive(groups, preferHorizontalFirst);
} else {
List<List<IObject>> groups = splitByVerticalCut(objects, verticalCut.position);
// Safety: if split produced only one group, fall back to prevent infinite recursion
if (groups.size() <= 1) {
return sortByYThenX(objects);
}
return flatMapRecursive(groups, preferHorizontalFirst);
}
}
/**
* Container for cut information including position and gap size.
*/
private static class CutInfo {
final double position;
final double gap;
CutInfo(double position, double gap) {
this.position = position;
this.gap = gap;
}
}
/**
* Recursively process groups and flatten results.
*/
private static List<IObject> flatMapRecursive(List<List<IObject>> groups, boolean preferHorizontalFirst) {
List<IObject> result = new ArrayList<>();
for (List<IObject> group : groups) {
result.addAll(recursiveSegment(group, preferHorizontalFirst));
}
return result;
}
/**
* Find the best vertical cut using projection profile.
* Projects all objects onto the X-axis and finds the largest gap.
*
* @param objects List of objects
* @return CutInfo containing position and gap size
*/
private static CutInfo findBestVerticalCutWithProjection(List<IObject> objects) {
if (objects.size() < 2) {
return new CutInfo(0, 0);
}
CutInfo edgeCut = findVerticalCutByEdges(objects);
// If the edge gap is already significant, use it directly.
if (edgeCut.gap >= MIN_GAP_THRESHOLD) {
return edgeCut;
}
// When edge gap is small, narrow outlier elements (e.g., page numbers,
// footnote markers) may bridge an otherwise clear column gap.
// Retry without elements narrower than 10% of the region width.
if (objects.size() >= 3) {
BoundingBox region = calculateBoundingRegion(objects);
if (region != null) {
double regionWidth = region.getWidth();
double narrowThreshold = regionWidth * NARROW_ELEMENT_WIDTH_RATIO;
List<IObject> filtered = new ArrayList<>();
for (IObject obj : objects) {
BoundingBox bbox = obj.getBoundingBox();
double width = bbox.getWidth();
if (width >= narrowThreshold) {
filtered.add(obj);
}
}
if (filtered.size() >= 2 && filtered.size() < objects.size()) {
CutInfo filteredCut = findVerticalCutByEdges(filtered);
if (filteredCut.gap > edgeCut.gap && filteredCut.gap >= MIN_GAP_THRESHOLD) {
return filteredCut;
}
}
}
}
return edgeCut;
}
/**
* Find vertical cut by edge gaps.
* Finds the largest gap between rightX of one element and leftX of the next.
*/
private static CutInfo findVerticalCutByEdges(List<IObject> objects) {
List<IObject> sorted = new ArrayList<>(objects);
sorted.sort(Comparator.comparingDouble((IObject o) -> o.getBoundingBox().getLeftX())
.thenComparingDouble(o -> o.getBoundingBox().getRightX()));
double largestGap = 0;
double cutPosition = 0;
Double prevRight = null;
for (IObject obj : sorted) {
double left = obj.getLeftX();
double right = obj.getRightX();
if (prevRight != null && left > prevRight) {
double gap = left - prevRight;
if (gap > largestGap) {
largestGap = gap;
cutPosition = (prevRight + left) / 2.0;
}
}
prevRight = (prevRight == null) ? right : Math.max(prevRight, right);
}
return new CutInfo(cutPosition, largestGap);
}
/**
* Find the best horizontal cut using projection profile.
* Projects all objects onto the Y-axis and finds the largest gap.
*
* @param objects List of objects
* @return CutInfo containing position and gap size
*/
private static CutInfo findBestHorizontalCutWithProjection(List<IObject> objects) {
if (objects.size() < 2) {
return new CutInfo(0, 0);
}
// Sort by topY descending (PDF: top to bottom)
List<IObject> sorted = new ArrayList<>(objects);
sorted.sort(Comparator.comparingDouble((IObject o) -> -o.getBoundingBox().getTopY())
.thenComparingDouble(o -> -o.getBoundingBox().getBottomY()));
double largestGap = 0;
double cutPosition = 0;
Double prevBottom = null;
for (IObject obj : sorted) {
double top = obj.getTopY();
double bottom = obj.getBottomY();
if (prevBottom != null && prevBottom > top) {
double gap = prevBottom - top;
if (gap > largestGap) {
largestGap = gap;
cutPosition = (prevBottom + top) / 2.0;
}
}
prevBottom = (prevBottom == null) ? bottom : Math.min(prevBottom, bottom);
}
return new CutInfo(cutPosition, largestGap);
}
/**
* Split objects by a horizontal cut at the given Y coordinate.
* Objects above the cut come first, then objects below.
*
* @param objects List of objects to split
* @param cutY Y coordinate of the cut
* @return List of two groups: [above, below]
*/
static List<List<IObject>> splitByHorizontalCut(List<IObject> objects, double cutY) {
List<IObject> above = new ArrayList<>();
List<IObject> below = new ArrayList<>();
for (IObject obj : objects) {
// Use center Y to determine which group
double centerY = obj.getCenterY();
if (centerY > cutY) {
above.add(obj);
} else {
below.add(obj);
}
}
List<List<IObject>> groups = new ArrayList<>();
if (!above.isEmpty()) {
groups.add(above);
}
if (!below.isEmpty()) {
groups.add(below);
}
return groups;
}
/**
* Split objects by a vertical cut at the given X coordinate.
* Objects to the left come first, then objects to the right.
*
* @param objects List of objects to split
* @param cutX X coordinate of the cut
* @return List of two groups: [left, right]
*/
static List<List<IObject>> splitByVerticalCut(List<IObject> objects, double cutX) {
List<IObject> left = new ArrayList<>();
List<IObject> right = new ArrayList<>();
for (IObject obj : objects) {
// Use center X to determine which group
double centerX = obj.getCenterX();
if (centerX < cutX) {
left.add(obj);
} else {
right.add(obj);
}
}
List<List<IObject>> groups = new ArrayList<>();
if (!left.isEmpty()) {
groups.add(left);
}
if (!right.isEmpty()) {
groups.add(right);
}
return groups;
}
// ========== PHASE 4: MERGING ==========
/**
* Merge cross-layout elements back into the sorted content at appropriate positions.
* Cross-layout elements are inserted based on their Y position relative to surrounding content.
*
* @param sortedMain Main content sorted by reading order
* @param crossLayoutElements Cross-layout elements to merge
* @return Merged list with cross-layout elements in correct positions
*/
static List<IObject> mergeCrossLayoutElements(List<IObject> sortedMain, List<IObject> crossLayoutElements) {
if (crossLayoutElements.isEmpty()) {
return sortedMain;
}
if (sortedMain.isEmpty()) {
return sortByYThenX(crossLayoutElements);
}
// Sort cross-layout elements by Y (top to bottom)
List<IObject> sortedCrossLayout = sortByYThenX(crossLayoutElements);
List<IObject> result = new ArrayList<>();
int mainIndex = 0;
int crossIndex = 0;
while (mainIndex < sortedMain.size() || crossIndex < sortedCrossLayout.size()) {
if (crossIndex >= sortedCrossLayout.size()) {
// No more cross-layout elements, add remaining main
result.add(sortedMain.get(mainIndex++));
} else if (mainIndex >= sortedMain.size()) {
// No more main elements, add remaining cross-layout
result.add(sortedCrossLayout.get(crossIndex++));
} else {
// Compare Y positions (PDF: higher Y = top)
IObject mainObj = sortedMain.get(mainIndex);
IObject crossObj = sortedCrossLayout.get(crossIndex);
double mainTopY = mainObj.getTopY();
double crossTopY = crossObj.getTopY();
if (crossTopY >= mainTopY) {
// Cross-layout element is above or at same level, add it first
result.add(crossObj);
crossIndex++;
} else {
// Main element is above, add it first
result.add(mainObj);
mainIndex++;
}
}
}
return result;
}
// ========== UTILITY METHODS ==========
/**
* Sort objects by Y coordinate (top to bottom), then X coordinate (left to right).
*
* @param objects List of objects to sort
* @return Sorted list
*/
static List<IObject> sortByYThenX(List<IObject> objects) {
List<IObject> sorted = new ArrayList<>(objects);
sorted.sort(Comparator
.comparingDouble((IObject o) -> -o.getBoundingBox().getTopY()) // Higher Y first (top)
.thenComparingDouble(o -> o.getBoundingBox().getLeftX())); // Lower X first (left)
return sorted;
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/text/TextGenerator.java
+268
View File
@@ -0,0 +1,268 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.text;
import org.opendataloader.pdf.api.Config;
import org.verapdf.wcag.algorithms.entities.*;
import org.verapdf.wcag.algorithms.entities.lists.ListItem;
import org.verapdf.wcag.algorithms.entities.lists.PDFList;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorder;
import org.verapdf.wcag.algorithms.entities.tables.tableBorders.TableBorderRow;
import java.io.Closeable;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
/**
* Generates a plain text representation of the extracted PDF contents.
*/
public class TextGenerator implements Closeable {
private static final Logger LOGGER = Logger.getLogger(TextGenerator.class.getCanonicalName());
private static final String INDENT = " ";
private final java.io.Writer textWriter;
private final String textFileName;
private final String lineSeparator = System.lineSeparator();
private final String textPageSeparator;
/**
* Page numbers (1-based) selected by --pages; an empty set means all pages.
* Sourced from the raw {@link Config#getPageNumbers()} list (not the
* validated set built by {@code DocumentProcessor.getValidPageNumbers}).
* Safe to compare against {@code pageIndex + 1} because the surrounding
* loop is bounded by the document's actual page count, so out-of-range
* values from the raw list are never tested for membership.
*/
private final Set<Integer> selectedPageNumbers;
private final boolean includeHeaderFooter;
public TextGenerator(File inputPdf, Config config) throws IOException {
String cutPdfFileName = inputPdf.getName();
this.textFileName = config.getOutputFolder() + File.separator + cutPdfFileName.substring(0, cutPdfFileName.length() - 3) + "txt";
this.textWriter = new FileWriter(textFileName, StandardCharsets.UTF_8);
this.textPageSeparator = config.getTextPageSeparator();
this.selectedPageNumbers = new HashSet<>(config.getPageNumbers());
this.includeHeaderFooter = config.isIncludeHeaderFooter();
}
/**
* Creates a TextGenerator that writes to an arbitrary Writer (e.g., stdout).
*/
public TextGenerator(java.io.Writer writer, Config config) {
this.textFileName = null;
this.textWriter = writer;
this.textPageSeparator = config.getTextPageSeparator();
this.selectedPageNumbers = new HashSet<>(config.getPageNumbers());
this.includeHeaderFooter = config.isIncludeHeaderFooter();
}
public void writeToText(List<List<IObject>> contents) {
try {
for (int pageIndex = 0; pageIndex < contents.size(); pageIndex++) {
if (selectedPageNumbers.isEmpty() || selectedPageNumbers.contains(pageIndex + 1)) {
writePageSeparator(pageIndex);
}
List<IObject> pageContents = contents.get(pageIndex);
writeContents(pageContents, 0);
if (pageIndex < contents.size() - 1) {
textWriter.write(lineSeparator);
}
}
LOGGER.log(Level.INFO, "Created {0}", textFileName);
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Unable to create text output: " + e.getMessage());
}
}
private void writePageSeparator(int pageIndex) throws IOException {
if (!textPageSeparator.isEmpty()) {
textWriter.write(textPageSeparator.contains(Config.PAGE_NUMBER_STRING)
? textPageSeparator.replace(Config.PAGE_NUMBER_STRING, String.valueOf(pageIndex + 1))
: textPageSeparator);
textWriter.write(lineSeparator);
}
}
private void writeContents(List<IObject> contents, int indentLevel) throws IOException {
for (int index = 0; index < contents.size(); index++) {
write(contents.get(index), indentLevel);
if (index < contents.size() - 1) {
textWriter.write(lineSeparator);
}
}
}
private void write(IObject object, int indentLevel) throws IOException {
if (object instanceof SemanticHeaderOrFooter) {
if (includeHeaderFooter) {
writeHeaderOrFooter((SemanticHeaderOrFooter) object, indentLevel);
}
} else if (object instanceof SemanticHeading) {
writeMultiline(((SemanticHeading) object).getValue(), indentLevel);
} else if (object instanceof SemanticParagraph) {
writeMultiline(((SemanticParagraph) object).getValue(), indentLevel);
} else if (object instanceof SemanticTextNode) {
writeMultiline(((SemanticTextNode) object).getValue(), indentLevel);
} else if (object instanceof PDFList) {
writeList((PDFList) object, indentLevel);
} else if (object instanceof SemanticTOC) {
writeTOC((SemanticTOC) object, indentLevel);
} else if (object instanceof TableBorder) {
writeTable((TableBorder) object, indentLevel);
}
}
private void writeHeaderOrFooter(SemanticHeaderOrFooter headerOrFooter, int indentLevel) throws IOException {
writeContents(headerOrFooter.getContents(), indentLevel);
}
private void writeList(PDFList list, int indentLevel) throws IOException {
for (ListItem item : list.getListItems()) {
writeMultiline(item.toString(), indentLevel);
if (!item.getContents().isEmpty()) {
writeContents(item.getContents(), indentLevel + 1);
}
}
}
private void writeTOC(SemanticTOC toc, int indentLevel) throws IOException {
for (IObject item : toc.getTOCItems()) {
if (item instanceof SemanticTOC) {
writeTOC((SemanticTOC)item, indentLevel + 1);
} else if (item instanceof SemanticTOCI) {
SemanticTOCI tocItem = (SemanticTOCI) item;
writeMultiline(tocItem.toString(), indentLevel);
if (!tocItem.getContents().isEmpty()) {
writeContents(tocItem.getContents(), indentLevel + 1);
}
}
}
}
private void writeTable(TableBorder table, int indentLevel) throws IOException {
String indent = indent(indentLevel);
for (TableBorderRow row : table.getRows()) {
String rowText = Arrays.stream(row.getCells())
.map(cell -> compactWhitespace(collectPlainText(cell.getContents())))
.filter(text -> !text.isEmpty())
.collect(Collectors.joining("\t"));
if (rowText.isEmpty()) {
continue;
}
textWriter.write(indent);
textWriter.write(rowText);
textWriter.write(lineSeparator);
}
}
private String collectPlainText(List<IObject> contents) {
StringBuilder builder = new StringBuilder();
for (IObject content : contents) {
String piece = extractPlainText(content);
if (piece.isEmpty()) {
continue;
}
if (builder.length() > 0) {
builder.append(' ');
}
builder.append(piece);
}
return builder.toString();
}
private String extractPlainText(IObject content) {
if (content instanceof SemanticHeaderOrFooter) {
if (includeHeaderFooter) {
return collectPlainText(((SemanticHeaderOrFooter) content).getContents());
}
return "";
} else if (content instanceof SemanticHeading) {
return sanitize(((SemanticHeading) content).getValue());
} else if (content instanceof SemanticParagraph) {
return sanitize(((SemanticParagraph) content).getValue());
} else if (content instanceof SemanticTextNode) {
return sanitize(((SemanticTextNode) content).getValue());
} else if (content instanceof PDFList) {
PDFList list = (PDFList) content;
return list.getListItems().stream()
.map(item -> compactWhitespace(collectPlainText(item.getContents())))
.filter(text -> !text.isEmpty())
.collect(Collectors.joining(" "));
} else if (content instanceof TableBorder) {
TableBorder table = (TableBorder) content;
return Arrays.stream(table.getRows())
.map(row -> Arrays.stream(row.getCells())
.map(cell -> compactWhitespace(collectPlainText(cell.getContents())))
.filter(text -> !text.isEmpty())
.collect(Collectors.joining(" ")))
.filter(text -> !text.isEmpty())
.collect(Collectors.joining(" "));
}
return "";
}
private void writeMultiline(String value, int indentLevel) throws IOException {
if (value == null) {
return;
}
String sanitized = sanitize(value);
String indent = indent(indentLevel);
String[] lines = sanitized.split("\r?\n", -1);
for (String line : lines) {
if (line.isBlank()) {
continue;
}
textWriter.write(indent);
textWriter.write(line);
textWriter.write(lineSeparator);
}
}
private String indent(int level) {
if (level <= 0) {
return "";
}
return INDENT.repeat(level);
}
private String sanitize(String value) {
return value == null ? "" : value.replace("\u0000", " ");
}
private String compactWhitespace(String value) {
if (value == null) {
return "";
}
String sanitized = sanitize(value);
return sanitized.replaceAll("\\s+", " ").trim();
}
@Override
public void close() throws IOException {
if (textWriter != null) {
textWriter.close();
}
}
}
java/opendataloader-pdf-core/src/main/java/org/opendataloader/pdf/utils/Base64ImageUtils.java
+100
View File
@@ -0,0 +1,100 @@
/*
* Copyright 2025-2026 Hancom Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opendataloader.pdf.utils;
import org.opendataloader.pdf.containers.StaticLayoutContainers;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.util.Base64;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* Utility class for converting images to Base64 data URIs.
*/
public final class Base64ImageUtils {
private static final Logger LOGGER = Logger.getLogger(Base64ImageUtils.class.getCanonicalName());
/**
* Maximum image file size for Base64 embedding (10MB).
* Larger images will be skipped to prevent memory exhaustion.
*/
public static final long MAX_EMBEDDED_IMAGE_SIZE = 10L * 1024 * 1024;
private Base64ImageUtils() {
// Private constructor to prevent instantiation
}
/**
* Converts an image file to a Base64 data URI string.
* Images larger than {@link #MAX_EMBEDDED_IMAGE_SIZE} will be skipped.
*
* @param imageFile The image file to convert
* @param format The image format (png, jpeg)
* @return The Base64 data URI string, or null if conversion fails or image is too large
*/
public static String toDataUri(File imageFile, String format) {
try {
// Embedded mode keeps encoded image bytes in memory keyed by the path
// generators expect — no disk roundtrip in production.
byte[] fileContent = StaticLayoutContainers.getEmbeddedImageBytes(imageFile.getPath());
if (fileContent == null) {
// Disk fallback: runs in external mode (where files exist on disk) and
// also serves unit tests that prepare PNG files directly and bypass
// the in-memory pipeline (see ImageSerializerTest, MarkdownGenerator tests).
long fileSize = imageFile.length();
if (fileSize > MAX_EMBEDDED_IMAGE_SIZE) {
LOGGER.log(Level.WARNING, "Image too large to embed ({0} bytes, max {1} bytes): {2}",
new Object[]{fileSize, MAX_EMBEDDED_IMAGE_SIZE, imageFile.getName()});
return null;
}
fileContent = Files.readAllBytes(imageFile.toPath());
} else if (fileContent.length > MAX_EMBEDDED_IMAGE_SIZE) {
LOGGER.log(Level.WARNING, "Image too large to embed ({0} bytes, max {1} bytes): {2}",
new Object[]{(long) fileContent.length, MAX_EMBEDDED_IMAGE_SIZE, imageFile.getName()});
return null;
}
String base64 = Base64.getEncoder().encodeToString(fileContent);
String mimeType = getMimeType(format);
return String.format("data:%s;base64,%s", mimeType, base64);
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Unable to convert image to Base64: " + e.getMessage());
return null;
}
}
/**
* Gets the MIME type for the given image format.
*
* @param format The image format (png, jpeg)
* @return The corresponding MIME type
*/
public static String getMimeType(String format) {
if (format == null) {
return "image/png";
}
switch (format.toLowerCase()) {
case "jpeg":
case "jpg":
return "image/jpeg";
case "png":
default:
return "image/png";
}
}
}

Some files were not shown because too many files have changed in this diff Show More