chore: import upstream snapshot with attribution
CI - Python Bindings / sdist (push) Failing after 1s
CI - Python Bindings / Build x86_64-unknown-linux-musl (push) Failing after 1s
CI / fmt (push) Failing after 1s
E2E Output Validation / compare-outputs (push) Failing after 1s
Sync Docs to Developer Hub / sync-docs (push) Failing after 1s
CI - Python Bindings / Build x86_64-unknown-linux-gnu (push) Failing after 1s
CI - WASM Bindings / Build WASM (push) Failing after 0s
CI / clippy (push) Failing after 1s
CI / build-and-test (ubuntu-latest) (push) Failing after 0s
CI - WASM Bindings / Edge runtime PDF parse test (push) Has been skipped
CI - WASM Bindings / Browser PDF parse test (push) Has been skipped
Deploy Demo to GitHub Pages / deploy (push) Failing after 1s
CI / build-docker-image (push) Failing after 3s
CI - Node Bindings / Build darwin-x64 (push) Has been cancelled
CI - Node Bindings / Test win32-x64-msvc (push) Has been cancelled
CI - Node Bindings / Build linux-arm64-gnu (push) Has been cancelled
CI - Node Bindings / Build linux-x64-gnu (push) Has been cancelled
CI - Node Bindings / Build linux-x64-musl (push) Has been cancelled
CI - Node Bindings / Build win32-arm64-msvc (push) Has been cancelled
CI - Node Bindings / Build win32-x64-msvc (push) Has been cancelled
CI - Node Bindings / Test darwin-arm64 (push) Has been cancelled
CI - Node Bindings / Test darwin-x64 (push) Has been cancelled
CI - Node Bindings / Test linux-x64-gnu (push) Has been cancelled
CI - Node Bindings / Test linux-x64-musl (push) Has been cancelled
CI - Node Bindings / Test win32-arm64-msvc (push) Has been cancelled
CI - Python Bindings / Build aarch64-pc-windows-msvc (push) Has been cancelled
CI - Python Bindings / Build x86_64-pc-windows-msvc (push) Has been cancelled
CI - Python Bindings / Build x86_64-apple-darwin (push) Has been cancelled
CI - Python Bindings / Build aarch64-apple-darwin (push) Has been cancelled
CI - Python Bindings / Build aarch64-unknown-linux-gnu (push) Has been cancelled
CI - Node Bindings / Build darwin-arm64 (push) Has been cancelled
CI - Python Bindings / Test x86_64-apple-darwin (push) Has been cancelled
CI - Python Bindings / Test aarch64-apple-darwin (push) Has been cancelled
CI - Python Bindings / Test x86_64-unknown-linux-gnu (push) Has been cancelled
CI - Python Bindings / Test x86_64-unknown-linux-musl (push) Has been cancelled
CI - Python Bindings / Test aarch64-pc-windows-msvc (push) Has been cancelled
CI - Python Bindings / Test x86_64-pc-windows-msvc (push) Has been cancelled
CI / build-and-test (macos-26-intel) (push) Has been cancelled
CI / build-and-test (macos-latest) (push) Has been cancelled
CI / build-and-test (windows-11-arm) (push) Has been cancelled
CI / build-and-test (windows-latest) (push) Has been cancelled
E2E Output Validation / upload-dataset (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:23:44 +08:00
commit 9f97f3abbe
209 changed files with 57998 additions and 0 deletions
+440
View File
@@ -0,0 +1,440 @@
use clap::{Args, Parser, Subcommand};
use liteparse::config::{ImageMode, LiteParseConfig, OutputFormat};
use liteparse::conversion;
use liteparse::output::{json, text};
use liteparse::parser::LiteParse;
#[derive(Parser, Debug)]
#[command(
name = "lit",
version,
about = "OSS document parsing tool (supports PDF, DOCX, XLSX, images, and more)"
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand, Debug)]
enum Commands {
/// Parse a document file (PDF, DOCX, XLSX, PPTX, images, etc.)
Parse(ParseCommand),
/// Generate screenshots of document pages (PDF, DOCX, XLSX, images, etc.)
Screenshot(ScreenshotCommand),
/// Parse multiple documents in batch mode
BatchParse(BatchParseCommand),
}
#[derive(Args, Debug)]
struct ParseCommand {
file: String,
#[arg(short, long)]
output: Option<String>,
#[arg(long, default_value = "text")]
format: String,
#[arg(long)]
no_ocr: bool,
#[arg(long, default_value = "eng")]
ocr_language: String,
#[arg(long, default_value = None)]
ocr_server_url: Option<String>,
/// Extra header for OCR server requests, "Name: Value" (repeatable).
/// e.g. --ocr-server-header "Authorization: Bearer <token>"
#[arg(long = "ocr-server-header", value_parser = parse_header)]
ocr_server_headers: Vec<(String, String)>,
#[arg(long)]
tessdata_path: Option<String>,
#[arg(long, default_value = "1000")]
max_pages: usize,
#[arg(long)]
target_pages: Option<String>,
#[arg(long, default_value = "150")]
dpi: f32,
#[arg(long)]
preserve_small_text: bool,
#[arg(long)]
password: Option<String>,
#[arg(short, long)]
quiet: bool,
#[arg(long)]
num_workers: Option<usize>,
/// How to surface raster images in markdown output: `off`, `placeholder`
/// (default), or `embed` (extracts PNG bytes, written next to the output
/// when `--image-output-dir` is set).
#[arg(long, default_value = "placeholder")]
image_mode: String,
/// Directory to write embedded images to when `--image-mode embed` is set.
/// Each image is written as `image_{id}.png` to match the markdown
/// references. Created if missing.
#[arg(long)]
image_output_dir: Option<String>,
/// Disable hyperlink extraction. By default URI link annotations render as
/// `[text](url)` in markdown output; pass this to emit plain anchor text.
#[arg(long)]
no_links: bool,
}
#[derive(Args, Debug)]
struct ScreenshotCommand {
file: String,
#[arg(short, long, default_value = "./screenshots")]
output_dir: String,
#[arg(long)]
target_pages: Option<String>,
#[arg(long, default_value = "150")]
dpi: f32,
#[arg(long)]
password: Option<String>,
#[arg(short, long)]
quiet: bool,
}
#[derive(Args, Debug)]
struct BatchParseCommand {
input_dir: String,
output_dir: String,
#[arg(long, default_value = "text")]
format: String,
#[arg(long)]
no_ocr: bool,
#[arg(long, default_value = "eng")]
ocr_language: String,
#[arg(long, default_value = None)]
ocr_server_url: Option<String>,
/// Extra header for OCR server requests, "Name: Value" (repeatable).
/// e.g. --ocr-server-header "Authorization: Bearer <token>"
#[arg(long = "ocr-server-header", value_parser = parse_header)]
ocr_server_headers: Vec<(String, String)>,
#[arg(long)]
tessdata_path: Option<String>,
#[arg(long, default_value = "1000")]
max_pages: usize,
#[arg(long, default_value = "150")]
dpi: f32,
#[arg(long)]
recursive: bool,
#[arg(long)]
extension: Option<String>,
#[arg(long)]
password: Option<String>,
#[arg(short, long)]
quiet: bool,
#[arg(long)]
num_workers: Option<usize>,
}
/// Parse a `Name: Value` header string into a `(name, value)` pair.
fn parse_header(s: &str) -> Result<(String, String), String> {
let (name, value) = s
.split_once(':')
.ok_or_else(|| format!("invalid header '{}', expected 'Name: Value'", s))?;
let name = name.trim();
if name.is_empty() {
return Err(format!("invalid header '{}', empty header name", s));
}
Ok((name.to_string(), value.trim().to_string()))
}
fn parse_output_format(s: &str) -> Result<OutputFormat, String> {
match s.to_lowercase().as_str() {
"json" => Ok(OutputFormat::Json),
"text" => Ok(OutputFormat::Text),
"markdown" | "md" => Ok(OutputFormat::Markdown),
_ => Err(format!(
"unknown format '{}', expected 'json', 'text', or 'markdown'",
s
)),
}
}
fn parse_image_mode(s: &str) -> Result<ImageMode, String> {
match s.to_lowercase().as_str() {
"off" | "none" => Ok(ImageMode::Off),
"placeholder" => Ok(ImageMode::Placeholder),
"embed" => Ok(ImageMode::Embed),
_ => Err(format!(
"unknown image-mode '{}', expected 'off', 'placeholder', or 'embed'",
s
)),
}
}
/// Run the CLI with the given args (typically from sys.argv).
pub fn run_cli(args: Vec<String>) -> Result<(), Box<dyn std::error::Error>> {
let cli = Cli::parse_from(args);
let rt = tokio::runtime::Runtime::new()?;
match cli.command {
Commands::Parse(cmd) => {
let format = parse_output_format(&cmd.format)?;
let image_mode = parse_image_mode(&cmd.image_mode)?;
let mut config = LiteParseConfig {
ocr_language: cmd.ocr_language,
ocr_enabled: !cmd.no_ocr,
tessdata_path: cmd.tessdata_path,
max_pages: cmd.max_pages,
target_pages: cmd.target_pages,
dpi: cmd.dpi,
output_format: format,
preserve_very_small_text: cmd.preserve_small_text,
password: cmd.password,
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
image_mode,
extract_links: !cmd.no_links,
..Default::default()
};
if let Some(n) = cmd.num_workers {
config.num_workers = n;
}
let lp = LiteParse::new(config);
let result = rt.block_on(lp.parse(&cmd.file))?;
let formatted = match lp.config().output_format {
OutputFormat::Json => json::format_json(&result.pages)?,
OutputFormat::Text => text::format_text(&result.pages),
OutputFormat::Markdown => result.text.clone(),
};
if let Some(dir) = cmd.image_output_dir.as_deref()
&& !result.images.is_empty()
{
std::fs::create_dir_all(dir)?;
for img in &result.images {
let path = format!("{}/image_{}.{}", dir, img.id, img.format);
std::fs::write(&path, &img.bytes)?;
}
if !cmd.quiet {
eprintln!(
"[liteparse] wrote {} image(s) to {}",
result.images.len(),
dir
);
}
}
match cmd.output {
Some(path) => {
std::fs::write(&path, &formatted)?;
if !cmd.quiet {
eprintln!("[liteparse] wrote output to {}", path);
}
}
None => println!("{}", formatted),
}
}
Commands::Screenshot(cmd) => {
let target_pages = cmd
.target_pages
.as_ref()
.map(|s| liteparse::config::parse_target_pages(s))
.transpose()
.map_err(|e| format!("invalid --target-pages: {}", e))?;
std::fs::create_dir_all(&cmd.output_dir)?;
let config = LiteParseConfig {
target_pages: cmd.target_pages.clone(),
dpi: cmd.dpi,
password: cmd.password.clone(),
quiet: cmd.quiet,
..Default::default()
};
let lp = LiteParse::new(config);
let results = rt.block_on(lp.screenshot(&cmd.file, target_pages))?;
for result in results {
let output_path = format!("{}/page_{}.png", cmd.output_dir, result.page_num);
std::fs::write(&output_path, &result.image_bytes)?;
if !cmd.quiet {
eprintln!(
"[liteparse] screenshot page {}{}",
result.page_num, output_path
);
}
}
}
Commands::BatchParse(cmd) => {
let format = parse_output_format(&cmd.format)?;
let ext_filter = cmd.extension.as_ref().map(|e| {
let e = e.to_lowercase();
if e.starts_with('.') {
e
} else {
format!(".{}", e)
}
});
let mut config = LiteParseConfig {
ocr_language: cmd.ocr_language,
ocr_enabled: !cmd.no_ocr,
tessdata_path: cmd.tessdata_path,
max_pages: cmd.max_pages,
target_pages: None,
dpi: cmd.dpi,
output_format: format.clone(),
preserve_very_small_text: false,
password: cmd.password,
quiet: cmd.quiet,
ocr_server_url: cmd.ocr_server_url,
ocr_server_headers: cmd.ocr_server_headers,
..Default::default()
};
if let Some(n) = cmd.num_workers {
config.num_workers = n;
}
let lp = LiteParse::new(config);
let out_ext = match format {
OutputFormat::Json => "json",
OutputFormat::Markdown => "md",
OutputFormat::Text => "txt",
};
std::fs::create_dir_all(&cmd.output_dir)?;
let files = collect_files(&cmd.input_dir, cmd.recursive, ext_filter.as_deref())?;
if files.is_empty() {
eprintln!("[liteparse] no matching files found in {}", cmd.input_dir);
return Ok(());
}
if !cmd.quiet {
eprintln!("[liteparse] found {} files to process", files.len());
}
let mut success = 0usize;
let mut errors = 0usize;
for file_path in &files {
let t0 = std::time::Instant::now();
let out_path =
batch_output_path(file_path, &cmd.input_dir, &cmd.output_dir, out_ext);
if let Some(parent) = out_path.parent() {
std::fs::create_dir_all(parent)?;
}
match rt.block_on(lp.parse(file_path)) {
Ok(result) => {
let fmt_result: Result<String, Box<dyn std::error::Error>> =
match lp.config().output_format {
OutputFormat::Json => {
json::format_json(&result.pages).map_err(|e| e.into())
}
OutputFormat::Text => Ok(text::format_text(&result.pages)),
OutputFormat::Markdown => Ok(result.text.clone()),
};
match fmt_result {
Ok(formatted) => {
std::fs::write(&out_path, &formatted)?;
success += 1;
if !cmd.quiet {
let elapsed = t0.elapsed().as_secs_f64() * 1000.0;
eprintln!(
"[liteparse] {}{} ({:.1}ms)",
file_path,
out_path.display(),
elapsed
);
}
}
Err(e) => {
eprintln!("[liteparse] error formatting {}: {}", file_path, e);
errors += 1;
}
}
}
Err(e) => {
eprintln!("[liteparse] error parsing {}: {}", file_path, e);
errors += 1;
}
}
}
eprintln!(
"[liteparse] batch complete: {} succeeded, {} failed",
success, errors
);
if errors > 0 {
std::process::exit(1);
}
}
}
Ok(())
}
fn collect_files(
dir: &str,
recursive: bool,
ext_filter: Option<&str>,
) -> Result<Vec<String>, Box<dyn std::error::Error>> {
let mut files = Vec::new();
collect_files_inner(std::path::Path::new(dir), recursive, ext_filter, &mut files)?;
files.sort();
Ok(files)
}
fn batch_output_path(
file_path: &str,
input_dir: &str,
output_dir: &str,
out_ext: &str,
) -> std::path::PathBuf {
let file_path = std::path::Path::new(file_path);
let rel = file_path
.strip_prefix(std::path::Path::new(input_dir))
.unwrap_or(file_path);
std::path::Path::new(output_dir)
.join(rel)
.with_extension(out_ext)
}
#[cfg(test)]
mod tests {
use super::batch_output_path;
use std::path::Path;
#[test]
fn batch_output_path_preserves_output_dir_without_trailing_slash() {
let out_path = batch_output_path("docs/report.pdf", "docs", "out", "txt");
assert_eq!(out_path, Path::new("out/report.txt"));
}
#[test]
fn batch_output_path_mirrors_nested_files_without_trailing_slash() {
let out_path = batch_output_path("docs/nested/report.pdf", "docs", "out", "md");
assert_eq!(out_path, Path::new("out/nested/report.md"));
}
}
fn collect_files_inner(
dir: &std::path::Path,
recursive: bool,
ext_filter: Option<&str>,
files: &mut Vec<String>,
) -> Result<(), Box<dyn std::error::Error>> {
for entry in std::fs::read_dir(dir)? {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
if recursive {
collect_files_inner(&path, recursive, ext_filter, files)?;
}
continue;
}
let path_str = path.to_string_lossy().to_string();
if let Some(filter) = ext_filter {
if !path_str.to_lowercase().ends_with(filter) {
continue;
}
} else if !conversion::is_supported_extension(&path_str) {
continue;
}
files.push(path_str);
}
Ok(())
}
+694
View File
@@ -0,0 +1,694 @@
use std::collections::HashMap;
use pyo3::prelude::*;
use pyo3::types::PyBytes;
use liteparse::config::{CropBox, ImageMode, LiteParseConfig, OutputFormat};
use liteparse::types::PdfInput;
mod cli;
// ---------------------------------------------------------------------------
// Python type wrappers
// ---------------------------------------------------------------------------
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyWordBox {
#[pyo3(get)]
text: String,
#[pyo3(get)]
x: f64,
#[pyo3(get)]
y: f64,
#[pyo3(get)]
width: f64,
#[pyo3(get)]
height: f64,
}
#[pymethods]
impl PyWordBox {
fn __repr__(&self) -> String {
format!(
"WordBox(text={:?}, x={}, y={}, width={}, height={})",
self.text, self.x, self.y, self.width, self.height
)
}
}
impl PyWordBox {
fn from_rust(word: liteparse::types::WordBox) -> Self {
Self {
text: word.text,
x: word.x as f64,
y: word.y as f64,
width: word.width as f64,
height: word.height as f64,
}
}
}
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyTextItem {
#[pyo3(get)]
text: String,
#[pyo3(get)]
x: f64,
#[pyo3(get)]
y: f64,
#[pyo3(get)]
width: f64,
#[pyo3(get)]
height: f64,
#[pyo3(get)]
font_name: Option<String>,
#[pyo3(get)]
font_size: Option<f64>,
#[pyo3(get)]
confidence: Option<f64>,
/// Rotation in degrees (viewport space). Defaults to 0.
#[pyo3(get)]
rotation: f64,
/// Per-word sub-boxes for attribution. Empty unless the parse was
/// configured with `emit_word_boxes=True`.
#[pyo3(get)]
words: Vec<PyWordBox>,
}
#[pymethods]
impl PyTextItem {
fn __repr__(&self) -> String {
format!(
"TextItem(text={:?}, x={}, y={}, width={}, height={})",
self.text, self.x, self.y, self.width, self.height
)
}
}
impl PyTextItem {
fn to_rust(&self) -> liteparse::types::TextItem {
liteparse::types::TextItem {
text: self.text.clone(),
x: self.x as f32,
y: self.y as f32,
width: self.width as f32,
height: self.height as f32,
rotation: self.rotation as f32,
font_name: self.font_name.clone(),
font_size: self.font_size.map(|v| v as f32),
confidence: self.confidence.map(|v| v as f32),
..Default::default()
}
}
fn from_rust(item: liteparse::types::TextItem) -> Self {
Self {
text: item.text,
x: item.x as f64,
y: item.y as f64,
width: item.width as f64,
height: item.height as f64,
font_name: item.font_name,
font_size: item.font_size.map(|v| v as f64),
confidence: item.confidence.map(|v| v as f64).or(Some(1.0)),
rotation: item.rotation as f64,
words: item.words.into_iter().map(PyWordBox::from_rust).collect(),
}
}
}
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyParsedPage {
#[pyo3(get)]
page_num: u32,
#[pyo3(get)]
width: f64,
#[pyo3(get)]
height: f64,
#[pyo3(get)]
text: String,
#[pyo3(get)]
markdown: String,
#[pyo3(get)]
text_items: Vec<PyTextItem>,
}
#[pymethods]
impl PyParsedPage {
fn __repr__(&self) -> String {
format!(
"ParsedPage(page_num={}, width={}, height={}, text_items={})",
self.page_num,
self.width,
self.height,
self.text_items.len()
)
}
}
impl PyParsedPage {
fn from_rust(page: liteparse::types::ParsedPage) -> Self {
Self {
page_num: page.page_number as u32,
width: page.page_width as f64,
height: page.page_height as f64,
text: page.text,
markdown: page.markdown,
text_items: page
.text_items
.into_iter()
.map(PyTextItem::from_rust)
.collect(),
}
}
}
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyParseResult {
#[pyo3(get)]
pages: Vec<PyParsedPage>,
#[pyo3(get)]
text: String,
#[pyo3(get)]
images: Vec<PyExtractedImage>,
}
#[pymethods]
impl PyParseResult {
#[getter]
fn num_pages(&self) -> usize {
self.pages.len()
}
fn get_page(&self, page_num: u32) -> Option<PyParsedPage> {
self.pages.iter().find(|p| p.page_num == page_num).cloned()
}
fn __repr__(&self) -> String {
format!(
"ParseResult(pages={}, text_len={}, images={})",
self.pages.len(),
self.text.len(),
self.images.len()
)
}
}
impl PyParseResult {
fn from_rust(result: liteparse::parser::ParseResult) -> Self {
Self {
pages: result
.pages
.into_iter()
.map(PyParsedPage::from_rust)
.collect(),
text: result.text,
images: result
.images
.into_iter()
.map(PyExtractedImage::from_rust)
.collect(),
}
}
}
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyExtractedImage {
#[pyo3(get)]
id: String,
#[pyo3(get)]
page: u32,
#[pyo3(get)]
format: String,
bytes_buffer: Vec<u8>,
}
#[pymethods]
impl PyExtractedImage {
#[getter]
fn bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
PyBytes::new(py, &self.bytes_buffer)
}
fn __repr__(&self) -> String {
format!(
"ExtractedImage(id='{}', page={}, format='{}', bytes_len={})",
self.id,
self.page,
self.format,
self.bytes_buffer.len()
)
}
}
impl PyExtractedImage {
fn from_rust(img: liteparse::types::ExtractedImage) -> Self {
Self {
id: img.id,
page: img.page,
format: img.format,
bytes_buffer: img.bytes,
}
}
}
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyScreenshotResult {
#[pyo3(get)]
page_num: u32,
#[pyo3(get)]
width: u32,
#[pyo3(get)]
height: u32,
image_buffer: Vec<u8>,
}
#[pymethods]
impl PyScreenshotResult {
#[getter]
fn image_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
PyBytes::new(py, &self.image_buffer)
}
fn __repr__(&self) -> String {
format!(
"ScreenshotResult(page_num={}, width={}, height={})",
self.page_num, self.width, self.height
)
}
}
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyPageComplexityStats {
#[pyo3(get)]
page_number: usize,
#[pyo3(get)]
text_length: usize,
#[pyo3(get)]
text_coverage: f32,
#[pyo3(get)]
has_substantial_images: bool,
#[pyo3(get)]
image_block_count: usize,
#[pyo3(get)]
image_coverage: f32,
#[pyo3(get)]
largest_image_coverage: f32,
#[pyo3(get)]
full_page_image: bool,
#[pyo3(get)]
uncovered_vector_area: Option<f32>,
#[pyo3(get)]
is_garbled: bool,
#[pyo3(get)]
page_area: f32,
#[pyo3(get)]
needs_ocr: bool,
#[pyo3(get)]
reasons: Vec<String>,
}
#[pymethods]
impl PyPageComplexityStats {
fn __repr__(&self) -> String {
format!(
"PageComplexityStats(page_number={}, text_length={}, text_coverage={:.2}, needs_ocr={})",
self.page_number, self.text_length, self.text_coverage, self.needs_ocr
)
}
}
impl PyPageComplexityStats {
fn from_rust(stats: &liteparse::ocr_merge::PageComplexityStats) -> Self {
Self {
page_number: stats.page_number,
text_length: stats.text_length,
text_coverage: stats.text_coverage,
has_substantial_images: stats.has_substantial_images,
image_block_count: stats.image_block_count,
image_coverage: stats.image_coverage,
largest_image_coverage: stats.largest_image_coverage,
full_page_image: stats.full_page_image,
uncovered_vector_area: stats.uncovered_vector_area,
is_garbled: stats.is_garbled,
page_area: stats.page_area,
needs_ocr: stats.needs_ocr,
reasons: stats
.reasons
.iter()
.map(|r| r.as_str().to_string())
.collect(),
}
}
}
// ---------------------------------------------------------------------------
// Config
// ---------------------------------------------------------------------------
#[pyclass(frozen, from_py_object)]
#[derive(Clone)]
struct PyLiteParseConfig {
#[pyo3(get)]
ocr_language: String,
#[pyo3(get)]
ocr_enabled: bool,
#[pyo3(get)]
ocr_server_url: Option<String>,
#[pyo3(get)]
ocr_server_headers: Option<HashMap<String, String>>,
#[pyo3(get)]
tessdata_path: Option<String>,
#[pyo3(get)]
max_pages: usize,
#[pyo3(get)]
target_pages: Option<String>,
#[pyo3(get)]
dpi: f32,
#[pyo3(get)]
output_format: String,
#[pyo3(get)]
preserve_very_small_text: bool,
#[pyo3(get)]
password: Option<String>,
#[pyo3(get)]
quiet: bool,
#[pyo3(get)]
num_workers: usize,
}
#[pymethods]
impl PyLiteParseConfig {
fn __repr__(&self) -> String {
format!(
"LiteParseConfig(ocr_enabled={}, dpi={}, max_pages={})",
self.ocr_enabled, self.dpi, self.max_pages
)
}
}
impl PyLiteParseConfig {
fn from_rust(cfg: &LiteParseConfig) -> Self {
Self {
ocr_language: cfg.ocr_language.clone(),
ocr_enabled: cfg.ocr_enabled,
ocr_server_url: cfg.ocr_server_url.clone(),
ocr_server_headers: if cfg.ocr_server_headers.is_empty() {
None
} else {
Some(cfg.ocr_server_headers.iter().cloned().collect())
},
tessdata_path: cfg.tessdata_path.clone(),
max_pages: cfg.max_pages,
target_pages: cfg.target_pages.clone(),
dpi: cfg.dpi,
output_format: match cfg.output_format {
OutputFormat::Json => "json".to_string(),
OutputFormat::Text => "text".to_string(),
OutputFormat::Markdown => "markdown".to_string(),
},
preserve_very_small_text: cfg.preserve_very_small_text,
password: cfg.password.clone(),
quiet: cfg.quiet,
num_workers: cfg.num_workers,
}
}
}
// ---------------------------------------------------------------------------
// Main LiteParse class
// ---------------------------------------------------------------------------
#[pyclass]
struct LiteParse {
inner: liteparse::parser::LiteParse,
config: LiteParseConfig,
runtime: tokio::runtime::Runtime,
}
#[pymethods]
impl LiteParse {
#[new]
#[pyo3(signature = (
*,
ocr_language = None,
ocr_enabled = None,
ocr_server_url = None,
ocr_server_headers = None,
tessdata_path = None,
max_pages = None,
target_pages = None,
dpi = None,
output_format = None,
preserve_very_small_text = None,
password = None,
quiet = None,
num_workers = None,
image_mode = None,
extract_links = None,
ocr_failure_fatal = None,
ocr_hedge_delays_ms = None,
emit_word_boxes = None,
crop_box = None,
skip_diagonal_text = None,
))]
fn new(
ocr_language: Option<String>,
ocr_enabled: Option<bool>,
ocr_server_url: Option<String>,
ocr_server_headers: Option<HashMap<String, String>>,
tessdata_path: Option<String>,
max_pages: Option<usize>,
target_pages: Option<String>,
dpi: Option<f32>,
output_format: Option<String>,
preserve_very_small_text: Option<bool>,
password: Option<String>,
quiet: Option<bool>,
num_workers: Option<usize>,
image_mode: Option<String>,
extract_links: Option<bool>,
ocr_failure_fatal: Option<bool>,
ocr_hedge_delays_ms: Option<Vec<u64>>,
emit_word_boxes: Option<bool>,
crop_box: Option<(f32, f32, f32, f32)>,
skip_diagonal_text: Option<bool>,
) -> PyResult<Self> {
let mut cfg = LiteParseConfig::default();
if let Some(v) = ocr_language {
cfg.ocr_language = v;
}
if let Some(v) = ocr_enabled {
cfg.ocr_enabled = v;
}
if let Some(v) = ocr_server_url {
cfg.ocr_server_url = Some(v);
}
if let Some(v) = ocr_server_headers {
cfg.ocr_server_headers = v.into_iter().collect();
}
if let Some(v) = tessdata_path {
cfg.tessdata_path = Some(v);
}
if let Some(v) = max_pages {
cfg.max_pages = v;
}
if let Some(v) = target_pages {
cfg.target_pages = Some(v);
}
if let Some(v) = dpi {
cfg.dpi = v;
}
if let Some(v) = output_format {
cfg.output_format = match v.as_str() {
"text" => OutputFormat::Text,
"markdown" | "md" => OutputFormat::Markdown,
_ => OutputFormat::Json,
};
}
if let Some(v) = preserve_very_small_text {
cfg.preserve_very_small_text = v;
}
if let Some(v) = password {
cfg.password = Some(v);
}
if let Some(v) = quiet {
cfg.quiet = v;
}
if let Some(v) = num_workers {
cfg.num_workers = v;
}
if let Some(v) = image_mode {
cfg.image_mode = match v.as_str() {
"off" | "none" => ImageMode::Off,
"embed" => ImageMode::Embed,
_ => ImageMode::Placeholder,
};
}
if let Some(v) = extract_links {
cfg.extract_links = v;
}
if let Some(v) = ocr_failure_fatal {
cfg.ocr_failure_fatal = v;
}
if let Some(v) = ocr_hedge_delays_ms {
cfg.ocr_hedge_delays_ms = v;
}
if let Some(v) = emit_word_boxes {
cfg.emit_word_boxes = v;
}
if let Some((top, right, bottom, left)) = crop_box {
cfg.crop_box = Some(CropBox {
top,
right,
bottom,
left,
});
}
if let Some(v) = skip_diagonal_text {
cfg.skip_diagonal_text = v;
}
let inner = liteparse::parser::LiteParse::new(cfg.clone());
let runtime = tokio::runtime::Runtime::new()
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(Self {
inner,
config: cfg,
runtime,
})
}
/// Parse a document from a file path.
fn parse(&self, py: Python<'_>, input: String) -> PyResult<PyParseResult> {
let pdf_input = PdfInput::Path(input);
let result = py
.detach(|| self.runtime.block_on(self.inner.parse_input(pdf_input)))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(PyParseResult::from_rust(result))
}
/// Parse a document from raw bytes.
fn parse_bytes(&self, py: Python<'_>, data: Vec<u8>) -> PyResult<PyParseResult> {
let pdf_input = PdfInput::Bytes(data);
let result = py
.detach(|| self.runtime.block_on(self.inner.parse_input(pdf_input)))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(PyParseResult::from_rust(result))
}
/// Determine per-page complexity for a document at the given path. Returns
/// a list of PageComplexityStats — a cheap pre-OCR check with per-page
/// signals and a `needs_ocr` verdict.
fn is_complex(&self, py: Python<'_>, input: String) -> PyResult<Vec<PyPageComplexityStats>> {
let pdf_input = PdfInput::Path(input);
let stats = py
.detach(|| self.runtime.block_on(self.inner.is_complex(pdf_input)))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(stats.iter().map(PyPageComplexityStats::from_rust).collect())
}
/// Determine per-page complexity for a document from raw bytes.
fn is_complex_bytes(
&self,
py: Python<'_>,
data: Vec<u8>,
) -> PyResult<Vec<PyPageComplexityStats>> {
let pdf_input = PdfInput::Bytes(data);
let stats = py
.detach(|| self.runtime.block_on(self.inner.is_complex(pdf_input)))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(stats.iter().map(PyPageComplexityStats::from_rust).collect())
}
/// Take screenshots of document pages. Returns a list of ScreenshotResult.
///
/// Non-PDF files are automatically converted to PDF before rendering when
/// LibreOffice/ImageMagick are available.
#[pyo3(signature = (input, page_numbers = None))]
fn screenshot(
&self,
py: Python<'_>,
input: String,
page_numbers: Option<Vec<u32>>,
) -> PyResult<Vec<PyScreenshotResult>> {
py.detach(|| {
let results = self
.runtime
.block_on(self.inner.screenshot(&input, page_numbers))
.map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))?;
Ok(results
.into_iter()
.map(|r| PyScreenshotResult {
page_num: r.page_num,
width: r.width,
height: r.height,
image_buffer: r.image_bytes,
})
.collect())
})
}
/// Get the resolved configuration.
#[getter]
fn config(&self) -> PyLiteParseConfig {
PyLiteParseConfig::from_rust(&self.config)
}
fn __repr__(&self) -> String {
format!(
"LiteParse(ocr_enabled={}, dpi={}, max_pages={})",
self.config.ocr_enabled, self.config.dpi, self.config.max_pages
)
}
}
// ---------------------------------------------------------------------------
// Module
// ---------------------------------------------------------------------------
/// Search text items for phrase matches, returning merged items with combined bounding boxes.
#[pyfunction]
#[pyo3(signature = (items, phrase, *, case_sensitive = false))]
fn search_items(items: Vec<PyTextItem>, phrase: String, case_sensitive: bool) -> Vec<PyTextItem> {
let rust_items: Vec<_> = items.iter().map(|i| i.to_rust()).collect();
let options = liteparse::search::SearchOptions {
phrase,
case_sensitive,
};
liteparse::search::search_items(&rust_items, &options)
.into_iter()
.map(PyTextItem::from_rust)
.collect()
}
/// Run the `lit` CLI with the given arguments.
#[pyfunction]
fn run_cli(args: Vec<String>) -> PyResult<()> {
cli::run_cli(args).map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
}
#[pymodule]
fn _liteparse(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<LiteParse>()?;
m.add_class::<PyLiteParseConfig>()?;
m.add_class::<PyParseResult>()?;
m.add_class::<PyExtractedImage>()?;
m.add_class::<PyParsedPage>()?;
m.add_class::<PyTextItem>()?;
m.add_class::<PyWordBox>()?;
m.add_class::<PyScreenshotResult>()?;
m.add_class::<PyPageComplexityStats>()?;
m.add_function(wrap_pyfunction!(run_cli, m)?)?;
m.add_function(wrap_pyfunction!(search_items, m)?)?;
Ok(())
}