#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ PDF Parser - Extract text and sections from PDF files. Features: - Extract text from PDF - Parse IMRaD sections (Introduction, Methods, Results, Discussion) - Extract metadata (title, authors, abstract) - Summarize content Requirements: pip install pymupdf Usage: python pdf_parser.py paper.pdf --output text.txt python pdf_parser.py paper.pdf --sections --json output.json python pdf_parser.py paper.pdf --summarize """ import argparse import json import re import sys from pathlib import Path try: import fitz # PyMuPDF except ImportError: print("Error: PyMuPDF required. Install with: pip install pymupdf") sys.exit(1) # Section patterns for academic papers SECTION_PATTERNS = { "abstract": [ r"^abstract\s*$", r"^摘要\s*$", r"^abstract[.:]", ], "introduction": [ r"^1\.?\s*introduction\s*$", r"^introduction\s*$", r"^一、引言\s*$", r"^1\s+引言\s*$", ], "methods": [ r"^2\.?\s*methods?\s*$", r"^methodology\s*$", r"^materials\s+and\s+methods\s*$", r"^二、方法\s*$", r"^2\s+方法\s*$", ], "results": [ r"^3\.?\s*results?\s*$", r"^findings\s*$", r"^三、结果\s*$", r"^3\s+结果\s*$", ], "discussion": [ r"^4\.?\s*discussion\s*$", r"^conclusions?\s*$", r"^四、讨论\s*$", r"^4\s+讨论\s*$", r"^五、结论\s*$", ], "references": [ r"^references?\s*$", r"^bibliography\s*$", r"^参考文献\s*$", ], } def extract_text(pdf_path: str) -> str: """Extract all text from PDF.""" doc = fitz.open(pdf_path) text_parts = [] for page in doc: text_parts.append(page.get_text()) doc.close() return "\n\n".join(text_parts) def extract_metadata(pdf_path: str) -> dict: """Extract metadata from PDF.""" doc = fitz.open(pdf_path) meta = doc.metadata metadata = { "title": meta.get("title", ""), "author": meta.get("author", ""), "subject": meta.get("subject", ""), "keywords": meta.get("keywords", ""), "pages": len(doc), "creator": meta.get("creator", ""), "producer": meta.get("producer", ""), } doc.close() return metadata def extract_abstract(text: str) -> str: """Extract abstract from paper text.""" lines = text.split("\n") abstract_start = None abstract_end = None for i, line in enumerate(lines): line_lower = line.strip().lower() # Find abstract start if abstract_start is None: for pattern in SECTION_PATTERNS["abstract"]: if re.match(pattern, line_lower, re.IGNORECASE): abstract_start = i + 1 break # Find abstract end (next section) elif abstract_end is None: for section in ["introduction", "methods"]: for pattern in SECTION_PATTERNS.get(section, []): if re.match(pattern, line_lower, re.IGNORECASE): abstract_end = i break if abstract_end: break if abstract_start is not None: if abstract_end is None: abstract_end = min(abstract_start + 50, len(lines)) abstract = "\n".join(lines[abstract_start:abstract_end]) # Clean up abstract = re.sub(r"\s+", " ", abstract).strip() return abstract[:2000] # Limit length return "" def parse_sections(text: str) -> dict: """Parse paper into IMRaD sections.""" lines = text.split("\n") sections = {} current_section = "other" current_content = [] for line in lines: line_stripped = line.strip() line_lower = line_stripped.lower() # Check if this line is a section header found_section = None for section_name, patterns in SECTION_PATTERNS.items(): for pattern in patterns: if re.match(pattern, line_lower, re.IGNORECASE): found_section = section_name break if found_section: break if found_section: # Save previous section if current_content: sections[current_section] = "\n".join(current_content).strip() current_section = found_section current_content = [] else: current_content.append(line) # Save last section if current_content: sections[current_section] = "\n".join(current_content).strip() return sections def summarize_text(text: str, max_length: int = 500) -> str: """Simple extractive summarization.""" # Split into sentences sentences = re.split(r'[。.!?]\s*', text) if len(sentences) <= 5: return text[:max_length] # Take first 3 and last 2 sentences selected = sentences[:3] + sentences[-2:] summary = ". ".join(selected) return summary[:max_length] def main(): parser = argparse.ArgumentParser( description="Extract text and sections from PDF files", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Extract all text python pdf_parser.py paper.pdf --output text.txt # Extract sections as JSON python pdf_parser.py paper.pdf --sections --json output.json # Extract metadata and abstract python pdf_parser.py paper.pdf --metadata --abstract # Summarize content python pdf_parser.py paper.pdf --summarize """ ) parser.add_argument("pdf_path", help="Path to PDF file") parser.add_argument("--output", "-o", help="Output file path") parser.add_argument("--json", help="Output as JSON to file") parser.add_argument("--sections", action="store_true", help="Parse IMRaD sections") parser.add_argument("--metadata", action="store_true", help="Extract metadata") parser.add_argument("--abstract", action="store_true", help="Extract abstract") parser.add_argument("--summarize", action="store_true", help="Summarize content") parser.add_argument("--all", action="store_true", help="Extract everything") args = parser.parse_args() if not Path(args.pdf_path).exists(): print(f"Error: File not found: {args.pdf_path}", file=sys.stderr) sys.exit(1) # Default to all extraction if no specific options if not any([args.sections, args.metadata, args.abstract, args.summarize, args.all]): args.all = True result = {} # Extract text print(f"Processing: {args.pdf_path}", file=sys.stderr) text = extract_text(args.pdf_path) if args.all or args.metadata: result["metadata"] = extract_metadata(args.pdf_path) if args.all or args.abstract: result["abstract"] = extract_abstract(text) if args.all or args.sections: result["sections"] = parse_sections(text) if args.all or args.summarize: result["summary"] = summarize_text(text) # Always include full text result["text"] = text[:10000] # Limit for practical use result["text_length"] = len(text) # Output if args.json: with open(args.json, "w", encoding="utf-8") as f: json.dump(result, f, indent=2, ensure_ascii=False) print(f"Saved to {args.json}", file=sys.stderr) elif args.output: with open(args.output, "w", encoding="utf-8") as f: f.write(text) print(f"Saved to {args.output}", file=sys.stderr) else: print(json.dumps(result, indent=2, ensure_ascii=False)) if __name__ == "__main__": main()