wehub-resource-sync
fed8b2eed7
Backend release / release (push) Waiting to run
Bandit Security Scan / bandit_scan (push) Waiting to run
Build and push multi-arch DocsGPT Docker image / build (linux/amd64, ubuntu-latest, amd64) (push) Waiting to run
Build and push multi-arch DocsGPT Docker image / build (linux/arm64, ubuntu-24.04-arm, arm64) (push) Waiting to run
Build and push multi-arch DocsGPT Docker image / manifest (push) Blocked by required conditions
Build and push DocsGPT FE Docker image for development / build (linux/amd64, ubuntu-latest, amd64) (push) Waiting to run
Build and push DocsGPT FE Docker image for development / build (linux/arm64, ubuntu-24.04-arm, arm64) (push) Waiting to run
Build and push DocsGPT FE Docker image for development / manifest (push) Blocked by required conditions
Python linting / ruff (push) Waiting to run
Run python tests with pytest / Run tests and count coverage (3.12) (push) Waiting to run
React Widget Build / build (push) Waiting to run
70 lines
2.0 KiB
Python
70 lines
2.0 KiB
Python
"""Docs parser.
|
|
|
|
Contains parsers for docx, pdf files.
|
|
|
|
"""
|
|
from pathlib import Path
|
|
from typing import Dict
|
|
|
|
from application.parser.file.base_parser import BaseParser
|
|
from application.core.settings import settings
|
|
import requests
|
|
|
|
class PDFParser(BaseParser):
|
|
"""PDF parser."""
|
|
|
|
def _init_parser(self) -> Dict:
|
|
"""Init parser."""
|
|
return {}
|
|
|
|
def parse_file(self, file: Path, errors: str = "ignore") -> str:
|
|
"""Parse file."""
|
|
if settings.PARSE_PDF_AS_IMAGE:
|
|
doc2md_service = "https://llm.arc53.com/doc2md"
|
|
# alternatively you can use local vision capable LLM
|
|
with open(file, "rb") as file_loaded:
|
|
files = {'file': file_loaded}
|
|
response = requests.post(doc2md_service, files=files, timeout=100)
|
|
data = response.json()["markdown"]
|
|
return data
|
|
|
|
try:
|
|
from pypdf import PdfReader
|
|
except ImportError:
|
|
raise ValueError("pypdf is required to read PDF files.")
|
|
text_list = []
|
|
with open(file, "rb") as fp:
|
|
# Create a PDF object
|
|
pdf = PdfReader(fp)
|
|
|
|
# Get the number of pages in the PDF document
|
|
num_pages = len(pdf.pages)
|
|
|
|
# Iterate over every page
|
|
for page_index in range(num_pages):
|
|
# Extract the text from the page
|
|
page = pdf.pages[page_index]
|
|
page_text = page.extract_text()
|
|
text_list.append(page_text)
|
|
text = "\n".join(text_list)
|
|
|
|
return text
|
|
|
|
|
|
class DocxParser(BaseParser):
|
|
"""Docx parser."""
|
|
|
|
def _init_parser(self) -> Dict:
|
|
"""Init parser."""
|
|
return {}
|
|
|
|
def parse_file(self, file: Path, errors: str = "ignore") -> str:
|
|
"""Parse file."""
|
|
try:
|
|
import docx2txt
|
|
except ImportError:
|
|
raise ValueError("docx2txt is required to read Microsoft Word files.")
|
|
|
|
text = docx2txt.process(file)
|
|
|
|
return text |