chore: import upstream snapshot with attribution

application/parser/__init__.py

@@ -0,0 +1 @@
+

application/parser/chunking.py

@@ -0,0 +1,101 @@
+import re
+from typing import List, Tuple
+import logging
+from application.parser.chunking_creator import ChunkerCreator
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+
+class Chunker:
+    """Classic token-window chunker (registered as ``classic_chunk``).
+
+    Strategy dispatch lives in ``ChunkerCreator``; this class is one
+    registered implementation. The ``chunking_strategy`` arg is retained for
+    backward-compatible construction and is not used for dispatch here.
+    """
+
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max_tokens
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+
+
+    def split_document(self, doc: Document) -> List[Document]:
+        split_docs = []
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.encoding.encode(header) if header else []
+        body_tokens = self.encoding.encode(body)
+
+        current_position = 0
+        part_index = 0
+        while current_position < len(body_tokens):
+            end_position = current_position + self.max_tokens - len(header_tokens)
+            chunk_tokens = (header_tokens + body_tokens[current_position:end_position]
+                            if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])
+            chunk_text = self.encoding.decode(chunk_tokens)
+            new_doc = Document(
+                text=chunk_text,
+                doc_id=f"{doc.doc_id}-{part_index}",
+                embedding=doc.embedding,
+                extra_info={**(doc.extra_info or {}), "token_count": len(chunk_tokens)}
+            )
+            split_docs.append(new_doc)
+            current_position = end_position
+            part_index += 1
+            header_tokens = []
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            tokens = self.encoding.encode(doc.text)
+            token_count = len(tokens)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        return self.classic_chunk(documents)
+
+
+ChunkerCreator.register("classic_chunk", Chunker)

application/parser/chunking_creator.py

@@ -0,0 +1,57 @@
+"""String-keyed registry for chunking strategies.
+
+Mirrors ``RetrieverCreator``: features register new strategies (``recursive``,
+``markdown``, ``parent_child``, ...) without touching the dispatch site. The
+classic strategy is registered under ``classic_chunk`` by ``chunking.py``.
+"""
+
+from __future__ import annotations
+
+from typing import Type
+
+
+class ChunkerCreator:
+    chunkers: dict[str, Type] = {}
+    _strategies_loaded: bool = False
+
+    @classmethod
+    def _ensure_builtin(cls) -> None:
+        """Register built-in chunkers if they are not registered yet.
+
+        Self-bootstraps so ``create_chunker`` works regardless of import order:
+        ``application.parser.chunking`` registers ``classic_chunk`` and
+        ``application.parser.chunking_strategies`` registers ``recursive`` /
+        ``markdown`` / ``parent_child``.
+        """
+        if not cls.chunkers:
+            import application.parser.chunking  # noqa: F401  (registers classic_chunk)
+        if not cls._strategies_loaded:
+            cls._strategies_loaded = True
+            import application.parser.chunking_strategies  # noqa: F401
+
+    @classmethod
+    def create_chunker(cls, strategy: str, *args, **kwargs):
+        """Instantiate the chunker registered under ``strategy``.
+
+        Args:
+            strategy: Registry key (e.g. ``classic_chunk``).
+            *args: Positional args forwarded to the chunker constructor.
+            **kwargs: Keyword args forwarded to the chunker constructor.
+
+        Returns:
+            A chunker instance exposing ``chunk(documents) -> List[Document]``.
+
+        Raises:
+            ValueError: If no chunker is registered for ``strategy``.
+        """
+        cls._ensure_builtin()
+        key = (strategy or "classic_chunk").lower()
+        chunker_class = cls.chunkers.get(key)
+        if not chunker_class:
+            raise ValueError(f"No chunker class found for strategy {strategy}")
+        return chunker_class(*args, **kwargs)
+
+    @classmethod
+    def register(cls, key: str, chunker_class: Type) -> None:
+        """Register ``chunker_class`` under ``key`` (idempotent)."""
+        cls.chunkers[key] = chunker_class

application/parser/chunking_strategies.py

@@ -0,0 +1,331 @@
+"""Additional chunking strategies registered with ``ChunkerCreator``.
+
+Each strategy honours ``max_tokens`` / ``min_tokens`` and reuses the classic
+``Chunker``'s tiktoken encoding for token counting, so token budgets stay
+consistent across strategies. Selecting a strategy is ingest-time only;
+changing it requires a re-ingest (D8). Registered keys: ``recursive``,
+``markdown``, ``parent_child``, ``semantic``.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import List
+
+from application.parser.chunking import Chunker
+from application.parser.chunking_creator import ChunkerCreator
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+
+class _BaseStrategyChunker:
+    """Shared token helpers for strategy chunkers.
+
+    Mirrors the classic ``Chunker`` constructor so the worker can build any
+    strategy with the same kwargs. ``chunking_strategy`` is accepted for
+    construction compatibility and not used for dispatch (dispatch lives in
+    ``ChunkerCreator``).
+    """
+
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max(1, int(max_tokens))
+        self.min_tokens = max(0, int(min_tokens))
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def _token_count(self, text: str) -> int:
+        return len(self.encoding.encode(text))
+
+    def _split_by_tokens(self, text: str) -> List[str]:
+        """Split ``text`` into pieces no larger than ``max_tokens`` tokens."""
+        tokens = self.encoding.encode(text)
+        pieces = []
+        for start in range(0, len(tokens), self.max_tokens):
+            chunk_tokens = tokens[start:start + self.max_tokens]
+            pieces.append(self.encoding.decode(chunk_tokens))
+        return pieces
+
+    def _emit(self, base: Document, part_index: int, text: str) -> Document:
+        """Build a child Document carrying token_count and inherited info."""
+        return Document(
+            text=text,
+            doc_id=f"{base.doc_id}-{part_index}" if base.doc_id else None,
+            embedding=base.embedding,
+            extra_info={
+                **(base.extra_info or {}),
+                "token_count": self._token_count(text),
+            },
+        )
+
+    def _merge_to_min(self, pieces: List[str], joiner: str) -> List[str]:
+        """Accumulate pieces up to ``max_tokens``, flushing past ``min_tokens``."""
+        merged: List[str] = []
+        buffer = ""
+        for piece in pieces:
+            candidate = f"{buffer}{joiner}{piece}" if buffer else piece
+            if self._token_count(candidate) <= self.max_tokens:
+                buffer = candidate
+            else:
+                if buffer:
+                    merged.append(buffer)
+                buffer = piece
+            if buffer and self._token_count(buffer) >= self.min_tokens:
+                merged.append(buffer)
+                buffer = ""
+        if buffer:
+            merged.append(buffer)
+        return merged
+
+
+class RecursiveChunker(_BaseStrategyChunker):
+    """Split on a separator hierarchy, capping at ``max_tokens``.
+
+    Tries paragraph, line, then sentence boundaries before falling back to a
+    hard token split, and merges adjacent fragments while their combined size
+    stays under ``max_tokens`` so chunks clear ``min_tokens`` where possible.
+    """
+
+    _SEPARATORS = ["\n\n", "\n", ". "]
+
+    def _recursive_split(self, text: str, sep_idx: int) -> List[str]:
+        if self._token_count(text) <= self.max_tokens:
+            return [text] if text.strip() else []
+        if sep_idx >= len(self._SEPARATORS):
+            return [p for p in self._split_by_tokens(text) if p.strip()]
+        sep = self._SEPARATORS[sep_idx]
+        parts = text.split(sep)
+        out: List[str] = []
+        for i, part in enumerate(parts):
+            piece = part + sep if i < len(parts) - 1 else part
+            if not piece.strip():
+                continue
+            if self._token_count(piece) <= self.max_tokens:
+                out.append(piece)
+            else:
+                out.extend(self._recursive_split(piece, sep_idx + 1))
+        return out
+
+    def _merge(self, fragments: List[str]) -> List[str]:
+        """Merge small fragments up to ``max_tokens`` to clear ``min_tokens``."""
+        return self._merge_to_min(fragments, "")
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        for doc in documents:
+            fragments = self._recursive_split(doc.text, 0)
+            for idx, text in enumerate(self._merge(fragments)):
+                processed.append(self._emit(doc, idx, text))
+        return processed
+
+
+class MarkdownChunker(_BaseStrategyChunker):
+    """Split on markdown heading boundaries, then token-cap oversized sections.
+
+    Each ``^#{1,6}\\s`` heading starts a new section; sections over
+    ``max_tokens`` are further split by token window so no chunk exceeds the
+    cap.
+    """
+
+    _HEADING = re.compile(r"^#{1,6}\s", re.MULTILINE)
+
+    def _sections(self, text: str) -> List[str]:
+        boundaries = [m.start() for m in self._HEADING.finditer(text)]
+        if not boundaries:
+            return [text] if text.strip() else []
+        if boundaries[0] != 0:
+            boundaries = [0] + boundaries
+        sections = []
+        for i, start in enumerate(boundaries):
+            end = boundaries[i + 1] if i + 1 < len(boundaries) else len(text)
+            section = text[start:end]
+            if section.strip():
+                sections.append(section)
+        return sections
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        for doc in documents:
+            part_index = 0
+            for section in self._sections(doc.text):
+                if self._token_count(section) <= self.max_tokens:
+                    processed.append(self._emit(doc, part_index, section))
+                    part_index += 1
+                else:
+                    for piece in self._split_by_tokens(section):
+                        if not piece.strip():
+                            continue
+                        processed.append(self._emit(doc, part_index, piece))
+                        part_index += 1
+        return processed
+
+
+class ParentChildChunker(_BaseStrategyChunker):
+    """Emit small child chunks for embedding with a larger parent window.
+
+    The document is first split into parent windows of ``max_tokens`` tokens;
+    each window is then split into children of ``min_tokens`` (a sane floor of
+    50) tokens. Each child stashes its parent window text in
+    ``extra_info["parent_text"]`` so retrieval can expand to the parent later.
+    The child text is what gets embedded; ``parent_text`` rides through
+    ``Document.to_langchain_format`` into vector-store metadata.
+    """
+
+    def _child_size(self) -> int:
+        size = self.min_tokens if self.min_tokens > 0 else 50
+        return min(size, self.max_tokens)
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        child_size = self._child_size()
+        for doc in documents:
+            tokens = self.encoding.encode(doc.text)
+            part_index = 0
+            for p_start in range(0, len(tokens), self.max_tokens):
+                parent_tokens = tokens[p_start:p_start + self.max_tokens]
+                parent_text = self.encoding.decode(parent_tokens)
+                if not parent_text.strip():
+                    continue
+                for c_start in range(0, len(parent_tokens), child_size):
+                    child_tokens = parent_tokens[c_start:c_start + child_size]
+                    child_text = self.encoding.decode(child_tokens)
+                    if not child_text.strip():
+                        continue
+                    child = Document(
+                        text=child_text,
+                        doc_id=(
+                            f"{doc.doc_id}-{part_index}" if doc.doc_id else None
+                        ),
+                        embedding=doc.embedding,
+                        extra_info={
+                            **(doc.extra_info or {}),
+                            "token_count": len(child_tokens),
+                            "parent_text": parent_text,
+                        },
+                    )
+                    processed.append(child)
+                    part_index += 1
+        return processed
+
+
+class SemanticChunker(_BaseStrategyChunker):
+    """Group adjacent sentences by embedding similarity into coherent chunks.
+
+    Sentences are embedded in one batched call and split where the cosine
+    distance between consecutive sentences crosses a high percentile, so
+    chunk boundaries fall at topic shifts. Chunks are then token-capped at
+    ``max_tokens`` and merged up to clear ``min_tokens``. Any failure (too
+    few sentences, embedding error, degenerate distances) falls back to
+    ``RecursiveChunker`` so ingest never crashes.
+    """
+
+    _SENTENCE = re.compile(r"(?<=[.!?])\s+")
+    _PERCENTILE = 95.0
+
+    def _split_sentences(self, text: str) -> List[str]:
+        return [s for s in (p.strip() for p in self._SENTENCE.split(text)) if s]
+
+    def _fallback(self, documents: List[Document]) -> List[Document]:
+        recursive = RecursiveChunker(
+            chunking_strategy=self.chunking_strategy,
+            max_tokens=self.max_tokens,
+            min_tokens=self.min_tokens,
+            duplicate_headers=self.duplicate_headers,
+        )
+        return recursive.chunk(documents)
+
+    def _breakpoints(self, embeddings) -> set:
+        """Indices after which a new chunk starts, from high-distance gaps."""
+        import numpy as np
+
+        matrix = np.asarray(embeddings, dtype=np.float64)
+        if matrix.ndim != 2 or matrix.shape[0] < 2:
+            return set()
+        norms = np.linalg.norm(matrix, axis=1)
+        norms[norms == 0] = 1.0
+        unit = matrix / norms[:, None]
+        sims = np.sum(unit[:-1] * unit[1:], axis=1)
+        distances = 1.0 - sims
+        if not np.all(np.isfinite(distances)):
+            raise ValueError("non-finite semantic distances")
+        if float(distances.max()) <= float(distances.min()):
+            return set()
+        threshold = np.percentile(distances, self._PERCENTILE)
+        return {int(i) for i in np.where(distances >= threshold)[0]}
+
+    def _group(self, sentences: List[str], breakpoints: set) -> List[str]:
+        groups: List[str] = []
+        current: List[str] = []
+        for idx, sentence in enumerate(sentences):
+            current.append(sentence)
+            if idx in breakpoints:
+                groups.append(" ".join(current))
+                current = []
+        if current:
+            groups.append(" ".join(current))
+        return groups
+
+    def _enforce_tokens(self, groups: List[str]) -> List[str]:
+        """Hard-split groups over ``max_tokens`` then merge to clear min."""
+        capped: List[str] = []
+        for group in groups:
+            if self._token_count(group) <= self.max_tokens:
+                capped.append(group)
+            else:
+                capped.extend(p for p in self._split_by_tokens(group) if p.strip())
+        return self._merge_to_min(capped, " ")
+
+    def _chunk_text(self, text: str) -> List[str]:
+        sentences = self._split_sentences(text)
+        if len(sentences) < 2:
+            raise ValueError("too few sentences for semantic chunking")
+        from application.vectorstore.base import EmbeddingsSingleton
+        from application.core.settings import settings
+
+        embeddings = EmbeddingsSingleton.get_instance(
+            settings.EMBEDDINGS_NAME
+        ).embed_documents(sentences)
+        breakpoints = self._breakpoints(embeddings)
+        groups = self._group(sentences, breakpoints)
+        return [g for g in self._enforce_tokens(groups) if g.strip()]
+
+    def chunk(self, documents: List[Document]) -> List[Document]:
+        processed: List[Document] = []
+        for doc in documents:
+            try:
+                texts = self._chunk_text(doc.text)
+            except Exception as exc:
+                logger.warning(
+                    "Semantic chunking failed (%s); falling back to recursive.",
+                    exc,
+                )
+                processed.extend(self._fallback([doc]))
+                continue
+            for idx, text in enumerate(texts):
+                processed.append(self._emit(doc, idx, text))
+        return processed
+
+
+ChunkerCreator.register("recursive", RecursiveChunker)
+ChunkerCreator.register("markdown", MarkdownChunker)
+ChunkerCreator.register("parent_child", ParentChildChunker)
+ChunkerCreator.register("semantic", SemanticChunker)
+
+# Reuse the classic Chunker reference so this module can be the single import
+# that pulls every strategy into the registry.
+__all__ = [
+    "RecursiveChunker",
+    "MarkdownChunker",
+    "ParentChildChunker",
+    "SemanticChunker",
+    "Chunker",
+]

application/parser/connectors/__init__.py

@@ -0,0 +1,18 @@
+"""
+External knowledge base connectors for DocsGPT.
+
+This module contains connectors for external knowledge bases and document storage systems
+that require authentication and specialized handling, separate from simple web scrapers.
+"""
+
+from .base import BaseConnectorAuth, BaseConnectorLoader
+from .connector_creator import ConnectorCreator
+from .google_drive import GoogleDriveAuth, GoogleDriveLoader
+
+__all__ = [
+    'BaseConnectorAuth',
+    'BaseConnectorLoader',
+    'ConnectorCreator',
+    'GoogleDriveAuth',
+    'GoogleDriveLoader'
+]

application/parser/connectors/_auth_utils.py

@@ -0,0 +1,37 @@
+"""Shared helpers for connector auth modules.
+
+These helpers exist so that sensitive values (session tokens, bearer
+credentials) never end up interpolated into exception messages or log
+lines. Exception messages frequently flow into ``stack_logs`` (Postgres)
+and Sentry via ``exc_info=True``, so the raw value must never be the
+thing we format.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+
+def session_token_fingerprint(session_token: str) -> str:
+    """Return a short, irreversible fingerprint for a session token.
+
+    The returned string is safe to embed in exception messages and log
+    lines: it is a prefix of a SHA-256 digest, clearly tagged so an
+    operator reading the log knows it is a hash and not the token
+    itself. It is stable for a given input, which lets operators
+    correlate "which token failed" across log lines without exposing
+    the credential.
+
+    Args:
+        session_token: The raw session token. Accepts ``None`` or the
+            empty string for defensive callers; both yield a distinct
+            sentinel rather than raising.
+
+    Returns:
+        A string of the form ``"sha256:<6 hex chars>"``, or
+        ``"sha256:<empty>"`` when the input is falsy.
+    """
+    if not session_token:
+        return "sha256:<empty>"
+    digest = hashlib.sha256(session_token.encode("utf-8")).hexdigest()
+    return f"sha256:{digest[:6]}"

application/parser/connectors/base.py

@@ -0,0 +1,140 @@
+"""
+Base classes for external knowledge base connectors.
+
+This module provides minimal abstract base classes that define the essential
+interface for external knowledge base connectors.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Optional
+
+from application.parser.schema.base import Document
+
+
+class BaseConnectorAuth(ABC):
+    """
+    Abstract base class for connector authentication.
+    
+    Defines the minimal interface that all connector authentication
+    implementations must follow.
+    """
+    
+    @abstractmethod
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        """
+        Generate authorization URL for OAuth flows.
+        
+        Args:
+            state: Optional state parameter for CSRF protection
+            
+        Returns:
+            Authorization URL
+        """
+        pass
+    
+    @abstractmethod
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        """
+        Exchange authorization code for access tokens.
+        
+        Args:
+            authorization_code: Authorization code from OAuth callback
+            
+        Returns:
+            Dictionary containing token information
+        """
+        pass
+    
+    @abstractmethod
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        """
+        Refresh an expired access token.
+        
+        Args:
+            refresh_token: Refresh token
+            
+        Returns:
+            Dictionary containing refreshed token information
+        """
+        pass
+    
+    @abstractmethod
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        """
+        Check if a token is expired.
+
+        Args:
+            token_info: Token information dictionary
+
+        Returns:
+            True if token is expired, False otherwise
+        """
+        pass
+
+    def sanitize_token_info(self, token_info: Dict[str, Any], **extra_fields) -> Dict[str, Any]:
+        """Extract the fields safe to persist in the session store.
+        """
+        return {
+            "access_token": token_info.get("access_token"),
+            "refresh_token": token_info.get("refresh_token"),
+            "token_uri": token_info.get("token_uri"),
+            "expiry": token_info.get("expiry"),
+            **extra_fields,
+        }
+
+
+class BaseConnectorLoader(ABC):
+    """
+    Abstract base class for connector loaders.
+    
+    Defines the minimal interface that all connector loader
+    implementations must follow.
+    """
+    
+    @abstractmethod
+    def __init__(self, session_token: str):
+        """
+        Initialize the connector loader.
+        
+        Args:
+            session_token: Authentication session token
+        """
+        pass
+    
+    @abstractmethod
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        """
+        Load documents from the external knowledge base.
+        
+        Args:
+            inputs: Configuration dictionary containing:
+                - file_ids: Optional list of specific file IDs to load
+                - folder_ids: Optional list of folder IDs to browse/download
+                - limit: Maximum number of items to return
+                - list_only: If True, return metadata without content
+                - recursive: Whether to recursively process folders
+                
+        Returns:
+            List of Document objects
+        """
+        pass
+    
+    @abstractmethod
+    def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:
+        """
+        Download files/folders to a local directory.
+        
+        Args:
+            local_dir: Local directory path to download files to
+            source_config: Configuration for what to download
+            
+        Returns:
+            Dictionary containing download results:
+                - files_downloaded: Number of files downloaded
+                - directory_path: Path where files were downloaded
+                - empty_result: Whether no files were downloaded
+                - source_type: Type of connector
+                - config_used: Configuration that was used
+                - error: Error message if download failed (optional)
+        """
+        pass

application/parser/connectors/confluence/__init__.py

@@ -0,0 +1,4 @@
+from .auth import ConfluenceAuth
+from .loader import ConfluenceLoader
+
+__all__ = ["ConfluenceAuth", "ConfluenceLoader"]

application/parser/connectors/confluence/auth.py

@@ -0,0 +1,221 @@
+import datetime
+import logging
+from typing import Any, Dict, Optional
+from urllib.parse import urlencode
+
+import requests
+
+from application.core.settings import settings
+from application.parser.connectors._auth_utils import session_token_fingerprint
+from application.parser.connectors.base import BaseConnectorAuth
+
+logger = logging.getLogger(__name__)
+
+
+class ConfluenceAuth(BaseConnectorAuth):
+
+    SCOPES = [
+        "read:page:confluence",
+        "read:space:confluence",
+        "read:attachment:confluence",
+        "read:me",
+        "offline_access",
+    ]
+
+    AUTH_URL = "https://auth.atlassian.com/authorize"
+    TOKEN_URL = "https://auth.atlassian.com/oauth/token"
+    RESOURCES_URL = "https://api.atlassian.com/oauth/token/accessible-resources"
+    ME_URL = "https://api.atlassian.com/me"
+
+    def __init__(self):
+        self.client_id = settings.CONFLUENCE_CLIENT_ID
+        self.client_secret = settings.CONFLUENCE_CLIENT_SECRET
+        self.redirect_uri = settings.CONNECTOR_REDIRECT_BASE_URI
+
+        if not self.client_id or not self.client_secret:
+            raise ValueError(
+                "Confluence OAuth credentials not configured. "
+                "Please set CONFLUENCE_CLIENT_ID and CONFLUENCE_CLIENT_SECRET in settings."
+            )
+
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        params = {
+            "audience": "api.atlassian.com",
+            "client_id": self.client_id,
+            "scope": " ".join(self.SCOPES),
+            "redirect_uri": self.redirect_uri,
+            "state": state,
+            "response_type": "code",
+            "prompt": "consent",
+        }
+        return f"{self.AUTH_URL}?{urlencode(params)}"
+
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        if not authorization_code:
+            raise ValueError("Authorization code is required")
+
+        response = requests.post(
+            self.TOKEN_URL,
+            json={
+                "grant_type": "authorization_code",
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "code": authorization_code,
+                "redirect_uri": self.redirect_uri,
+            },
+            headers={"Content-Type": "application/json"},
+            timeout=30,
+        )
+        response.raise_for_status()
+        token_data = response.json()
+
+        access_token = token_data.get("access_token")
+        if not access_token:
+            raise ValueError("OAuth flow did not return an access token")
+
+        refresh_token = token_data.get("refresh_token")
+        if not refresh_token:
+            raise ValueError("OAuth flow did not return a refresh token")
+
+        expires_in = token_data.get("expires_in", 3600)
+        expiry = (
+            datetime.datetime.now(datetime.timezone.utc)
+            + datetime.timedelta(seconds=expires_in)
+        ).isoformat()
+
+        cloud_id = self._fetch_cloud_id(access_token)
+        user_info = self._fetch_user_info(access_token)
+
+        return {
+            "access_token": access_token,
+            "refresh_token": refresh_token,
+            "token_uri": self.TOKEN_URL,
+            "scopes": self.SCOPES,
+            "expiry": expiry,
+            "cloud_id": cloud_id,
+            "user_info": {
+                "name": user_info.get("display_name", ""),
+                "email": user_info.get("email", ""),
+            },
+        }
+
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        if not refresh_token:
+            raise ValueError("Refresh token is required")
+
+        response = requests.post(
+            self.TOKEN_URL,
+            json={
+                "grant_type": "refresh_token",
+                "client_id": self.client_id,
+                "client_secret": self.client_secret,
+                "refresh_token": refresh_token,
+            },
+            headers={"Content-Type": "application/json"},
+            timeout=30,
+        )
+        response.raise_for_status()
+        token_data = response.json()
+
+        access_token = token_data.get("access_token")
+        new_refresh_token = token_data.get("refresh_token", refresh_token)
+
+        expires_in = token_data.get("expires_in", 3600)
+        expiry = (
+            datetime.datetime.now(datetime.timezone.utc)
+            + datetime.timedelta(seconds=expires_in)
+        ).isoformat()
+
+        cloud_id = self._fetch_cloud_id(access_token)
+
+        return {
+            "access_token": access_token,
+            "refresh_token": new_refresh_token,
+            "token_uri": self.TOKEN_URL,
+            "scopes": self.SCOPES,
+            "expiry": expiry,
+            "cloud_id": cloud_id,
+        }
+
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        if not token_info:
+            return True
+
+        expiry = token_info.get("expiry")
+        if not expiry:
+            return bool(token_info.get("access_token"))
+
+        try:
+            expiry_dt = datetime.datetime.fromisoformat(expiry)
+            now = datetime.datetime.now(datetime.timezone.utc)
+            return now >= expiry_dt - datetime.timedelta(seconds=60)
+        except Exception:
+            return True
+
+    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
+        from application.storage.db.repositories.connector_sessions import (
+            ConnectorSessionsRepository,
+        )
+        from application.storage.db.session import db_readonly
+
+        with db_readonly() as conn:
+            session = ConnectorSessionsRepository(conn).get_by_session_token(
+                session_token
+            )
+        if not session:
+            raise ValueError(
+                f"Invalid session token ({session_token_fingerprint(session_token)})"
+            )
+
+        token_info = session.get("token_info")
+        if not token_info:
+            raise ValueError("Session missing token information")
+
+        required = ["access_token", "refresh_token", "cloud_id"]
+        missing = [f for f in required if not token_info.get(f)]
+        if missing:
+            raise ValueError(f"Missing required token fields: {missing}")
+
+        return token_info
+
+    def sanitize_token_info(
+        self, token_info: Dict[str, Any], **extra_fields
+    ) -> Dict[str, Any]:
+        return super().sanitize_token_info(
+            token_info,
+            cloud_id=token_info.get("cloud_id"),
+            **extra_fields,
+        )
+
+    def _fetch_cloud_id(self, access_token: str) -> str:
+        response = requests.get(
+            self.RESOURCES_URL,
+            headers={
+                "Authorization": f"Bearer {access_token}",
+                "Accept": "application/json",
+            },
+            timeout=30,
+        )
+        response.raise_for_status()
+        resources = response.json()
+
+        if not resources:
+            raise ValueError("No accessible Confluence sites found for this account")
+
+        return resources[0]["id"]
+
+    def _fetch_user_info(self, access_token: str) -> Dict[str, Any]:
+        try:
+            response = requests.get(
+                self.ME_URL,
+                headers={
+                    "Authorization": f"Bearer {access_token}",
+                    "Accept": "application/json",
+                },
+                timeout=30,
+            )
+            response.raise_for_status()
+            return response.json()
+        except Exception as e:
+            logger.warning("Could not fetch user info: %s", e)
+            return {}

application/parser/connectors/confluence/loader.py

@@ -0,0 +1,417 @@
+import functools
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from application.parser.connectors.base import BaseConnectorLoader
+from application.parser.connectors.confluence.auth import ConfluenceAuth
+from application.parser.schema.base import Document
+
+logger = logging.getLogger(__name__)
+
+API_V2 = "https://api.atlassian.com/ex/confluence/{cloud_id}/wiki/api/v2"
+DOWNLOAD_BASE = "https://api.atlassian.com/ex/confluence/{cloud_id}/wiki"
+
+SUPPORTED_ATTACHMENT_TYPES = {
+    "application/pdf": ".pdf",
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+    "application/msword": ".doc",
+    "application/vnd.ms-powerpoint": ".ppt",
+    "application/vnd.ms-excel": ".xls",
+    "text/plain": ".txt",
+    "text/csv": ".csv",
+    "text/html": ".html",
+    "text/markdown": ".md",
+    "application/json": ".json",
+    "application/epub+zip": ".epub",
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+}
+
+
+def _retry_on_auth_failure(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code in (401, 403):
+                logger.info(
+                    "Auth failure in %s, refreshing token and retrying", func.__name__
+                )
+                try:
+                    new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                    self.access_token = new_token_info["access_token"]
+                    self.refresh_token = new_token_info.get(
+                        "refresh_token", self.refresh_token
+                    )
+                    self._persist_refreshed_tokens(new_token_info)
+                except Exception as refresh_err:
+                    raise ValueError(
+                        f"Authentication failed and could not be refreshed: {refresh_err}"
+                    ) from e
+                return func(self, *args, **kwargs)
+            raise
+
+    return wrapper
+
+
+class ConfluenceLoader(BaseConnectorLoader):
+
+    def __init__(self, session_token: str):
+        self.auth = ConfluenceAuth()
+        self.session_token = session_token
+
+        token_info = self.auth.get_token_info_from_session(session_token)
+        self.access_token = token_info["access_token"]
+        self.refresh_token = token_info["refresh_token"]
+        self.cloud_id = token_info["cloud_id"]
+
+        self.base_url = API_V2.format(cloud_id=self.cloud_id)
+        self.download_base = DOWNLOAD_BASE.format(cloud_id=self.cloud_id)
+        self.next_page_token = None
+
+    def _headers(self) -> Dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.access_token}",
+            "Accept": "application/json",
+        }
+
+    def _persist_refreshed_tokens(self, token_info: Dict[str, Any]) -> None:
+        try:
+            from application.storage.db.repositories.connector_sessions import (
+                ConnectorSessionsRepository,
+            )
+            from application.storage.db.session import db_session
+
+            sanitized = self.auth.sanitize_token_info(token_info)
+            with db_session() as conn:
+                repo = ConnectorSessionsRepository(conn)
+                session = repo.get_by_session_token(self.session_token)
+                if session:
+                    repo.update(str(session["id"]), {"token_info": sanitized})
+        except Exception as e:
+            logger.warning("Failed to persist refreshed tokens: %s", e)
+
+    @_retry_on_auth_failure
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        folder_id = inputs.get("folder_id")
+        file_ids = inputs.get("file_ids", [])
+        limit = inputs.get("limit", 100)
+        list_only = inputs.get("list_only", False)
+        page_token = inputs.get("page_token")
+        search_query = inputs.get("search_query")
+        self.next_page_token = None
+
+        if file_ids:
+            return self._load_pages_by_ids(file_ids, list_only, search_query)
+
+        if folder_id:
+            return self._list_pages_in_space(
+                folder_id, limit, list_only, page_token, search_query
+            )
+
+        return self._list_spaces(limit, page_token, search_query)
+
+    @_retry_on_auth_failure
+    def download_to_directory(self, local_dir: str, source_config: dict = None) -> dict:
+        config = source_config or getattr(self, "config", {})
+        file_ids = config.get("file_ids", [])
+        folder_ids = config.get("folder_ids", [])
+        files_downloaded = 0
+
+        os.makedirs(local_dir, exist_ok=True)
+
+        if isinstance(file_ids, str):
+            file_ids = [file_ids]
+        if isinstance(folder_ids, str):
+            folder_ids = [folder_ids]
+
+        for page_id in file_ids:
+            if self._download_page(page_id, local_dir):
+                files_downloaded += 1
+            files_downloaded += self._download_page_attachments(page_id, local_dir)
+
+        for space_id in folder_ids:
+            files_downloaded += self._download_space(space_id, local_dir)
+
+        return {
+            "files_downloaded": files_downloaded,
+            "directory_path": local_dir,
+            "empty_result": files_downloaded == 0,
+            "source_type": "confluence",
+            "config_used": config,
+        }
+
+    def _list_spaces(
+        self, limit: int, cursor: Optional[str], search_query: Optional[str]
+    ) -> List[Document]:
+        documents: List[Document] = []
+        params: Dict[str, Any] = {"limit": min(limit, 250)}
+        if cursor:
+            params["cursor"] = cursor
+
+        response = requests.get(
+            f"{self.base_url}/spaces",
+            headers=self._headers(),
+            params=params,
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        for space in data.get("results", []):
+            name = space.get("name", "")
+            if search_query and search_query.lower() not in name.lower():
+                continue
+
+            documents.append(
+                Document(
+                    text="",
+                    doc_id=space["id"],
+                    extra_info={
+                        "file_name": name,
+                        "mime_type": "folder",
+                        "size": None,
+                        "created_time": space.get("createdAt"),
+                        "modified_time": None,
+                        "source": "confluence",
+                        "is_folder": True,
+                        "space_key": space.get("key"),
+                    },
+                )
+            )
+
+        next_link = data.get("_links", {}).get("next")
+        self.next_page_token = self._extract_cursor(next_link)
+        return documents
+
+    def _list_pages_in_space(
+        self,
+        space_id: str,
+        limit: int,
+        list_only: bool,
+        cursor: Optional[str],
+        search_query: Optional[str],
+    ) -> List[Document]:
+        documents: List[Document] = []
+        params: Dict[str, Any] = {"limit": min(limit, 250)}
+        if cursor:
+            params["cursor"] = cursor
+
+        response = requests.get(
+            f"{self.base_url}/spaces/{space_id}/pages",
+            headers=self._headers(),
+            params=params,
+            timeout=30,
+        )
+        response.raise_for_status()
+        data = response.json()
+
+        for page in data.get("results", []):
+            title = page.get("title", "")
+            if search_query and search_query.lower() not in title.lower():
+                continue
+
+            doc = self._page_to_document(
+                page, load_content=not list_only, space_id=space_id
+            )
+            if doc:
+                documents.append(doc)
+
+        next_link = data.get("_links", {}).get("next")
+        self.next_page_token = self._extract_cursor(next_link)
+        return documents
+
+    def _load_pages_by_ids(
+        self, page_ids: List[str], list_only: bool, search_query: Optional[str]
+    ) -> List[Document]:
+        documents: List[Document] = []
+        for page_id in page_ids:
+            try:
+                params: Dict[str, str] = {}
+                if not list_only:
+                    params["body-format"] = "storage"
+
+                response = requests.get(
+                    f"{self.base_url}/pages/{page_id}",
+                    headers=self._headers(),
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                page = response.json()
+
+                title = page.get("title", "")
+                if search_query and search_query.lower() not in title.lower():
+                    continue
+
+                doc = self._page_to_document(page, load_content=not list_only)
+                if doc:
+                    documents.append(doc)
+            except Exception as e:
+                logger.error("Error loading page %s: %s", page_id, e)
+        return documents
+
+    def _page_to_document(
+        self,
+        page: Dict[str, Any],
+        load_content: bool = False,
+        space_id: Optional[str] = None,
+    ) -> Optional[Document]:
+        page_id = page.get("id")
+        title = page.get("title", "Unknown")
+        version = page.get("version", {})
+        modified_time = version.get("createdAt") if isinstance(version, dict) else None
+        created_time = page.get("createdAt")
+        resolved_space_id = space_id or page.get("spaceId")
+
+        text = ""
+        if load_content:
+            body = page.get("body", {})
+            storage = body.get("storage", {}) if isinstance(body, dict) else {}
+            text = storage.get("value", "") if isinstance(storage, dict) else ""
+
+        return Document(
+            text=text,
+            doc_id=str(page_id),
+            extra_info={
+                "file_name": title,
+                "mime_type": "text/html",
+                "size": len(text) if text else None,
+                "created_time": created_time,
+                "modified_time": modified_time,
+                "source": "confluence",
+                "is_folder": False,
+                "page_id": str(page_id),
+                "space_id": resolved_space_id,
+                "cloud_id": self.cloud_id,
+            },
+        )
+
+    def _download_page(self, page_id: str, local_dir: str) -> bool:
+        try:
+            response = requests.get(
+                f"{self.base_url}/pages/{page_id}",
+                headers=self._headers(),
+                params={"body-format": "storage"},
+                timeout=30,
+            )
+            response.raise_for_status()
+            page = response.json()
+
+            title = page.get("title", page_id)
+            safe_name = "".join(c if c.isalnum() or c in " -_" else "_" for c in title)
+            body = page.get("body", {}).get("storage", {}).get("value", "")
+
+            file_path = os.path.join(local_dir, f"{safe_name}.html")
+            with open(file_path, "w", encoding="utf-8") as f:
+                f.write(body)
+
+            return True
+        except Exception as e:
+            logger.error("Error downloading page %s: %s", page_id, e)
+            return False
+
+    def _download_page_attachments(self, page_id: str, local_dir: str) -> int:
+        downloaded = 0
+        try:
+            cursor = None
+            while True:
+                params: Dict[str, Any] = {"limit": 100}
+                if cursor:
+                    params["cursor"] = cursor
+
+                response = requests.get(
+                    f"{self.base_url}/pages/{page_id}/attachments",
+                    headers=self._headers(),
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                data = response.json()
+
+                for att in data.get("results", []):
+                    media_type = att.get("mediaType", "")
+                    if media_type not in SUPPORTED_ATTACHMENT_TYPES:
+                        continue
+
+                    download_link = att.get("_links", {}).get("download")
+                    if not download_link:
+                        continue
+
+                    raw_name = att.get("title", att.get("id", "attachment"))
+                    file_name = "".join(
+                        c if c.isalnum() or c in " -_." else "_"
+                        for c in os.path.basename(raw_name)
+                    ) or "attachment"
+                    file_path = os.path.join(local_dir, file_name)
+
+                    url = f"{self.download_base}{download_link}"
+                    file_resp = requests.get(
+                        url, headers=self._headers(), timeout=60, stream=True
+                    )
+                    file_resp.raise_for_status()
+
+                    with open(file_path, "wb") as f:
+                        for chunk in file_resp.iter_content(chunk_size=8192):
+                            f.write(chunk)
+
+                    downloaded += 1
+
+                next_link = data.get("_links", {}).get("next")
+                cursor = self._extract_cursor(next_link)
+                if not cursor:
+                    break
+
+        except Exception as e:
+            logger.error("Error downloading attachments for page %s: %s", page_id, e)
+        return downloaded
+
+    def _download_space(self, space_id: str, local_dir: str) -> int:
+        downloaded = 0
+        cursor = None
+        while True:
+            params: Dict[str, Any] = {"limit": 250}
+            if cursor:
+                params["cursor"] = cursor
+
+            try:
+                response = requests.get(
+                    f"{self.base_url}/spaces/{space_id}/pages",
+                    headers=self._headers(),
+                    params=params,
+                    timeout=30,
+                )
+                response.raise_for_status()
+                data = response.json()
+            except Exception as e:
+                logger.error("Error listing pages in space %s: %s", space_id, e)
+                break
+
+            for page in data.get("results", []):
+                page_id = page.get("id")
+                if self._download_page(str(page_id), local_dir):
+                    downloaded += 1
+                downloaded += self._download_page_attachments(str(page_id), local_dir)
+
+            next_link = data.get("_links", {}).get("next")
+            cursor = self._extract_cursor(next_link)
+            if not cursor:
+                break
+
+        return downloaded
+
+    @staticmethod
+    def _extract_cursor(next_link: Optional[str]) -> Optional[str]:
+        if not next_link:
+            return None
+        from urllib.parse import parse_qs, urlparse
+
+        parsed = urlparse(next_link)
+        cursors = parse_qs(parsed.query).get("cursor")
+        return cursors[0] if cursors else None

application/parser/connectors/connector_creator.py

@@ -0,0 +1,89 @@
+from application.parser.connectors.confluence.auth import ConfluenceAuth
+from application.parser.connectors.confluence.loader import ConfluenceLoader
+from application.parser.connectors.google_drive.auth import GoogleDriveAuth
+from application.parser.connectors.google_drive.loader import GoogleDriveLoader
+from application.parser.connectors.share_point.auth import SharePointAuth
+from application.parser.connectors.share_point.loader import SharePointLoader
+
+
+class ConnectorCreator:
+    """
+    Factory class for creating external knowledge base connectors and auth providers.
+
+    These are different from remote loaders as they typically require
+    authentication and connect to external document storage systems.
+    """
+
+    connectors = {
+        "confluence": ConfluenceLoader,
+        "google_drive": GoogleDriveLoader,
+        "share_point": SharePointLoader,
+    }
+
+    auth_providers = {
+        "confluence": ConfluenceAuth,
+        "google_drive": GoogleDriveAuth,
+        "share_point": SharePointAuth,
+    }
+
+    @classmethod
+    def create_connector(cls, connector_type, *args, **kwargs):
+        """
+        Create a connector instance for the specified type.
+
+        Args:
+            connector_type: Type of connector to create (e.g., 'google_drive')
+            *args, **kwargs: Arguments to pass to the connector constructor
+
+        Returns:
+            Connector instance
+
+        Raises:
+            ValueError: If connector type is not supported
+        """
+        connector_class = cls.connectors.get(connector_type.lower())
+        if not connector_class:
+            raise ValueError(f"No connector class found for type {connector_type}")
+        return connector_class(*args, **kwargs)
+
+    @classmethod
+    def create_auth(cls, connector_type):
+        """
+        Create an auth provider instance for the specified connector type.
+
+        Args:
+            connector_type: Type of connector auth to create (e.g., 'google_drive')
+
+        Returns:
+            Auth provider instance
+
+        Raises:
+            ValueError: If connector type is not supported for auth
+        """
+        auth_class = cls.auth_providers.get(connector_type.lower())
+        if not auth_class:
+            raise ValueError(f"No auth class found for type {connector_type}")
+        return auth_class()
+
+    @classmethod
+    def get_supported_connectors(cls):
+        """
+        Get list of supported connector types.
+
+        Returns:
+            List of supported connector type strings
+        """
+        return list(cls.connectors.keys())
+
+    @classmethod
+    def is_supported(cls, connector_type):
+        """
+        Check if a connector type is supported.
+
+        Args:
+            connector_type: Type of connector to check
+
+        Returns:
+            True if supported, False otherwise
+        """
+        return connector_type.lower() in cls.connectors

application/parser/connectors/google_drive/__init__.py

@@ -0,0 +1,10 @@
+"""
+Google Drive connector for DocsGPT.
+
+This module provides authentication and document loading capabilities for Google Drive.
+"""
+
+from .auth import GoogleDriveAuth
+from .loader import GoogleDriveLoader
+
+__all__ = ['GoogleDriveAuth', 'GoogleDriveLoader']

application/parser/connectors/google_drive/auth.py

@@ -0,0 +1,269 @@
+import logging
+import datetime
+from typing import Optional, Dict, Any
+
+from google.oauth2.credentials import Credentials
+from google_auth_oauthlib.flow import Flow
+from googleapiclient.discovery import build
+from googleapiclient.errors import HttpError
+
+from application.core.settings import settings
+from application.parser.connectors._auth_utils import session_token_fingerprint
+from application.parser.connectors.base import BaseConnectorAuth
+
+
+class GoogleDriveAuth(BaseConnectorAuth):
+    """
+    Handles Google OAuth 2.0 authentication for Google Drive access.
+    """
+    
+    SCOPES = [
+        'https://www.googleapis.com/auth/drive.readonly'
+    ]
+    
+    def __init__(self):
+        self.client_id = settings.GOOGLE_CLIENT_ID
+        self.client_secret = settings.GOOGLE_CLIENT_SECRET
+        self.redirect_uri = f"{settings.CONNECTOR_REDIRECT_BASE_URI}"
+        
+        if not self.client_id or not self.client_secret:
+            raise ValueError("Google OAuth credentials not configured. Please set GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET in settings.")
+
+
+
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        try:
+            flow = Flow.from_client_config(
+                {
+                    "web": {
+                        "client_id": self.client_id,
+                        "client_secret": self.client_secret,
+                        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                        "token_uri": "https://oauth2.googleapis.com/token",
+                        "redirect_uris": [self.redirect_uri]
+                    }
+                },
+                scopes=self.SCOPES,
+                # Flow is rebuilt at exchange time, so an auto-generated
+                # code_verifier wouldn't survive; confidential client doesn't
+                # need PKCE anyway.
+                autogenerate_code_verifier=False,
+            )
+            flow.redirect_uri = self.redirect_uri
+
+            authorization_url, _ = flow.authorization_url(
+                access_type='offline',
+                prompt='consent',
+                include_granted_scopes='false',
+                state=state
+            )
+
+            return authorization_url
+
+        except Exception as e:
+            logging.error(f"Error generating authorization URL: {e}")
+            raise
+
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        try:
+            if not authorization_code:
+                raise ValueError("Authorization code is required")
+
+            flow = Flow.from_client_config(
+                {
+                    "web": {
+                        "client_id": self.client_id,
+                        "client_secret": self.client_secret,
+                        "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+                        "token_uri": "https://oauth2.googleapis.com/token",
+                        "redirect_uris": [self.redirect_uri]
+                    }
+                },
+                scopes=self.SCOPES,
+                autogenerate_code_verifier=False,
+            )
+            flow.redirect_uri = self.redirect_uri
+
+            flow.fetch_token(code=authorization_code)
+
+            credentials = flow.credentials
+
+            if not credentials.refresh_token:
+                logging.warning("OAuth flow did not return a refresh_token.")
+            if not credentials.token:
+                raise ValueError("OAuth flow did not return an access token")
+
+            if not credentials.token_uri:
+                credentials.token_uri = "https://oauth2.googleapis.com/token"
+
+            if not credentials.client_id:
+                credentials.client_id = self.client_id
+
+            if not credentials.client_secret:
+                credentials.client_secret = self.client_secret
+
+            if not credentials.refresh_token:
+                raise ValueError(
+                    "No refresh token received. This typically happens when offline access wasn't granted. "
+                )
+
+            return {
+                'access_token': credentials.token,
+                'refresh_token': credentials.refresh_token,
+                'token_uri': credentials.token_uri,
+                'client_id': credentials.client_id,
+                'client_secret': credentials.client_secret,
+                'scopes': credentials.scopes,
+                'expiry': credentials.expiry.isoformat() if credentials.expiry else None
+            }
+
+        except Exception as e:
+            logging.error(f"Error exchanging code for tokens: {e}")
+            raise
+    
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        try:
+            if not refresh_token:
+                raise ValueError("Refresh token is required")
+
+            credentials = Credentials(
+                token=None,
+                refresh_token=refresh_token,
+                token_uri="https://oauth2.googleapis.com/token",
+                client_id=self.client_id,
+                client_secret=self.client_secret
+            )
+
+            from google.auth.transport.requests import Request
+            credentials.refresh(Request())
+
+            return {
+                'access_token': credentials.token,
+                'refresh_token': refresh_token,
+                'token_uri': credentials.token_uri,
+                'client_id': credentials.client_id,
+                'client_secret': credentials.client_secret,
+                'scopes': credentials.scopes,
+                'expiry': credentials.expiry.isoformat() if credentials.expiry else None
+            }
+        except Exception as e:
+            logging.error(f"Error refreshing access token: {e}", exc_info=True)
+            raise
+    
+    def create_credentials_from_token_info(self, token_info: Dict[str, Any]) -> Credentials:
+        from application.core.settings import settings
+
+        access_token = token_info.get('access_token')
+        if not access_token:
+            raise ValueError("No access token found in token_info")
+
+        credentials = Credentials(
+            token=access_token,
+            refresh_token=token_info.get('refresh_token'),
+            token_uri= 'https://oauth2.googleapis.com/token',
+            client_id=settings.GOOGLE_CLIENT_ID,
+            client_secret=settings.GOOGLE_CLIENT_SECRET,
+            scopes=token_info.get('scopes', ['https://www.googleapis.com/auth/drive.readonly'])
+        )
+
+        if not credentials.token:
+            raise ValueError("Credentials created without valid access token")
+
+        return credentials
+    
+    def build_drive_service(self, credentials: Credentials):
+        try:
+            if not credentials:
+                raise ValueError("No credentials provided")
+
+            if not credentials.token and not credentials.refresh_token:
+                raise ValueError("No access token or refresh token available. User must re-authorize with offline access.")
+
+            needs_refresh = credentials.expired or not credentials.token
+            if needs_refresh:
+                if credentials.refresh_token:
+                    try:
+                        from google.auth.transport.requests import Request
+                        credentials.refresh(Request())
+                    except Exception as refresh_error:
+                        raise ValueError(f"Failed to refresh credentials: {refresh_error}")
+                else:
+                    raise ValueError("No access token or refresh token available. User must re-authorize with offline access.")
+
+            return build('drive', 'v3', credentials=credentials)
+
+        except HttpError as e:
+            raise ValueError(f"Failed to build Google Drive service: HTTP {e.resp.status}")
+        except Exception as e:
+            raise ValueError(f"Failed to build Google Drive service: {str(e)}")
+        
+    def is_token_expired(self, token_info):
+        if 'expiry' in token_info and token_info['expiry']:
+            try:
+                from dateutil import parser
+                # Google Drive provides timezone-aware ISO8601 dates
+                expiry_dt = parser.parse(token_info['expiry'])
+                current_time = datetime.datetime.now(datetime.timezone.utc)
+                return current_time >= expiry_dt - datetime.timedelta(seconds=60)
+            except Exception:
+                return True
+
+        if 'access_token' in token_info and token_info['access_token']:
+            return False
+
+        return True
+    
+    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
+        try:
+            from application.storage.db.repositories.connector_sessions import (
+                ConnectorSessionsRepository,
+            )
+            from application.storage.db.session import db_readonly
+
+            with db_readonly() as conn:
+                session = ConnectorSessionsRepository(conn).get_by_session_token(
+                    session_token
+                )
+            if not session:
+                raise ValueError(
+                    f"Invalid session token ({session_token_fingerprint(session_token)})"
+                )
+
+            token_info = session.get("token_info")
+            if not token_info:
+                raise ValueError("Session missing token information")
+
+            required_fields = ["access_token", "refresh_token"]
+            missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]
+            if missing_fields:
+                raise ValueError(f"Missing required token fields: {missing_fields}")
+
+            if 'token_uri' not in token_info:
+                token_info['token_uri'] = 'https://oauth2.googleapis.com/token'
+
+            return token_info
+
+        except Exception as e:
+            raise ValueError(f"Failed to retrieve Google Drive token information: {str(e)}")
+
+    def validate_credentials(self, credentials: Credentials) -> bool:
+        """
+        Validate Google Drive credentials by making a test API call.
+
+        Args:
+            credentials: Google credentials object
+
+        Returns:
+            True if credentials are valid, False otherwise
+        """
+        try:
+            service = self.build_drive_service(credentials)
+            service.about().get(fields="user").execute()
+            return True
+
+        except HttpError as e:
+            logging.error(f"HTTP error validating credentials: {e}")
+            return False
+        except Exception as e:
+            logging.error(f"Error validating credentials: {e}")
+            return False

application/parser/connectors/google_drive/loader.py

@@ -0,0 +1,561 @@
+"""
+Google Drive loader for DocsGPT.
+Loads documents from Google Drive using Google Drive API.
+"""
+
+import io
+import logging
+import os
+from typing import List, Dict, Any, Optional
+
+from googleapiclient.http import MediaIoBaseDownload
+from googleapiclient.errors import HttpError
+
+from application.parser.connectors.base import BaseConnectorLoader
+from application.parser.connectors.google_drive.auth import GoogleDriveAuth
+from application.parser.schema.base import Document
+
+
+class GoogleDriveLoader(BaseConnectorLoader):
+
+    SUPPORTED_MIME_TYPES = {
+        'application/pdf': '.pdf',
+        'application/vnd.google-apps.document': '.docx',
+        'application/vnd.google-apps.presentation': '.pptx',
+        'application/vnd.google-apps.spreadsheet': '.xlsx',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+        'application/msword': '.doc',
+        'application/vnd.ms-powerpoint': '.ppt',
+        'application/vnd.ms-excel': '.xls',
+        'text/plain': '.txt',
+        'text/csv': '.csv',
+        'text/html': '.html',
+        'text/markdown': '.md',
+        'text/x-rst': '.rst',
+        'application/json': '.json',
+        'application/epub+zip': '.epub',
+        'application/rtf': '.rtf',
+        'image/jpeg': '.jpg',
+        'image/jpg': '.jpg',
+        'image/png': '.png',
+    }
+
+    EXPORT_FORMATS = {
+        'application/vnd.google-apps.document': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        'application/vnd.google-apps.presentation': 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        'application/vnd.google-apps.spreadsheet': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
+    }
+
+    def __init__(self, session_token: str):
+        self.auth = GoogleDriveAuth()
+        self.session_token = session_token
+
+        token_info = self.auth.get_token_info_from_session(session_token)
+        self.credentials = self.auth.create_credentials_from_token_info(token_info)
+
+        try:
+            self.service = self.auth.build_drive_service(self.credentials)
+        except Exception as e:
+            logging.warning(f"Could not build Google Drive service: {e}")
+            self.service = None
+
+        self.next_page_token = None
+
+
+
+    def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:
+        try:
+            file_id = file_metadata.get('id')
+            file_name = file_metadata.get('name', 'Unknown')
+            mime_type = file_metadata.get('mimeType', 'application/octet-stream')
+
+            if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):
+                return None
+            if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):
+                logging.info(f"Skipping unsupported file type: {mime_type} for file {file_name}")
+                return None
+            # Google Drive provides timezone-aware ISO8601 dates
+            doc_metadata = {
+                'file_name': file_name,
+                'mime_type': mime_type,
+                'size': file_metadata.get('size', None),
+                'created_time': file_metadata.get('createdTime'),
+                'modified_time': file_metadata.get('modifiedTime'),
+                'parents': file_metadata.get('parents', []),
+                'source': 'google_drive'
+            }
+
+            if not load_content:
+                return Document(
+                    text="",
+                    doc_id=file_id,
+                    extra_info=doc_metadata
+                )
+
+            content = self._download_file_content(file_id, mime_type)
+            if content is None:
+                logging.warning(f"Could not load content for file {file_name} ({file_id})")
+                return None
+
+            return Document(
+                text=content,
+                doc_id=file_id,
+                extra_info=doc_metadata
+            )
+
+        except Exception as e:
+            logging.error(f"Error processing file: {e}")
+            return None
+
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        session_token = inputs.get('session_token')
+        if session_token and session_token != self.session_token:
+            logging.warning("Session token in inputs differs from loader's session token. Using loader's session token.")
+        self.config = inputs
+
+        try:
+            documents: List[Document] = []
+
+            folder_id = inputs.get('folder_id')
+            file_ids = inputs.get('file_ids', [])
+            limit = inputs.get('limit', 100)
+            list_only = inputs.get('list_only', False)
+            load_content = not list_only
+            page_token = inputs.get('page_token')
+            search_query = inputs.get('search_query')
+            self.next_page_token = None
+
+            if file_ids:
+                # Specific files requested: load them
+                for file_id in file_ids:
+                    try:
+                        doc = self._load_file_by_id(file_id, load_content=load_content)
+                        if doc:
+                            if not search_query or (
+                                search_query.lower() in doc.extra_info.get('file_name', '').lower()
+                            ):
+                                documents.append(doc)
+                        elif hasattr(self, '_credential_refreshed') and self._credential_refreshed:
+                            self._credential_refreshed = False
+                            logging.info(f"Retrying load of file {file_id} after credential refresh")
+                            doc = self._load_file_by_id(file_id, load_content=load_content)
+                            if doc and (
+                                not search_query or 
+                                search_query.lower() in doc.extra_info.get('file_name', '').lower()
+                            ):
+                                documents.append(doc)
+                    except Exception as e:
+                        logging.error(f"Error loading file {file_id}: {e}")
+                        continue
+            else:
+                # Browsing mode: list immediate children of provided folder or root
+                parent_id = folder_id if folder_id else 'root'
+                documents = self._list_items_in_parent(
+                    parent_id, 
+                    limit=limit, 
+                    load_content=load_content, 
+                    page_token=page_token,
+                    search_query=search_query
+                )
+
+            logging.info(f"Loaded {len(documents)} documents from Google Drive")
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error loading data from Google Drive: {e}", exc_info=True)
+            raise
+
+
+
+    def _load_file_by_id(self, file_id: str, load_content: bool = True) -> Optional[Document]:
+        self._ensure_service()
+
+        try:
+            file_metadata = self.service.files().get(
+                fileId=file_id,
+                fields='id,name,mimeType,size,createdTime,modifiedTime,parents',
+                supportsAllDrives=True
+            ).execute()
+
+            return self._process_file(file_metadata, load_content=load_content)
+
+        except HttpError as e:
+            logging.error(f"HTTP error loading file {file_id}: {e.resp.status} - {e.content}")
+
+            if e.resp.status in [401, 403]:
+                if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                    try:
+                        from google.auth.transport.requests import Request
+                        self.credentials.refresh(Request())
+                        self._ensure_service()
+                        return None
+                    except Exception as refresh_error:
+                        raise ValueError(f"Authentication failed and could not be refreshed: {refresh_error}")
+                else:
+                    raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+            return None
+        except Exception as e:
+            logging.error(f"Error loading file {file_id}: {e}")
+            return None
+
+
+    def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:
+        self._ensure_service()
+
+        documents: List[Document] = []
+
+        try:
+            query = f"'{parent_id}' in parents and trashed=false"
+
+            if search_query:
+                safe_search = search_query.replace("'", "\\'")
+                query += f" and name contains '{safe_search}'"
+
+            next_token_out: Optional[str] = None
+
+            while True:
+                page_size = 100
+                if limit:
+                    remaining = max(0, limit - len(documents))
+                    if remaining == 0:
+                        break
+                    page_size = min(100, remaining)
+
+                results = self.service.files().list(
+                    q=query,
+                    fields='nextPageToken,files(id,name,mimeType,size,createdTime,modifiedTime,parents)',
+                    pageToken=page_token,
+                    pageSize=page_size,
+                    orderBy='name',
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True
+                ).execute()
+
+                items = results.get('files', [])
+                for item in items:
+                    mime_type = item.get('mimeType')
+                    if mime_type == 'application/vnd.google-apps.folder':
+                        doc_metadata = {
+                            'file_name': item.get('name', 'Unknown'),
+                            'mime_type': mime_type,
+                            'size': item.get('size', None),
+                            'created_time': item.get('createdTime'),
+                            'modified_time': item.get('modifiedTime'),
+                            'parents': item.get('parents', []),
+                            'source': 'google_drive',
+                            'is_folder': True
+                        }
+                        documents.append(Document(text="", doc_id=item.get('id'), extra_info=doc_metadata))
+                    else:
+                        doc = self._process_file(item, load_content=load_content)
+                        if doc:
+                            documents.append(doc)
+
+                    if limit and len(documents) >= limit:
+                        self.next_page_token = results.get('nextPageToken')
+                        return documents
+
+                page_token = results.get('nextPageToken')
+                next_token_out = page_token
+                if not page_token:
+                    break
+
+            self.next_page_token = next_token_out
+            return documents
+        except Exception as e:
+            logging.error(f"Error listing items under parent {parent_id}: {e}")
+            return documents
+
+
+
+
+    def _download_file_content(self, file_id: str, mime_type: str) -> Optional[str]:
+        if not self.credentials.token:
+            logging.warning("No access token in credentials, attempting to refresh")
+            if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                try:
+                    from google.auth.transport.requests import Request
+                    self.credentials.refresh(Request())
+                    logging.info("Credentials refreshed successfully")
+                    self._ensure_service()
+                except Exception as e:
+                    logging.error(f"Failed to refresh credentials: {e}")
+                    raise ValueError("Authentication failed and cannot be refreshed: missing or invalid refresh_token")
+            else:
+                logging.error("No access token and no refresh_token available")
+                raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+        if self.credentials.expired:
+            logging.warning("Credentials are expired, attempting to refresh")
+            if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                try:
+                    from google.auth.transport.requests import Request
+                    self.credentials.refresh(Request())
+                    logging.info("Credentials refreshed successfully")
+                    self._ensure_service()
+                except Exception as e:
+                    logging.error(f"Failed to refresh expired credentials: {e}")
+                    raise ValueError("Authentication failed and cannot be refreshed: expired credentials")
+            else:
+                logging.error("Credentials expired and no refresh_token available")
+                raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+        try:
+            if mime_type in self.EXPORT_FORMATS:
+                export_mime_type = self.EXPORT_FORMATS[mime_type]
+                request = self.service.files().export_media(
+                    fileId=file_id,
+                    mimeType=export_mime_type
+                )
+            else:
+                request = self.service.files().get_media(fileId=file_id, supportsAllDrives=True)
+
+            file_io = io.BytesIO()
+            downloader = MediaIoBaseDownload(file_io, request)
+
+            done = False
+            while done is False:
+                try:
+                    _, done = downloader.next_chunk()
+                except HttpError as e:
+                    logging.error(f"HTTP error downloading file {file_id}: {e.resp.status} - {e.content}")
+                    return None
+                except Exception as e:
+                    logging.error(f"Error during download of file {file_id}: {e}")
+                    return None
+
+            content_bytes = file_io.getvalue()
+
+            try:
+                return content_bytes.decode('utf-8')
+            except UnicodeDecodeError:
+                logging.error(f"Could not decode file {file_id} as text")
+                return None
+
+        except HttpError as e:
+            logging.error(f"HTTP error downloading file {file_id}: {e.resp.status} - {e.content}")
+
+            if e.resp.status in [401, 403]:
+                logging.error(f"Authentication error downloading file {file_id}")
+
+                if hasattr(self.credentials, 'refresh_token') and self.credentials.refresh_token:
+                    logging.info(f"Attempting to refresh credentials for file {file_id}")
+                    try:
+                        from google.auth.transport.requests import Request
+                        self.credentials.refresh(Request())
+                        logging.info("Credentials refreshed successfully")
+                        self._credential_refreshed = True
+                        self._ensure_service()
+                        return None
+                    except Exception as refresh_error:
+                        logging.error(f"Error refreshing credentials: {refresh_error}")
+                        raise ValueError(f"Authentication failed and could not be refreshed: {refresh_error}")
+                else:
+                    logging.error("Cannot refresh credentials: missing refresh_token")
+                    raise ValueError("Authentication failed and cannot be refreshed: missing refresh_token")
+
+            return None
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}")
+            return None
+
+
+    def _download_file_to_directory(self, file_id: str, local_dir: str) -> bool:
+        try:
+            self._ensure_service()
+            return self._download_single_file(file_id, local_dir)
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}", exc_info=True)
+            return False
+
+    def _ensure_service(self):
+        if not self.service:
+            try:
+                self.service = self.auth.build_drive_service(self.credentials)
+            except Exception as e:
+                raise ValueError(f"Cannot access Google Drive: {e}")
+
+    def _download_single_file(self, file_id: str, local_dir: str) -> bool:
+        file_metadata = self.service.files().get(
+            fileId=file_id,
+            fields='name,mimeType',
+            supportsAllDrives=True
+        ).execute()
+
+        file_name = file_metadata['name']
+        mime_type = file_metadata['mimeType']
+
+        if mime_type not in self.SUPPORTED_MIME_TYPES and not mime_type.startswith('application/vnd.google-apps.'):
+            return False
+
+        os.makedirs(local_dir, exist_ok=True)
+        full_path = os.path.join(local_dir, file_name)
+
+        if mime_type in self.EXPORT_FORMATS:
+            export_mime_type = self.EXPORT_FORMATS[mime_type]
+            request = self.service.files().export_media(
+                fileId=file_id,
+                mimeType=export_mime_type
+            )
+            extension = self._get_extension_for_mime_type(export_mime_type)
+            if not full_path.endswith(extension):
+                full_path += extension
+        else:
+            request = self.service.files().get_media(fileId=file_id, supportsAllDrives=True)
+
+        with open(full_path, 'wb') as f:
+            downloader = MediaIoBaseDownload(f, request)
+            done = False
+            while not done:
+                _, done = downloader.next_chunk()
+
+        return True
+
+    def _download_folder_recursive(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        files_downloaded = 0
+        try:
+            os.makedirs(local_dir, exist_ok=True)
+
+            query = f"'{folder_id}' in parents and trashed=false"
+            page_token = None
+
+            while True:
+                results = self.service.files().list(
+                    q=query,
+                    fields='nextPageToken, files(id, name, mimeType)',
+                    pageToken=page_token,
+                    pageSize=1000,
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True
+                ).execute()
+
+                items = results.get('files', [])
+                logging.info(f"Found {len(items)} items in folder {folder_id}")
+
+                for item in items:
+                    item_name = item['name']
+                    item_id = item['id']
+                    mime_type = item['mimeType']
+
+                    if mime_type == 'application/vnd.google-apps.folder':
+                        if recursive:
+                            # Create subfolder and recurse
+                            subfolder_path = os.path.join(local_dir, item_name)
+                            os.makedirs(subfolder_path, exist_ok=True)
+                            subfolder_files = self._download_folder_recursive(
+                                item_id,
+                                subfolder_path,
+                                recursive
+                            )
+                            files_downloaded += subfolder_files
+                            logging.info(f"Downloaded {subfolder_files} files from subfolder {item_name}")
+                    else:
+                        # Download file
+                        success = self._download_single_file(item_id, local_dir)
+                        if success:
+                            files_downloaded += 1
+                            logging.info(f"Downloaded file: {item_name}")
+                        else:
+                            logging.warning(f"Failed to download file: {item_name}")
+
+                page_token = results.get('nextPageToken')
+                if not page_token:
+                    break
+
+            return files_downloaded
+
+        except Exception as e:
+            logging.error(f"Error in _download_folder_recursive for folder {folder_id}: {e}", exc_info=True)
+            return files_downloaded
+
+    def _get_extension_for_mime_type(self, mime_type: str) -> str:
+        extensions = {
+            'application/pdf': '.pdf',
+            'text/plain': '.txt',
+            'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+            'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+            'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+            'text/html': '.html',
+            'text/markdown': '.md',
+        }
+        return extensions.get(mime_type, '.bin')
+
+    def _download_folder_contents(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        try:
+            self._ensure_service()
+            return self._download_folder_recursive(folder_id, local_dir, recursive)
+        except Exception as e:
+            logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+            return 0
+
+    def download_to_directory(self, local_dir: str, source_config: dict = None) -> dict:
+        if source_config is None:
+            source_config = {}
+
+        config = source_config if source_config else getattr(self, 'config', {})
+        files_downloaded = 0
+
+        try:
+            folder_ids = config.get('folder_ids', [])
+            file_ids = config.get('file_ids', [])
+            recursive = config.get('recursive', True)
+
+            self._ensure_service()
+
+            if file_ids:
+                if isinstance(file_ids, str):
+                    file_ids = [file_ids]
+
+                for file_id in file_ids:
+                    if self._download_file_to_directory(file_id, local_dir):
+                        files_downloaded += 1
+
+            # Process folders
+            if folder_ids:
+                if isinstance(folder_ids, str):
+                    folder_ids = [folder_ids]
+
+                for folder_id in folder_ids:
+                    try:
+                        folder_metadata = self.service.files().get(
+                            fileId=folder_id,
+                            fields='name',
+                            supportsAllDrives=True
+                        ).execute()
+                        folder_name = folder_metadata.get('name', '')
+                        folder_path = os.path.join(local_dir, folder_name)
+                        os.makedirs(folder_path, exist_ok=True)
+
+                        folder_files = self._download_folder_recursive(
+                            folder_id,
+                            folder_path,
+                            recursive
+                        )
+                        files_downloaded += folder_files
+                        logging.info(f"Downloaded {folder_files} files from folder {folder_name}")
+                    except Exception as e:
+                        logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+
+            if not file_ids and not folder_ids:
+                raise ValueError("No folder_ids or file_ids provided for download")
+
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": files_downloaded == 0,
+                "source_type": "google_drive",
+                "config_used": config
+            }
+
+        except Exception as e:
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": True,
+                "source_type": "google_drive",
+                "config_used": config,
+                "error": str(e)
+            }

application/parser/connectors/share_point/__init__.py

@@ -0,0 +1,10 @@
+"""
+Share Point connector package for DocsGPT.
+
+This module provides authentication and document loading capabilities for Share Point.
+"""
+
+from .auth import SharePointAuth
+from .loader import SharePointLoader
+
+__all__ = ['SharePointAuth', 'SharePointLoader']

application/parser/connectors/share_point/auth.py

@@ -0,0 +1,153 @@
+import datetime
+import logging
+from typing import Optional, Dict, Any
+
+from msal import ConfidentialClientApplication
+
+from application.core.settings import settings
+from application.parser.connectors._auth_utils import session_token_fingerprint
+from application.parser.connectors.base import BaseConnectorAuth
+
+logger = logging.getLogger(__name__)
+
+
+class SharePointAuth(BaseConnectorAuth):
+    """
+    Handles Microsoft OAuth 2.0 authentication for SharePoint/OneDrive.
+
+    Note: Files.Read scope allows access to files the user has granted access to,
+    similar to Google Drive's drive.file scope.
+    """
+
+    SCOPES = [
+        "Files.Read",
+        "Sites.Read.All",
+        "User.Read",
+    ]
+
+    def __init__(self):
+        self.client_id = settings.MICROSOFT_CLIENT_ID
+        self.client_secret = settings.MICROSOFT_CLIENT_SECRET
+
+        if not self.client_id:
+            raise ValueError(
+                "Microsoft OAuth credentials not configured. Please set MICROSOFT_CLIENT_ID in settings."
+            )
+        
+        if not self.client_secret:
+            raise ValueError(
+                "Microsoft OAuth credentials not configured. Please set MICROSOFT_CLIENT_SECRET in settings."
+            )
+
+        self.redirect_uri = settings.CONNECTOR_REDIRECT_BASE_URI
+        self.tenant_id = settings.MICROSOFT_TENANT_ID
+        self.authority = getattr(settings, "MICROSOFT_AUTHORITY", f"https://login.microsoftonline.com/{self.tenant_id}")
+
+        self.auth_app = ConfidentialClientApplication(
+            client_id=self.client_id,
+            client_credential=self.client_secret,
+            authority=self.authority
+        )
+
+    def get_authorization_url(self, state: Optional[str] = None) -> str:
+        return self.auth_app.get_authorization_request_url(
+            scopes=self.SCOPES, state=state, redirect_uri=self.redirect_uri
+        )
+
+    def exchange_code_for_tokens(self, authorization_code: str) -> Dict[str, Any]:
+        result = self.auth_app.acquire_token_by_authorization_code(
+            code=authorization_code,
+            scopes=self.SCOPES,
+            redirect_uri=self.redirect_uri
+        )
+
+        if "error" in result:
+            logger.error("Token exchange failed: %s", result.get("error_description"))
+            raise ValueError(f"Error acquiring token: {result.get('error_description')}")
+
+        return self.map_token_response(result)
+
+    def refresh_access_token(self, refresh_token: str) -> Dict[str, Any]:
+        result = self.auth_app.acquire_token_by_refresh_token(refresh_token=refresh_token, scopes=self.SCOPES)
+
+        if "error" in result:
+            logger.error("Token refresh failed: %s", result.get("error_description"))
+            raise ValueError(f"Error refreshing token: {result.get('error_description')}")
+
+        return self.map_token_response(result)
+
+    def get_token_info_from_session(self, session_token: str) -> Dict[str, Any]:
+        try:
+            from application.storage.db.repositories.connector_sessions import (
+                ConnectorSessionsRepository,
+            )
+            from application.storage.db.session import db_readonly
+
+            with db_readonly() as conn:
+                session = ConnectorSessionsRepository(conn).get_by_session_token(
+                    session_token
+                )
+
+            if not session:
+                raise ValueError(
+                    f"Invalid session token ({session_token_fingerprint(session_token)})"
+                )
+
+            token_info = session.get("token_info")
+            if not token_info:
+                raise ValueError("Session missing token information")
+
+            required_fields = ["access_token", "refresh_token"]
+            missing_fields = [field for field in required_fields if field not in token_info or not token_info.get(field)]
+            if missing_fields:
+                raise ValueError(f"Missing required token fields: {missing_fields}")
+
+            if 'token_uri' not in token_info:
+                token_info['token_uri'] = f"https://login.microsoftonline.com/{settings.MICROSOFT_TENANT_ID}/oauth2/v2.0/token"
+
+            return token_info
+
+        except Exception as e:
+            logger.error("Failed to retrieve token from session: %s", e)
+            raise ValueError(f"Failed to retrieve SharePoint token information: {str(e)}")
+
+    def is_token_expired(self, token_info: Dict[str, Any]) -> bool:
+        if not token_info:
+            return True
+
+        expiry_timestamp = token_info.get("expiry")
+
+        if expiry_timestamp is None:
+            return True
+
+        current_timestamp = int(datetime.datetime.now().timestamp())
+        return (expiry_timestamp - current_timestamp) < 60
+
+    def sanitize_token_info(self, token_info: Dict[str, Any], **extra_fields) -> Dict[str, Any]:
+        return super().sanitize_token_info(
+            token_info,
+            allows_shared_content=token_info.get("allows_shared_content", False),
+            **extra_fields,
+        )
+
+    PERSONAL_ACCOUNT_TENANT_ID = "9188040d-6c67-4c5b-b112-36a304b66dad"
+
+    def _allows_shared_content(self, id_token_claims: Dict[str, Any]) -> bool:
+        """Return True when the account is a work/school tenant that can access SharePoint shared content."""
+        tid = id_token_claims.get("tid", "")
+        return bool(tid) and tid != self.PERSONAL_ACCOUNT_TENANT_ID
+
+    def map_token_response(self, result) -> Dict[str, Any]:
+        claims = result.get("id_token_claims", {})
+        return {
+            "access_token": result.get("access_token"),
+            "refresh_token": result.get("refresh_token"),
+            "token_uri": claims.get("iss"),
+            "scopes": result.get("scope"),
+            "expiry": claims.get("exp"),
+            "allows_shared_content": self._allows_shared_content(claims),
+            "user_info": {
+                "name": claims.get("name"),
+                "email": claims.get("preferred_username"),
+            },
+        }

application/parser/connectors/share_point/loader.py

@@ -0,0 +1,650 @@
+"""
+SharePoint/OneDrive loader for DocsGPT.
+Loads documents from SharePoint/OneDrive using Microsoft Graph API.
+"""
+
+import functools
+import logging
+import os
+from typing import List, Dict, Any, Optional, Tuple
+from urllib.parse import quote
+
+import requests
+
+from application.parser.connectors.base import BaseConnectorLoader
+from application.parser.connectors.share_point.auth import SharePointAuth
+from application.parser.schema.base import Document
+
+
+def _retry_on_auth_failure(func):
+    """Retry once after refreshing the access token on 401/403 responses."""
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except requests.exceptions.HTTPError as e:
+            if e.response is not None and e.response.status_code in (401, 403):
+                logging.info(f"Auth failure in {func.__name__}, refreshing token and retrying")
+                try:
+                    new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                    self.access_token = new_token_info.get('access_token')
+                except Exception as refresh_error:
+                    raise ValueError(
+                        f"Authentication failed and could not be refreshed: {refresh_error}"
+                    ) from e
+                return func(self, *args, **kwargs)
+            raise
+    return wrapper
+
+
+class SharePointLoader(BaseConnectorLoader):
+
+    SUPPORTED_MIME_TYPES = {
+        'application/pdf': '.pdf',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+        'application/msword': '.doc',
+        'application/vnd.ms-powerpoint': '.ppt',
+        'application/vnd.ms-excel': '.xls',
+        'text/plain': '.txt',
+        'text/csv': '.csv',
+        'text/html': '.html',
+        'text/markdown': '.md',
+        'text/x-rst': '.rst',
+        'application/json': '.json',
+        'application/epub+zip': '.epub',
+        'application/rtf': '.rtf',
+        'image/jpeg': '.jpg',
+        'image/png': '.png',
+    }
+
+    EXTENSION_TO_MIME = {v: k for k, v in SUPPORTED_MIME_TYPES.items()}
+
+    GRAPH_API_BASE = "https://graph.microsoft.com/v1.0"
+
+    def __init__(self, session_token: str):
+        self.auth = SharePointAuth()
+        self.session_token = session_token
+
+        token_info = self.auth.get_token_info_from_session(session_token)
+        self.access_token = token_info.get('access_token')
+        self.refresh_token = token_info.get('refresh_token')
+        self.allows_shared_content = token_info.get('allows_shared_content', False)
+
+        if not self.access_token:
+            raise ValueError("No access token found in session")
+
+        self.next_page_token = None
+
+    def _get_headers(self) -> Dict[str, str]:
+        return {
+            'Authorization': f'Bearer {self.access_token}',
+            'Accept': 'application/json'
+        }
+
+    def _ensure_valid_token(self):
+        if not self.access_token:
+            raise ValueError("No access token available")
+
+        token_info = {'access_token': self.access_token, 'expiry': None}
+        if self.auth.is_token_expired(token_info):
+            logging.info("Token expired, attempting refresh")
+            try:
+                new_token_info = self.auth.refresh_access_token(self.refresh_token)
+                self.access_token = new_token_info.get('access_token')
+            except Exception:
+                raise ValueError("Failed to refresh access token")
+
+    def _get_item_url(self, item_ref: str) -> str:
+        if ':' in item_ref:
+            drive_id, item_id = item_ref.split(':', 1)
+            return f"{self.GRAPH_API_BASE}/drives/{drive_id}/items/{item_id}"
+        return f"{self.GRAPH_API_BASE}/me/drive/items/{item_ref}"
+
+    def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:
+        try:
+            drive_item_id = file_metadata.get('id')
+            file_name = file_metadata.get('name', 'Unknown')
+            file_data = file_metadata.get('file', {})
+            mime_type = file_data.get('mimeType', 'application/octet-stream')
+
+            if mime_type not in self.SUPPORTED_MIME_TYPES:
+                logging.info(f"Skipping unsupported file type: {mime_type} for file {file_name}")
+                return None
+
+            doc_metadata = {
+                'file_name': file_name,
+                'mime_type': mime_type,
+                'size': file_metadata.get('size'),
+                'created_time': file_metadata.get('createdDateTime'),
+                'modified_time': file_metadata.get('lastModifiedDateTime'),
+                'source': 'share_point'
+            }
+
+            if not load_content:
+                return Document(
+                    text="",
+                    doc_id=drive_item_id,
+                    extra_info=doc_metadata
+                )
+
+            content = self._download_file_content(drive_item_id)
+            if content is None:
+                logging.warning(f"Could not load content for file {file_name} ({drive_item_id})")
+                return None
+
+            return Document(
+                text=content,
+                doc_id=drive_item_id,
+                extra_info=doc_metadata
+            )
+
+        except Exception as e:
+            logging.error(f"Error processing file: {e}")
+            return None
+
+    def load_data(self, inputs: Dict[str, Any]) -> List[Document]:
+        try:
+            documents: List[Document] = []
+
+            folder_id = inputs.get('folder_id')
+            file_ids = inputs.get('file_ids', [])
+            limit = inputs.get('limit', 100)
+            list_only = inputs.get('list_only', False)
+            load_content = not list_only
+            page_token = inputs.get('page_token')
+            search_query = inputs.get('search_query')
+            self.next_page_token = None
+
+            shared = inputs.get('shared', False)
+
+            if file_ids:
+                for file_id in file_ids:
+                    try:
+                        doc = self._load_file_by_id(file_id, load_content=load_content)
+                        if doc:
+                            if not search_query or (
+                                search_query.lower() in doc.extra_info.get('file_name', '').lower()
+                            ):
+                                documents.append(doc)
+                    except Exception as e:
+                        logging.error(f"Error loading file {file_id}: {e}")
+                        continue
+            elif shared:
+                if not self.allows_shared_content:
+                    logging.warning("Shared content is only available for work/school Microsoft accounts")
+                    return []
+                documents = self._list_shared_items(
+                    limit=limit,
+                    load_content=load_content,
+                    page_token=page_token,
+                    search_query=search_query
+                )
+            else:
+                parent_id = folder_id if folder_id else 'root'
+                documents = self._list_items_in_parent(
+                    parent_id,
+                    limit=limit,
+                    load_content=load_content,
+                    page_token=page_token,
+                    search_query=search_query
+                )
+
+            logging.info(f"Loaded {len(documents)} documents from SharePoint/OneDrive")
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error loading data from SharePoint/OneDrive: {e}", exc_info=True)
+            raise
+
+    @_retry_on_auth_failure
+    def _load_file_by_id(self, file_id: str, load_content: bool = True) -> Optional[Document]:
+        self._ensure_valid_token()
+
+        try:
+            url = self._get_item_url(file_id)
+            params = {'$select': 'id,name,file,createdDateTime,lastModifiedDateTime,size'}
+            response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+            response.raise_for_status()
+
+            file_metadata = response.json()
+            return self._process_file(file_metadata, load_content=load_content)
+
+        except requests.exceptions.HTTPError:
+            raise
+        except Exception as e:
+            logging.error(f"Error loading file {file_id}: {e}")
+            return None
+
+    @_retry_on_auth_failure
+    def _list_items_in_parent(self, parent_id: str, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:
+        self._ensure_valid_token()
+
+        documents: List[Document] = []
+
+        try:
+            url = f"{self._get_item_url(parent_id)}/children"
+            params = {'$top': min(100, limit) if limit else 100, '$select': 'id,name,file,folder,createdDateTime,lastModifiedDateTime,size'}
+            if page_token:
+                params['$skipToken'] = page_token
+
+            if search_query:
+                encoded_query = quote(search_query, safe='')
+                if ':' in parent_id:
+                    drive_id = parent_id.split(':', 1)[0]
+                    search_url = f"{self.GRAPH_API_BASE}/drives/{drive_id}/root/search(q='{encoded_query}')"
+                else:
+                    search_url = f"{self.GRAPH_API_BASE}/me/drive/search(q='{encoded_query}')"
+                response = requests.get(search_url, headers=self._get_headers(), params=params, timeout=100)
+            else:
+                response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+
+            response.raise_for_status()
+
+            results = response.json()
+
+            items = results.get('value', [])
+            for item in items:
+                if 'folder' in item:
+                    doc_metadata = {
+                        'file_name': item.get('name', 'Unknown'),
+                        'mime_type': 'folder',
+                        'size': item.get('size'),
+                        'created_time': item.get('createdDateTime'),
+                        'modified_time': item.get('lastModifiedDateTime'),
+                        'source': 'share_point',
+                        'is_folder': True
+                    }
+                    documents.append(Document(text="", doc_id=item.get('id'), extra_info=doc_metadata))
+                else:
+                    doc = self._process_file(item, load_content=load_content)
+                    if doc:
+                        documents.append(doc)
+
+                if limit and len(documents) >= limit:
+                    break
+
+            next_link = results.get('@odata.nextLink')
+            if next_link:
+                from urllib.parse import urlparse, parse_qs
+                parsed = urlparse(next_link)
+                query_params = parse_qs(parsed.query)
+                skiptoken_list = query_params.get('$skiptoken')
+                if skiptoken_list:
+                    self.next_page_token = skiptoken_list[0]
+                else:
+                    self.next_page_token = None
+            else:
+                self.next_page_token = None
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error listing items under parent {parent_id}: {e}")
+            return documents
+
+
+
+
+    def _resolve_mime_type(self, resource: Dict[str, Any]) -> Tuple[str, bool]:
+        """Resolve mime type from resource, falling back to file extension."""
+        file_data = resource.get('file', {})
+        mime_type = file_data.get('mimeType') if file_data else None
+
+        if mime_type and mime_type in self.SUPPORTED_MIME_TYPES:
+            return mime_type, True
+
+        name = resource.get('name', '')
+        ext = os.path.splitext(name)[1].lower()
+        if ext in self.EXTENSION_TO_MIME:
+            return self.EXTENSION_TO_MIME[ext], True
+
+        return mime_type or 'application/octet-stream', False
+
+    def _get_user_drive_web_url(self) -> Optional[str]:
+        """Fetch the current user's OneDrive web URL for KQL path exclusion."""
+        try:
+            response = requests.get(
+                f"{self.GRAPH_API_BASE}/me/drive",
+                headers=self._get_headers(),
+                params={'$select': 'webUrl'},
+                timeout=100,
+            )
+            response.raise_for_status()
+            return response.json().get('webUrl')
+        except Exception as e:
+            logging.warning(f"Could not fetch user drive web URL: {e}")
+            return None
+
+    def _build_shared_kql_query(self, search_query: Optional[str], user_drive_url: Optional[str]) -> str:
+        """Build KQL query string that excludes the user's own drive items."""
+        base_query = search_query if search_query else "*"
+        if user_drive_url:
+            return f'{base_query} AND -path:"{user_drive_url}"'
+        return base_query
+
+    def _list_shared_items(self, limit: int = 100, load_content: bool = False, page_token: Optional[str] = None, search_query: Optional[str] = None) -> List[Document]:
+        """Fetch shared drive items using Microsoft Graph Search API with local offset paging.
+
+        We always fetch up to a fixed maximum number of hits from Graph (single request),
+        then page through that array locally using `page_token` as a simple integer offset.
+        This avoids relying on buggy or inconsistent remote `from`/`size` semantics.
+        """
+        self._ensure_valid_token()
+        documents: List[Document] = []
+
+        try:
+            user_drive_url = self._get_user_drive_web_url()
+            query_text = self._build_shared_kql_query(search_query, user_drive_url)
+
+            url = f"{self.GRAPH_API_BASE}/search/query"
+            page_size = 500  # maximum number of hits we care about for selection
+
+            body = {
+                "requests": [
+                    {
+                        "entityTypes": ["driveItem"],
+                        "query": {"queryString": query_text},
+                        "from": 0,
+                        "size": page_size,
+                    }
+                ]
+            }
+
+            headers = self._get_headers()
+            headers["Content-Type"] = "application/json"
+            response = requests.post(url, headers=headers, json=body, timeout=100)
+            response.raise_for_status()
+            results = response.json()
+
+            search_response = results.get("value", [])
+            if not search_response:
+                logging.warning("Search API returned empty value array")
+                self.next_page_token = None
+                return documents
+
+            hits_containers = search_response[0].get("hitsContainers", [])
+            if not hits_containers:
+                logging.warning("Search API returned no hitsContainers")
+                self.next_page_token = None
+                return documents
+
+            container = hits_containers[0]
+            total = container.get("total", 0)
+            raw_hits = container.get("hits", [])
+
+            # Deduplicate by effective item ID (driveId:itemId) to avoid the same
+            # resource appearing multiple times across the result set.
+            deduped_hits = []
+            seen_ids = set()
+            for hit in raw_hits:
+                resource = hit.get("resource", {})
+                item_id = resource.get("id")
+                drive_id = resource.get("parentReference", {}).get("driveId")
+                effective_id = f"{drive_id}:{item_id}" if drive_id and item_id else item_id
+                if not effective_id or effective_id in seen_ids:
+                    continue
+                seen_ids.add(effective_id)
+                deduped_hits.append(hit)
+
+            hits = deduped_hits
+            logging.info(
+                f"Search API returned {total} total results, {len(raw_hits)} raw hits, {len(hits)} unique hits in this batch"
+            )
+            try:
+                offset = int(page_token) if page_token is not None else 0
+            except (TypeError, ValueError):
+                logging.warning(
+                    f"Invalid page_token '{page_token}' for shared items search, defaulting to 0"
+                )
+                offset = 0
+
+            if offset < 0:
+                offset = 0
+            if offset >= len(hits):
+                self.next_page_token = None
+                return documents
+
+            end_index = offset + limit if limit else len(hits)
+            end_index = min(end_index, len(hits))
+
+            for hit in hits[offset:end_index]:
+                resource = hit.get("resource", {})
+                item_name = resource.get("name", "Unknown")
+                item_id = resource.get("id")
+                drive_id = resource.get("parentReference", {}).get("driveId")
+
+                effective_id = f"{drive_id}:{item_id}" if drive_id and item_id else item_id
+
+                is_folder = "folder" in resource
+
+                if is_folder:
+                    doc_metadata = {
+                        "file_name": item_name,
+                        "mime_type": "folder",
+                        "size": resource.get("size"),
+                        "created_time": resource.get("createdDateTime"),
+                        "modified_time": resource.get("lastModifiedDateTime"),
+                        "source": "share_point",
+                        "is_folder": True,
+                    }
+                    documents.append(
+                        Document(text="", doc_id=effective_id, extra_info=doc_metadata)
+                    )
+                else:
+                    mime_type, supported = self._resolve_mime_type(resource)
+                    if not supported:
+                        logging.info(
+                            f"Skipping unsupported shared file: {item_name} (mime: {mime_type})"
+                        )
+                        continue
+
+                    doc_metadata = {
+                        "file_name": item_name,
+                        "mime_type": mime_type,
+                        "size": resource.get("size"),
+                        "created_time": resource.get("createdDateTime"),
+                        "modified_time": resource.get("lastModifiedDateTime"),
+                        "source": "share_point",
+                    }
+
+                    content = ""
+                    if load_content:
+                        content = self._download_file_content(effective_id) or ""
+
+                    documents.append(
+                        Document(text=content, doc_id=effective_id, extra_info=doc_metadata)
+                    )
+
+            if limit and end_index < len(hits):
+                self.next_page_token = str(end_index)
+            else:
+                self.next_page_token = None
+
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error listing shared items via search API: {e}", exc_info=True)
+            return documents
+
+    @_retry_on_auth_failure
+    def _download_file_content(self, file_id: str) -> Optional[str]:
+        self._ensure_valid_token()
+
+        try:
+            url = f"{self._get_item_url(file_id)}/content"
+            response = requests.get(url, headers=self._get_headers(), timeout=100)
+            response.raise_for_status()
+
+            try:
+                return response.content.decode('utf-8')
+            except UnicodeDecodeError:
+                logging.error(f"Could not decode file {file_id} as text")
+                return None
+
+        except requests.exceptions.HTTPError:
+            raise
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}")
+            return None
+
+    def _download_single_file(self, file_id: str, local_dir: str) -> bool:
+        try:
+            url = self._get_item_url(file_id)
+            params = {'$select': 'id,name,file'}
+            response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+            response.raise_for_status()
+
+            metadata = response.json()
+            file_name = metadata.get('name', 'unknown')
+            file_data = metadata.get('file', {})
+            mime_type = file_data.get('mimeType', 'application/octet-stream')
+
+            if mime_type not in self.SUPPORTED_MIME_TYPES:
+                logging.info(f"Skipping unsupported file type: {mime_type}")
+                return False
+
+            os.makedirs(local_dir, exist_ok=True)
+            full_path = os.path.join(local_dir, file_name)
+
+            download_url = f"{self._get_item_url(file_id)}/content"
+            download_response = requests.get(download_url, headers=self._get_headers(), timeout=100)
+            download_response.raise_for_status()
+
+            with open(full_path, 'wb') as f:
+                f.write(download_response.content)
+
+            return True
+        except Exception as e:
+            logging.error(f"Error in _download_single_file: {e}")
+            return False
+
+    def _download_folder_recursive(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        files_downloaded = 0
+        try:
+            os.makedirs(local_dir, exist_ok=True)
+
+            url = f"{self._get_item_url(folder_id)}/children"
+            params = {'$top': 1000}
+
+            while url:
+                response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+                response.raise_for_status()
+
+                results = response.json()
+                items = results.get('value', [])
+                logging.info(f"Found {len(items)} items in folder {folder_id}")
+
+                for item in items:
+                    item_name = item.get('name', 'unknown')
+                    item_id = item.get('id')
+
+                    if 'folder' in item:
+                        if recursive:
+                            subfolder_path = os.path.join(local_dir, item_name)
+                            os.makedirs(subfolder_path, exist_ok=True)
+                            subfolder_files = self._download_folder_recursive(
+                                item_id,
+                                subfolder_path,
+                                recursive
+                            )
+                            files_downloaded += subfolder_files
+                            logging.info(f"Downloaded {subfolder_files} files from subfolder {item_name}")
+                    else:
+                        success = self._download_single_file(item_id, local_dir)
+                        if success:
+                            files_downloaded += 1
+                            logging.info(f"Downloaded file: {item_name}")
+                        else:
+                            logging.warning(f"Failed to download file: {item_name}")
+
+                url = results.get('@odata.nextLink')
+
+            return files_downloaded
+
+        except Exception as e:
+            logging.error(f"Error in _download_folder_recursive for folder {folder_id}: {e}", exc_info=True)
+            return files_downloaded
+
+    def _download_folder_contents(self, folder_id: str, local_dir: str, recursive: bool = True) -> int:
+        try:
+            self._ensure_valid_token()
+            return self._download_folder_recursive(folder_id, local_dir, recursive)
+        except Exception as e:
+            logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+            return 0
+
+    def _download_file_to_directory(self, file_id: str, local_dir: str) -> bool:
+        try:
+            self._ensure_valid_token()
+            return self._download_single_file(file_id, local_dir)
+        except Exception as e:
+            logging.error(f"Error downloading file {file_id}: {e}", exc_info=True)
+            return False
+
+    def download_to_directory(self, local_dir: str, source_config: Dict[str, Any] = None) -> Dict[str, Any]:
+        if source_config is None:
+            source_config = {}
+
+        config = source_config if source_config else getattr(self, 'config', {})
+        files_downloaded = 0
+
+        try:
+            folder_ids = config.get('folder_ids', [])
+            file_ids = config.get('file_ids', [])
+            recursive = config.get('recursive', True)
+
+            if file_ids:
+                if isinstance(file_ids, str):
+                    file_ids = [file_ids]
+
+                for file_id in file_ids:
+                    if self._download_file_to_directory(file_id, local_dir):
+                        files_downloaded += 1
+
+            if folder_ids:
+                if isinstance(folder_ids, str):
+                    folder_ids = [folder_ids]
+
+                for folder_id in folder_ids:
+                    try:
+                        url = self._get_item_url(folder_id)
+                        params = {'$select': 'id,name'}
+                        response = requests.get(url, headers=self._get_headers(), params=params, timeout=100)
+                        response.raise_for_status()
+
+                        folder_metadata = response.json()
+                        folder_name = folder_metadata.get('name', '')
+                        folder_path = os.path.join(local_dir, folder_name)
+                        os.makedirs(folder_path, exist_ok=True)
+
+                        folder_files = self._download_folder_recursive(
+                            folder_id,
+                            folder_path,
+                            recursive
+                        )
+                        files_downloaded += folder_files
+                        logging.info(f"Downloaded {folder_files} files from folder {folder_name}")
+                    except Exception as e:
+                        logging.error(f"Error downloading folder {folder_id}: {e}", exc_info=True)
+
+            if not file_ids and not folder_ids:
+                raise ValueError("No folder_ids or file_ids provided for download")
+
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": files_downloaded == 0,
+                "source_type": "share_point",
+                "config_used": config
+            }
+
+        except Exception as e:
+            return {
+                "files_downloaded": files_downloaded,
+                "directory_path": local_dir,
+                "empty_result": True,
+                "source_type": "share_point",
+                "config_used": config,
+                "error": str(e)
+            }

application/parser/document_reader.py

@@ -0,0 +1,508 @@
+"""In-process document parsing for the ``read_document`` tool, run on the Celery parsing worker.
+
+``parse_document_bytes`` turns untrusted document bytes into a bounded, shaped
+result (markdown/text/structured/chunks) using the BACKEND parsers (Docling by
+default). It applies the same untrusted-content safeguards as uploads — an
+extension whitelist, a byte cap, ``safe_filename`` staging into a temp file, and
+temp cleanup — so a hostile filename or document is treated as inert data.
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import os
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any, Dict, Iterator, List, Optional
+
+from application.core.settings import settings
+from application.parser.file.bulk import get_default_file_extractor
+from application.parser.file.constants import SUPPORTED_SOURCE_EXTENSIONS
+from application.utils import safe_filename
+
+logger = logging.getLogger(__name__)
+
+# Default cap for the LLM-facing VIEW of the text (applied in ``bound_parse_payload``,
+# NOT during parsing) so a huge document can't flood context; the full result is still
+# persisted as a ``data`` artifact. When the text exceeds the cap a head+tail window keeps
+# both the document's beginning AND end (e.g. totals/signatures) within the byte budget.
+_TEXT_MAX_BYTES = 8000
+_MAX_TABLES_RETURNED = 20
+_MAX_TABLE_ROWS = 50
+_MAX_CELL_CHARS = 200
+# Caps applied to the bounded view that rides back through the Redis result
+# backend (the full result still lives in the persisted artifact).
+_MAX_CHUNKS_RETURNED = 50
+_MAX_PAGE_SELECTIONS = 10_000
+_MAX_PAGE_SELECTOR_TOKENS = 20_000
+
+_VALID_OUTPUTS = ("markdown", "text", "structured", "chunks")
+_VALID_OCR = ("auto", "on", "off")
+_VALID_ENGINES = ("auto", "docling", "fast")
+
+
+def truncate_text_head_tail(text: str, max_bytes: Optional[int] = None) -> str:
+    """Bound text to a head+tail byte window so a large file can't flood context."""
+    cap = int(max_bytes or _TEXT_MAX_BYTES)
+    if cap <= 0:
+        return text
+    encoded = text.encode("utf-8")
+    if len(encoded) <= cap:
+        return text
+    head = cap // 2
+    tail = cap - head
+    dropped = len(encoded) - head - tail
+    head_text = encoded[:head].decode("utf-8", errors="ignore")
+    tail_text = encoded[-tail:].decode("utf-8", errors="ignore")
+    return f"{head_text}\n\n...[truncated {dropped} bytes]...\n\n{tail_text}"
+
+
+def bound_parse_payload(payload: Dict[str, Any], max_chars: Optional[int] = None) -> Dict[str, Any]:
+    """Bound every shape of a parse payload so the Redis-backed result stays small.
+
+    This is where ALL view-bounding happens: parsing now returns the FULL content and the
+    persisted ``data`` artifact keeps it, while the view ridden back through the Redis result
+    backend is bounded here. ``content`` is capped to ``max_chars`` when given, else re-windowed
+    to a head+tail byte window; ``chunks`` is capped in count and per-chunk length. ``structured``
+    is left as-is: it rides back so json_schema validation in the tool can run against it, and it
+    is already bounded by the input byte cap plus the table caps (``_compact_table`` /
+    ``summary``). ``payload['truncated']`` is set when the content view actually cut. The dict is
+    mutated in place.
+    """
+    content = payload.get("content")
+    if isinstance(content, str):
+        if max_chars and int(max_chars) > 0:
+            capped = content[: int(max_chars)]
+        else:
+            capped = truncate_text_head_tail(content)
+        payload["content"] = capped
+        payload["truncated"] = capped != content
+
+    chunks = payload.get("chunks")
+    if isinstance(chunks, list):
+        bounded = [
+            truncate_text_head_tail(chunk) if isinstance(chunk, str) else chunk
+            for chunk in chunks[:_MAX_CHUNKS_RETURNED]
+        ]
+        if len(chunks) > _MAX_CHUNKS_RETURNED:
+            payload["chunks_truncated"] = True
+            payload["total_chunks"] = len(chunks)
+        payload["chunks"] = bounded
+
+    return payload
+
+
+def _max_input_bytes() -> int:
+    """Return the size cap for a parsed document (its own setting, else the sandbox cap)."""
+    explicit = int(getattr(settings, "DOCUMENT_PARSE_MAX_BYTES", 0) or 0)
+    if explicit > 0:
+        return explicit
+    return int(getattr(settings, "SANDBOX_MAX_INPUT_BYTES", 25 * 1024 * 1024))
+
+
+_ZIP_CONTAINER_EXTENSIONS = frozenset({".docx", ".xlsx", ".pptx", ".epub"})
+
+
+def _reject_zip_bomb(data: bytes, suffix: str) -> Optional[str]:
+    """Return an error string if a zip-based document declares an implausible expansion, else None."""
+    if suffix not in _ZIP_CONTAINER_EXTENSIONS:
+        return None
+    max_entries = int(getattr(settings, "DOCUMENT_MAX_ARCHIVE_ENTRIES", 10000))
+    cap = int(getattr(settings, "DOCUMENT_MAX_DECOMPRESSED_BYTES", 300 * 1024 * 1024))
+    try:
+        with zipfile.ZipFile(io.BytesIO(data)) as zf:
+            infos = zf.infolist()
+            if len(infos) > max_entries:
+                return f"document archive has too many entries ({len(infos)} > {max_entries})."
+            total = 0
+            for info in infos:
+                total += info.file_size
+                if total > cap:
+                    return f"document decompresses to too much data: exceeds the {cap}-byte cap."
+    except zipfile.BadZipFile:
+        # Not a readable zip; the format-specific parser will surface a clean error.
+        return None
+    return None
+
+
+def _resolve_ocr_enabled(ocr: str) -> bool:
+    """Resolve the OCR flag from the ``ocr`` arg and the deployment setting."""
+    if ocr == "on":
+        return True
+    if ocr == "off":
+        return False
+    return bool(getattr(settings, "DOCLING_OCR_ENABLED", False))
+
+
+def _pick_parser(suffix: str, *, ocr_enabled: bool, engine: str):
+    """Select the parser for ``suffix`` honoring the requested engine; None when unsupported."""
+    if engine == "fast":
+        legacy = _legacy_parser_for(suffix)
+        if legacy is not None:
+            return legacy
+    extractor = get_default_file_extractor(ocr_enabled=ocr_enabled)
+    return extractor.get(suffix)
+
+
+def _legacy_parser_for(suffix: str):
+    """Return a non-Docling parser for ``suffix`` (the ``fast`` engine), or None."""
+    from application.parser.file.docs_parser import DocxParser, PDFParser
+    from application.parser.file.html_parser import HTMLParser
+    from application.parser.file.markdown_parser import MarkdownParser
+    from application.parser.file.tabular_parser import ExcelParser, PandasCSVParser
+
+    legacy = {
+        ".pdf": PDFParser,
+        ".docx": DocxParser,
+        ".csv": PandasCSVParser,
+        ".xlsx": ExcelParser,
+        ".html": HTMLParser,
+        ".md": MarkdownParser,
+        ".mdx": MarkdownParser,
+    }
+    cls = legacy.get(suffix)
+    return cls() if cls is not None else None
+
+
+def _parse_to_text(parser: Any, path: Path) -> str:
+    """Run a parser and coerce its ``str | List[str]`` result to a single text blob."""
+    if not parser.parser_config_set:
+        parser.init_parser()
+    parsed = parser.parse_file(path, errors="ignore")
+    if isinstance(parsed, list):
+        return "\n\n".join(str(part) for part in parsed)
+    return str(parsed)
+
+
+def _is_docling_parser(parser: Any) -> bool:
+    """True when ``parser`` is Docling-backed (collecting tables would otherwise re-convert)."""
+    if parser is None:
+        return False
+    try:
+        from application.parser.file.docling_parser import DoclingParser
+    except Exception:
+        return False
+    return isinstance(parser, DoclingParser)
+
+
+def _compact_table(table: Dict[str, Any]) -> Dict[str, Any]:
+    """Bound a single table's rows and cell sizes so one giant table can't bloat context."""
+
+    def _cell(value: Any) -> Any:
+        if isinstance(value, str) and len(value) > _MAX_CELL_CHARS:
+            return value[:_MAX_CELL_CHARS] + "...[truncated]"
+        return value
+
+    rows = table.get("rows")
+    if not isinstance(rows, list):
+        return table
+    capped = [[_cell(c) for c in row] if isinstance(row, list) else _cell(row) for row in rows[:_MAX_TABLE_ROWS]]
+    compact = dict(table)
+    compact["rows"] = capped
+    if len(rows) > _MAX_TABLE_ROWS:
+        compact["rows_truncated"] = True
+        compact["total_rows"] = len(rows)
+    return compact
+
+
+def _docling_structured(path: Path, *, ocr_enabled: bool, include_tables: bool, parser: Any = None) -> Dict[str, Any]:
+    """Convert a document with Docling and return markdown + structured dict + bounded tables.
+
+    When ``parser`` is the configured ``DoclingParser`` (the collapse-the-double-convert
+    path), reuse ITS converter + export so OCR/pipeline options and the export fallback
+    are honored and the content matches the legacy single-parse output exactly; otherwise
+    fall back to a vanilla converter.
+    """
+    if _is_docling_parser(parser):
+        if getattr(parser, "_converter", None) is None:
+            parser._init_parser()
+        doc = parser._converter.convert(str(path)).document
+        markdown = parser._export_content(doc)
+    else:
+        from docling.document_converter import DocumentConverter
+
+        converter = DocumentConverter()
+        doc = converter.convert(str(path)).document
+        markdown = doc.export_to_markdown()
+    structured = doc.export_to_dict()
+    tables: List[Dict[str, Any]] = []
+    if include_tables:
+        for tbl in getattr(doc, "tables", []) or []:
+            try:
+                df = tbl.export_to_dataframe()
+                tables.append({"columns": [str(c) for c in df.columns], "rows": df.astype(str).values.tolist()})
+            except Exception:
+                try:
+                    tables.append({"markdown": tbl.export_to_markdown()})
+                except Exception:
+                    continue
+            if len(tables) >= _MAX_TABLES_RETURNED:
+                break
+    page_count = len(getattr(doc, "pages", {}) or {})
+    return {"markdown": markdown, "structured": structured, "tables": tables, "page_count": page_count}
+
+
+def _structure_summary(structured: Any) -> Dict[str, int]:
+    """Summarize the Docling structured dict by top-level element counts (keeps context compact)."""
+    if not isinstance(structured, dict):
+        return {}
+    counts: Dict[str, int] = {}
+    for key in ("texts", "tables", "pictures", "groups", "pages"):
+        value = structured.get(key)
+        if isinstance(value, (list, dict)):
+            counts[key] = len(value)
+    return counts
+
+
+def _apply_pages(text: str, pages: Any) -> str:
+    """Best-effort page-range slice on a page-delimited markdown blob (``\\f`` separated)."""
+    if not pages or "\f" not in text:
+        return text
+    total = text.count("\f") + 1
+    selected = _selected_page_indices(pages, total)
+    if not selected:
+        return text
+
+    # Splitting a hostile form-feed blob could allocate millions of strings.
+    # Walk boundaries and retain only the bounded set of requested pages.
+    wanted = set(selected)
+    found: Dict[int, tuple[int, int]] = {}
+    page_index = 0
+    start = 0
+    while wanted:
+        end = text.find("\f", start)
+        if end < 0:
+            end = len(text)
+        if page_index in wanted:
+            found[page_index] = (start, end)
+            wanted.remove(page_index)
+        if end == len(text):
+            break
+        page_index += 1
+        start = end + 1
+
+    # Preserve requested order and ordinary duplicates, but never let repeated
+    # selectors amplify the output beyond the source text's resident size.
+    output = io.StringIO()
+    output_chars = 0
+    wrote_page = False
+    for index in selected:
+        span = found.get(index)
+        if span is None:
+            continue
+        page_start, page_end = span
+        added = (page_end - page_start) + (1 if wrote_page else 0)
+        if output_chars + added > len(text):
+            break
+        if wrote_page:
+            output.write("\f")
+        output.write(text[page_start:page_end])
+        output_chars += added
+        wrote_page = True
+    return output.getvalue() if wrote_page else text
+
+
+def _iter_page_tokens(pages: Any) -> Iterator[Any]:
+    """Yield bounded selector tokens without materializing a comma-split list."""
+    if isinstance(pages, list):
+        for position, token in enumerate(pages):
+            if position >= _MAX_PAGE_SELECTOR_TOKENS:
+                break
+            yield token
+        return
+
+    raw = str(pages)
+    start = 0
+    emitted = 0
+    while emitted < _MAX_PAGE_SELECTOR_TOKENS:
+        end = raw.find(",", start)
+        if end < 0:
+            yield raw[start:]
+            return
+        yield raw[start:end]
+        emitted += 1
+        start = end + 1
+
+
+def _selected_page_indices(pages: Any, total: int) -> List[int]:
+    """Parse ``pages`` into a bounded list of valid 0-based page occurrences."""
+    if total <= 0:
+        return []
+
+    indices: List[int] = []
+    for token in _iter_page_tokens(pages):
+        if len(indices) >= _MAX_PAGE_SELECTIONS:
+            break
+        token = str(token).strip()
+        if "-" in token:
+            try:
+                lo, hi = (int(p) for p in token.split("-", 1))
+            except ValueError:
+                continue
+            start = max(lo - 1, 0)
+            stop = min(hi, total, start + (_MAX_PAGE_SELECTIONS - len(indices)))
+            indices.extend(range(start, stop))
+        else:
+            try:
+                index = int(token) - 1
+            except ValueError:
+                continue
+            if 0 <= index < total:
+                indices.append(index)
+    return indices
+
+
+def _to_chunks(text: str, max_chars: Optional[int]) -> List[str]:
+    """Chunk parsed text via the ingestion chunker; bounded and JSON-safe for the result."""
+    from application.parser.chunking_creator import ChunkerCreator
+    from application.parser.schema.base import Document
+
+    chunker = ChunkerCreator.create_chunker("classic_chunk")
+    chunks = chunker.chunk([Document(text=text)])
+    cap = int(max_chars or 0)
+    out: List[str] = []
+    for chunk in chunks:
+        body = getattr(chunk, "text", str(chunk))
+        out.append(body[:cap] if cap > 0 else body)
+        if len(out) >= 200:
+            break
+    return out
+
+
+def parse_document_bytes(
+    data: bytes,
+    filename: str,
+    *,
+    output: str = "markdown",
+    ocr: str = "auto",
+    pages: Any = None,
+    engine: str = "auto",
+    max_chars: Optional[int] = None,
+    include_tables: bool = True,
+) -> Dict[str, Any]:
+    """Parse untrusted document bytes into a bounded shaped result; whitelist + size + cleanup guarded."""
+    if output not in _VALID_OUTPUTS:
+        return {"error": f"unsupported output '{output}'; expected one of {_VALID_OUTPUTS}."}
+    if ocr not in _VALID_OCR:
+        return {"error": f"unsupported ocr '{ocr}'; expected one of {_VALID_OCR}."}
+    if engine not in _VALID_ENGINES:
+        return {"error": f"unsupported engine '{engine}'; expected one of {_VALID_ENGINES}."}
+
+    safe_name = safe_filename(filename) or "document"
+    suffix = os.path.splitext(safe_name)[1].lower()
+    if suffix not in SUPPORTED_SOURCE_EXTENSIONS:
+        return {"error": f"unsupported file type '{suffix or filename}'."}
+
+    cap = _max_input_bytes()
+    if len(data) > cap:
+        return {"error": f"input document is too large: {len(data)} bytes exceeds the {cap}-byte cap."}
+
+    bomb = _reject_zip_bomb(data, suffix)
+    if bomb is not None:
+        return {"error": bomb}
+
+    ocr_enabled = _resolve_ocr_enabled(ocr)
+    tmp_dir = tempfile.mkdtemp(prefix="docparse-")
+    tmp_path = Path(tmp_dir) / safe_name
+    try:
+        tmp_path.write_bytes(data)
+        return _shape(tmp_path, suffix, output, ocr_enabled, engine, pages, max_chars, include_tables)
+    except Exception as exc:
+        logger.exception("parse_document_bytes: parsing failed")
+        return {"error": f"parsing failed: {type(exc).__name__}: {exc}"}
+    finally:
+        try:
+            tmp_path.unlink(missing_ok=True)
+            os.rmdir(tmp_dir)
+        except OSError:
+            logger.warning("parse_document_bytes: temp cleanup failed for %s", tmp_dir, exc_info=True)
+
+
+def _shape(
+    path: Path,
+    suffix: str,
+    output: str,
+    ocr_enabled: bool,
+    engine: str,
+    pages: Any,
+    max_chars: Optional[int],
+    include_tables: bool,
+) -> Dict[str, Any]:
+    """Run the selected parser/engine and shape the result per ``output``; bounded throughout.
+
+    ``output='structured'`` always uses Docling regardless of ``engine`` — the ``fast``
+    engine is markdown/text only and cannot produce the structured dict.
+    """
+    if output == "structured":
+        try:
+            extracted = _docling_structured(path, ocr_enabled=ocr_enabled, include_tables=include_tables)
+        except Exception as exc:
+            return {"error": f"structured parsing requires Docling: {type(exc).__name__}: {exc}"}
+        bounded, truncated = _bounded(extracted["markdown"])
+        return {
+            "output": "structured",
+            "content": bounded,
+            "truncated": truncated,
+            "tables": [_compact_table(t) for t in extracted["tables"]],
+            "structured": extracted["structured"],
+            "summary": _structure_summary(extracted["structured"]),
+            "page_count": extracted["page_count"],
+        }
+
+    parser = _pick_parser(suffix, ocr_enabled=ocr_enabled, engine=engine)
+    wants_tables = include_tables and engine != "fast" and output != "chunks"
+
+    # A Docling-backed parser already converts the whole document to produce its text.
+    # When tables are also requested, reuse that single conversion for both the markdown
+    # content and the tables instead of converting a second time just to collect tables
+    # (Docling/torch conversion dominates the cost, so a re-convert ~doubles latency).
+    if wants_tables and _is_docling_parser(parser):
+        try:
+            extracted = _docling_structured(path, ocr_enabled=ocr_enabled, include_tables=True, parser=parser)
+            text = extracted["markdown"]
+            tables: List[Dict[str, Any]] = [_compact_table(t) for t in extracted["tables"]]
+        except Exception:
+            text, tables = _parse_to_text(parser, path), []
+        text = _apply_pages(text, pages)
+        bounded, truncated = _bounded(text)
+        payload: Dict[str, Any] = {"output": output, "content": bounded, "truncated": truncated}
+        if tables:
+            payload["tables"] = tables
+        return payload
+
+    if parser is None:
+        # A whitelisted extension with no dedicated parser (e.g. .txt) reads as plain
+        # text, matching SimpleDirectoryReader's standard-read fallback.
+        text = path.read_text(errors="ignore")
+    else:
+        text = _parse_to_text(parser, path)
+    text = _apply_pages(text, pages)
+
+    if output == "chunks":
+        return {"output": "chunks", "chunks": _to_chunks(text, max_chars), "truncated": False}
+
+    tables: List[Dict[str, Any]] = []
+    if wants_tables:
+        try:
+            tables = [
+                _compact_table(t)
+                for t in _docling_structured(path, ocr_enabled=ocr_enabled, include_tables=True)["tables"]
+            ]
+        except Exception:
+            tables = []
+    bounded, truncated = _bounded(text)
+    payload: Dict[str, Any] = {"output": output, "content": bounded, "truncated": truncated}
+    if tables:
+        payload["tables"] = tables
+    return payload
+
+
+def _bounded(text: str) -> tuple[str, bool]:
+    """Return the FULL extracted text (never truncated here); the view is bounded in ``bound_parse_payload``.
+
+    Parsing keeps the complete text so the persisted ``data`` artifact is the full parse;
+    ``max_chars`` and the default head+tail window now bound only the LLM-facing view.
+    """
+    return text, False

application/parser/embedding_pipeline.py

@@ -0,0 +1,339 @@
+import os
+import logging
+from typing import Any, List, Optional
+from retry import retry
+from tqdm import tqdm
+from application.core.settings import settings
+from application.events.publisher import publish_user_event
+from application.storage.db.repositories.ingest_chunk_progress import (
+    IngestChunkProgressRepository,
+)
+from application.storage.db.session import db_session
+from application.vectorstore.vector_creator import VectorCreator
+
+
+class EmbeddingPipelineError(Exception):
+    """Raised when the per-chunk embed loop produces a partial index.
+
+    Escapes into Celery's ``autoretry_for`` so a transient cause (rate
+    limit, network blip) gets another shot. The chunk-progress
+    checkpoint makes retries cheap — only the failed-and-after chunks
+    re-run. After ``MAX_TASK_ATTEMPTS`` the poison-loop guard in
+    ``with_idempotency`` finalises the row as ``failed``.
+    """
+
+
+def sanitize_content(content: str) -> str:
+    """
+    Remove NUL characters that can cause vector store ingestion to fail.
+    
+    Args:
+        content (str): Raw content that may contain NUL characters
+        
+    Returns:
+        str: Sanitized content with NUL characters removed
+    """
+    if not content:
+        return content
+    return content.replace('\x00', '')
+
+
+# Per-chunk inline retry. Aggressive defaults (tries=10, delay=60) blocked
+# the loop for up to 9 min per chunk and wedged the heartbeat: lower the
+# tail so a transient failure fails-fast and the chunk-progress checkpoint
+# resumes cleanly on next dispatch.
+@retry(tries=3, delay=5, backoff=2)
+def add_text_to_store_with_retry(store: Any, doc: Any, source_id: str) -> None:
+    """Add a document's text and metadata to the vector store with retry logic.
+    
+    Args:
+        store: The vector store object.
+        doc: The document to be added.
+        source_id: Unique identifier for the source.
+        
+    Raises:
+        Exception: If document addition fails after all retry attempts.
+    """
+    try:
+        # Sanitize content to remove NUL characters that cause ingestion failures
+        doc.page_content = sanitize_content(doc.page_content)
+        
+        doc.metadata["source_id"] = str(source_id)
+        store.add_texts([doc.page_content], metadatas=[doc.metadata])
+    except Exception as e:
+        logging.error(f"Failed to add document with retry: {e}", exc_info=True)
+        raise
+
+
+def _init_progress_and_resume_index(
+    source_id: str, total_chunks: int, attempt_id: Optional[str],
+) -> int:
+    """Upsert the progress row and return the next chunk index to embed.
+
+    The repository's upsert preserves ``last_index`` only when the
+    incoming ``attempt_id`` matches the stored one (a Celery autoretry
+    of the same task). On a fresh attempt — including any caller that
+    doesn't pass an ``attempt_id``, e.g. legacy code or tests — the
+    row's checkpoint is reset so the loop starts from chunk 0. This
+    is what prevents a completed checkpoint from any prior run
+    silently no-op'ing the next sync/reingest.
+
+    Best-effort: a DB outage falls back to ``0`` (fresh run from
+    chunk 0). The embed loop's own re-raise still ensures partial
+    runs don't get cached as complete.
+    """
+    try:
+        with db_session() as conn:
+            progress = IngestChunkProgressRepository(conn).init_progress(
+                source_id, total_chunks, attempt_id,
+            )
+    except Exception as e:
+        logging.warning(
+            f"Could not init ingest progress for {source_id}: {e}",
+            exc_info=True,
+        )
+        return 0
+    if not progress:
+        return 0
+    last_index = progress.get("last_index", -1)
+    if last_index is None or last_index < 0:
+        return 0
+    return int(last_index) + 1
+
+
+def _record_progress(source_id: str, last_index: int, embedded_chunks: int) -> None:
+    """Best-effort checkpoint after each chunk; logged but never raised."""
+    try:
+        with db_session() as conn:
+            IngestChunkProgressRepository(conn).record_chunk(
+                source_id, last_index=last_index, embedded_chunks=embedded_chunks
+            )
+    except Exception as e:
+        logging.warning(
+            f"Could not record ingest progress for {source_id}: {e}", exc_info=True
+        )
+
+
+def assert_index_complete(source_id: str) -> None:
+    """Raise ``EmbeddingPipelineError`` if ``ingest_chunk_progress``
+    shows a partial embed for ``source_id``.
+
+    Defense-in-depth tripwire that workers run after
+    ``embed_and_store_documents`` to catch any future swallow path
+    that bypasses the function's own re-raise — the chunk-progress
+    row is the authoritative record of how many chunks landed.
+    No-op when no row exists (zero-doc validation raised before init,
+    or progress repo was unreachable).
+    """
+    try:
+        with db_session() as conn:
+            progress = IngestChunkProgressRepository(conn).get_progress(source_id)
+    except Exception as e:
+        logging.warning(
+            f"assert_index_complete: progress lookup failed for "
+            f"{source_id}: {e}",
+            exc_info=True,
+        )
+        return
+    if not progress:
+        return
+    embedded = int(progress.get("embedded_chunks") or 0)
+    total = int(progress.get("total_chunks") or 0)
+    if embedded < total:
+        raise EmbeddingPipelineError(
+            f"partial index for source {source_id}: "
+            f"{embedded}/{total} chunks embedded"
+        )
+
+
+def embed_and_store_documents(
+    docs: List[Any],
+    folder_name: str,
+    source_id: str,
+    task_status: Any,
+    *,
+    attempt_id: Optional[str] = None,
+    user_id: Optional[str] = None,
+    progress_start: int = 0,
+    progress_end: int = 100,
+) -> None:
+    """Embeds documents and stores them in a vector store.
+
+    Resumable across Celery autoretries of the *same* task: when
+    ``attempt_id`` matches the stored checkpoint's ``attempt_id``,
+    the loop resumes from ``last_index + 1``. A different
+    ``attempt_id`` (a fresh sync / reingest invocation) resets the
+    checkpoint so the index is rebuilt from chunk 0 — this is what
+    keeps a completed checkpoint from poisoning the next sync.
+
+    Args:
+        docs: List of documents to be embedded and stored.
+        folder_name: Directory to save the vector store.
+        source_id: Unique identifier for the source.
+        task_status: Task state manager for progress updates.
+        attempt_id: Stable id of the current task invocation,
+            typically ``self.request.id`` from the Celery task body.
+            ``None`` is treated as a fresh attempt every time.
+        user_id: When provided, per-percent SSE progress events are
+            published to ``user:{user_id}`` for the in-app upload toast.
+            ``None`` is the safe default — workers without a user
+            context (e.g. background syncs) skip the publish.
+        progress_start: Percent the reported progress maps to at chunk 0.
+            Lets a caller reserve the lower band for an earlier stage
+            (e.g. parsing). Defaults to ``0`` (embed owns the whole bar).
+        progress_end: Percent the reported progress maps to at the final
+            chunk. Defaults to ``100``.
+
+    Returns:
+        None
+
+    Raises:
+        OSError: If unable to create folder or save vector store.
+        EmbeddingPipelineError: If a chunk fails after retries.
+    """
+    # Ensure the folder exists
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # Validate docs is not empty
+    if not docs:
+        raise ValueError("No documents to embed - check file format and extension")
+
+    total_docs = len(docs)
+    # Atomic upsert that preserves checkpoint state on attempt-id match
+    # (autoretry of same task) and resets it on mismatch (fresh sync /
+    # reingest). Returns the new resume index — 0 means "start fresh".
+    resume_index = _init_progress_and_resume_index(
+        source_id, total_docs, attempt_id,
+    )
+    is_resume = resume_index > 0
+
+    # Initialize vector store
+    if settings.VECTOR_STORE == "faiss":
+        if is_resume:
+            # Load the existing FAISS index from storage so chunks
+            # already embedded by the prior attempt survive the
+            # save_local rewrite at the end of this run.
+            store = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE,
+                source_id=source_id,
+                embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+            )
+            loop_start = resume_index
+        else:
+            # FAISS requires at least one doc to construct the store;
+            # seed with ``docs[0]`` and let the loop pick up at index 1.
+            store = VectorCreator.create_vectorstore(
+                settings.VECTOR_STORE,
+                docs_init=[docs[0]],
+                source_id=source_id,
+                embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+            )
+            # Record the seeded chunk so single-doc ingests don't fail
+            # ``assert_index_complete`` — the loop never runs for
+            # ``total_docs == 1`` and would otherwise leave
+            # ``embedded_chunks`` at 0 / ``last_index`` at -1. The loop
+            # body's per-iteration ``_record_progress`` overshoots
+            # correctly for multi-chunk runs (counts seed + iterations),
+            # so writing this checkpoint up-front is a no-op for those.
+            _record_progress(source_id, last_index=0, embedded_chunks=1)
+            loop_start = 1
+    else:
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            source_id=source_id,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+        # Only wipe the index on a fresh run — a resume must keep the
+        # chunks that earlier attempts already embedded.
+        if not is_resume:
+            store.delete_index()
+        loop_start = resume_index
+
+    if is_resume and loop_start >= total_docs:
+        # Nothing left to do; the loop runs zero iterations and
+        # downstream finalize logic still executes. This is only
+        # reachable on a same-attempt retry of a task whose previous
+        # attempt finished — typically a Celery acks_late redelivery
+        # after the task already returned. The ``assert_index_complete``
+        # tripwire still validates ``embedded == total`` afterwards.
+        loop_start = total_docs
+
+    # Process and embed documents
+    chunk_error: Exception | None = None
+    failed_idx: int | None = None
+    last_published_pct = -1
+    source_id_str = str(source_id)
+    progress_span = progress_end - progress_start
+    for idx in tqdm(
+        range(loop_start, total_docs),
+        desc="Embedding 🦖",
+        unit="docs",
+        total=total_docs - loop_start,
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
+        doc = docs[idx]
+        try:
+            # Map the embed loop into [progress_start, progress_end].
+            progress = progress_start + int(
+                ((idx + 1) / total_docs) * progress_span
+            )
+            task_status.update_state(state="PROGRESS", meta={"current": progress})
+
+            # SSE push for sub-second upload-toast updates. Throttled to one
+            # event per percent so a 10k-chunk ingest emits ~100 events,
+            # not 10k. The Celery update_state above stays the source of
+            # truth for the polling-fallback path.
+            if user_id and progress > last_published_pct:
+                publish_user_event(
+                    user_id,
+                    "source.ingest.progress",
+                    {
+                        "current": progress,
+                        "total": total_docs,
+                        "embedded_chunks": idx + 1,
+                        "stage": "embedding",
+                    },
+                    scope={"kind": "source", "id": source_id_str},
+                )
+                last_published_pct = progress
+
+            # Add document to vector store
+            add_text_to_store_with_retry(store, doc, source_id)
+            _record_progress(source_id, last_index=idx, embedded_chunks=idx + 1)
+        except Exception as e:
+            chunk_error = e
+            failed_idx = idx
+            logging.error(f"Error embedding document {idx}: {e}", exc_info=True)
+            logging.info(f"Saving progress at document {idx} out of {total_docs}")
+            try:
+                store.save_local(folder_name)
+                logging.info("Progress saved successfully")
+            except Exception as save_error:
+                logging.error(f"CRITICAL: Failed to save progress: {save_error}", exc_info=True)
+                # Continue without breaking to attempt final save
+            break
+
+    # Save the vector store
+    if settings.VECTOR_STORE == "faiss":
+        try:
+            store.save_local(folder_name)
+            logging.info("Vector store saved successfully.")
+        except Exception as e:
+            logging.error(f"CRITICAL: Failed to save final vector store: {e}", exc_info=True)
+            raise OSError(f"Unable to save vector store to {folder_name}: {e}") from e
+    else:
+        logging.info("Vector store saved successfully.")
+
+    # Re-raise after the partial save: the chunks that *did* embed are
+    # flushed to disk and recorded in ``ingest_chunk_progress``, so a
+    # Celery autoretry resumes via ``_read_resume_index`` and only
+    # re-runs the failed-and-after chunks. Without the raise, the
+    # task body returns success and ``with_idempotency`` finalises
+    # ``task_dedup`` as ``completed`` for a partial index — poisoning
+    # the cache for 24h.
+    if chunk_error is not None:
+        raise EmbeddingPipelineError(
+            f"embed failure at chunk {failed_idx}/{total_docs} "
+            f"for source {source_id}"
+        ) from chunk_error

application/parser/file/__init__.py

@@ -0,0 +1 @@
+

application/parser/file/audio_parser.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import Dict, Union
+
+from application.core.settings import settings
+from application.parser.file.base_parser import BaseParser
+from application.stt.stt_creator import STTCreator
+from application.stt.upload_limits import enforce_audio_file_size_limit
+
+
+class AudioParser(BaseParser):
+    def __init__(self, parser_config=None):
+        super().__init__(parser_config=parser_config)
+        self._transcript_metadata: Dict[str, Dict] = {}
+
+    def _init_parser(self) -> Dict:
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        _ = errors
+        try:
+            enforce_audio_file_size_limit(file.stat().st_size)
+        except OSError:
+            pass
+        stt = STTCreator.create_stt(settings.STT_PROVIDER)
+        result = stt.transcribe(
+            file,
+            language=settings.STT_LANGUAGE,
+            timestamps=settings.STT_ENABLE_TIMESTAMPS,
+            diarize=settings.STT_ENABLE_DIARIZATION,
+        )
+
+        transcript_metadata = {
+            "transcript_duration_s": result.get("duration_s"),
+            "transcript_language": result.get("language"),
+            "transcript_provider": result.get("provider"),
+        }
+        if result.get("segments"):
+            transcript_metadata["transcript_segments"] = result["segments"]
+
+        self._transcript_metadata[str(file)] = {
+            key: value
+            for key, value in transcript_metadata.items()
+            if value not in (None, [], {})
+        }
+        return result.get("text", "")
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        return self._transcript_metadata.get(str(file), {})

application/parser/file/base.py

@@ -0,0 +1,19 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from langchain_core.documents import Document as LCDocument
+from application.parser.schema.base import Document
+
+
+class BaseReader:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_langchain_documents(self, **load_kwargs: Any) -> List[LCDocument]:
+        """Load data in LangChain document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_langchain_format() for d in docs]

application/parser/file/base_parser.py

@@ -0,0 +1,43 @@
+"""Base parser and config class."""
+
+from abc import abstractmethod
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+
+class BaseParser:
+    """Base class for all parsers."""
+
+    def __init__(self, parser_config: Optional[Dict] = None):
+        """Init params."""
+        self._parser_config = parser_config
+
+    def init_parser(self) -> None:
+        """Init parser and store it."""
+        parser_config = self._init_parser()
+        self._parser_config = parser_config
+
+    @property
+    def parser_config_set(self) -> bool:
+        """Check if parser config is set."""
+        return self._parser_config is not None
+
+    @property
+    def parser_config(self) -> Dict:
+        """Check if parser config is set."""
+        if self._parser_config is None:
+            raise ValueError("Parser config not set.")
+        return self._parser_config
+
+    @abstractmethod
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+
+    @abstractmethod
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+
+    def get_file_metadata(self, file: Path) -> Dict:
+        """Return parser-specific metadata for the most recently parsed file."""
+        _ = file
+        return {}

application/parser/file/bulk.py

@@ -0,0 +1,356 @@
+"""Simple reader that reads files of different formats from a directory."""
+import logging
+from pathlib import Path
+from typing import Callable, Dict, List, Optional, Union
+
+from application.parser.file.base import BaseReader
+from application.parser.file.base_parser import BaseParser
+from application.parser.file.docs_parser import DocxParser, PDFParser
+from application.parser.file.epub_parser import EpubParser
+from application.parser.file.html_parser import HTMLParser
+from application.parser.file.markdown_parser import MarkdownParser
+from application.parser.file.rst_parser import RstParser
+from application.parser.file.tabular_parser import PandasCSVParser, ExcelParser
+from application.parser.file.json_parser import JSONParser
+from application.parser.file.pptx_parser import PPTXParser
+from application.parser.file.image_parser import ImageParser
+from application.parser.file.audio_parser import AudioParser
+from application.parser.schema.base import Document
+from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
+from application.utils import num_tokens_from_string
+from application.core.settings import settings
+
+
+def _build_audio_parser_mapping() -> Dict[str, BaseParser]:
+    return {extension: AudioParser() for extension in SUPPORTED_AUDIO_EXTENSIONS}
+
+
+def get_default_file_extractor(
+    ocr_enabled: Optional[bool] = None,
+) -> Dict[str, BaseParser]:
+    """Get the default file extractor.
+
+    Uses docling parsers by default for advanced document processing.
+    Falls back to standard parsers if docling is not installed.
+    """
+    try:
+        from application.parser.file.docling_parser import (
+            DoclingPDFParser,
+            DoclingDocxParser,
+            DoclingPPTXParser,
+            DoclingXLSXParser,
+            DoclingHTMLParser,
+            DoclingImageParser,
+            DoclingCSVParser,
+            DoclingAsciiDocParser,
+            DoclingVTTParser,
+            DoclingXMLParser,
+        )
+        if ocr_enabled is None:
+            ocr_enabled = settings.DOCLING_OCR_ENABLED
+        return {
+            # Documents
+            ".pdf": DoclingPDFParser(ocr_enabled=ocr_enabled),
+            ".docx": DoclingDocxParser(),
+            ".pptx": DoclingPPTXParser(),
+            ".xlsx": DoclingXLSXParser(),
+            # Web formats
+            ".html": DoclingHTMLParser(),
+            ".xhtml": DoclingHTMLParser(),
+            # Data formats
+            ".csv": DoclingCSVParser(),
+            ".json": JSONParser(),  # Keep JSON parser (specialized handling)
+            # Text/markup formats
+            ".md": MarkdownParser(),  # Keep markdown parser (specialized handling)
+            ".mdx": MarkdownParser(),
+            ".rst": RstParser(),
+            ".adoc": DoclingAsciiDocParser(),
+            ".asciidoc": DoclingAsciiDocParser(),
+            # Images (with OCR) - only use Docling when OCR is enabled
+            ".png": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            ".jpg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            ".jpeg": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            ".tiff": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            ".tif": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            ".bmp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            ".webp": DoclingImageParser(ocr_enabled=ocr_enabled) if ocr_enabled else ImageParser(),
+            # Media/subtitles
+            ".vtt": DoclingVTTParser(),
+            **_build_audio_parser_mapping(),
+            # Specialized XML formats
+            ".xml": DoclingXMLParser(),
+            # Formats docling doesn't support - use standard parsers
+            ".epub": EpubParser(),
+        }
+    except ImportError:
+        logging.warning(
+            "docling is not installed. Using standard parsers. "
+            "For advanced document parsing, install with: pip install docling"
+        )
+        # Fallback to standard parsers
+        return {
+            ".pdf": PDFParser(),
+            ".docx": DocxParser(),
+            ".csv": PandasCSVParser(),
+            ".xlsx": ExcelParser(),
+            ".epub": EpubParser(),
+            ".md": MarkdownParser(),
+            ".rst": RstParser(),
+            ".html": HTMLParser(),
+            ".mdx": MarkdownParser(),
+            ".json": JSONParser(),
+            ".pptx": PPTXParser(),
+            ".png": ImageParser(),
+            ".jpg": ImageParser(),
+            ".jpeg": ImageParser(),
+            **_build_audio_parser_mapping(),
+        }
+
+
+# For backwards compatibility
+DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = get_default_file_extractor()
+
+
+class SimpleDirectoryReader(BaseReader):
+    """Simple directory reader.
+
+    Can read files into separate documents, or concatenates
+    files into one document text.
+
+    Args:
+        input_dir (str): Path to the directory.
+        input_files (List): List of file paths to read (Optional; overrides input_dir)
+        exclude_hidden (bool): Whether to exclude hidden files (dotfiles).
+        errors (str): how encoding and decoding errors are to be handled,
+              see https://docs.python.org/3/library/functions.html#open
+        recursive (bool): Whether to recursively search in subdirectories.
+            False by default.
+        required_exts (Optional[List[str]]): List of required extensions.
+            Default is None.
+        file_extractor (Optional[Dict[str, BaseParser]]): A mapping of file
+            extension to a BaseParser class that specifies how to convert that file
+            to text. See DEFAULT_FILE_EXTRACTOR.
+        num_files_limit (Optional[int]): Maximum number of files to read.
+            Default is None.
+        file_metadata (Optional[Callable[str, Dict]]): A function that takes
+            in a filename and returns a Dict of metadata for the Document.
+            Default is None.
+    """
+
+    def __init__(
+            self,
+            input_dir: Optional[str] = None,
+            input_files: Optional[List] = None,
+            exclude_hidden: bool = True,
+            errors: str = "ignore",
+            recursive: bool = True,
+            required_exts: Optional[List[str]] = None,
+            file_extractor: Optional[Dict[str, BaseParser]] = None,
+            num_files_limit: Optional[int] = None,
+            file_metadata: Optional[Callable[[str], Dict]] = None,
+    ) -> None:
+        """Initialize with parameters."""
+        super().__init__()
+
+        if not input_dir and not input_files:
+            raise ValueError("Must provide either `input_dir` or `input_files`.")
+
+        self.errors = errors
+
+        self.recursive = recursive
+        self.exclude_hidden = exclude_hidden
+        # Normalize extensions to lowercase for case-insensitive matching
+        self.required_exts = (
+            [ext.lower() for ext in required_exts] if required_exts else None
+        )
+        self.num_files_limit = num_files_limit
+
+        if input_files:
+            self.input_files = []
+            for path in input_files:
+                print(path)
+                input_file = Path(path)
+                self.input_files.append(input_file)
+        elif input_dir:
+            self.input_dir = Path(input_dir)
+            self.input_files = self._add_files(self.input_dir)
+
+        self.file_extractor = file_extractor or DEFAULT_FILE_EXTRACTOR
+        self.file_metadata = file_metadata
+
+    def _add_files(self, input_dir: Path) -> List[Path]:
+        """Add files."""
+        input_files = sorted(input_dir.iterdir())
+        new_input_files = []
+        dirs_to_explore = []
+        for input_file in input_files:
+            if input_file.is_dir():
+                if self.recursive:
+                    dirs_to_explore.append(input_file)
+            elif self.exclude_hidden and input_file.name.startswith("."):
+                continue
+            elif (
+                    self.required_exts is not None
+                    and input_file.suffix.lower() not in self.required_exts
+            ):
+                continue
+            else:
+                new_input_files.append(input_file)
+
+        for dir_to_explore in dirs_to_explore:
+            sub_input_files = self._add_files(dir_to_explore)
+            new_input_files.extend(sub_input_files)
+
+        if self.num_files_limit is not None and self.num_files_limit > 0:
+            new_input_files = new_input_files[0: self.num_files_limit]
+
+        # print total number of files added
+        logging.debug(
+            f"> [SimpleDirectoryReader] Total files added: {len(new_input_files)}"
+        )
+
+        return new_input_files
+
+    def load_data(
+        self,
+        concatenate: bool = False,
+        progress_callback: Optional[Callable[[int, int], None]] = None,
+    ) -> List[Document]:
+        """Load data from the input directory.
+
+        Args:
+            concatenate (bool): whether to concatenate all files into one document.
+                If set to True, file metadata is ignored.
+                False by default.
+            progress_callback (Optional[Callable[[int, int], None]]): Called
+                after each file is parsed with ``(files_done, total_files)``.
+                Lets callers surface parse/OCR progress before embedding
+                begins. Exceptions raised by the callback are swallowed so
+                progress reporting can never fail ingestion.
+
+        Returns:
+            List[Document]: A list of documents.
+        """
+        data: Union[str, List[str]] = ""
+        data_list: List[str] = []
+        metadata_list = []
+        self.file_token_counts = {}
+
+        total_files = len(self.input_files)
+        for file_index, input_file in enumerate(self.input_files):
+            suffix_lower = input_file.suffix.lower()
+            parser_metadata = {}
+            if suffix_lower in self.file_extractor:
+                parser = self.file_extractor[suffix_lower]
+                if not parser.parser_config_set:
+                    parser.init_parser()
+                data = parser.parse_file(input_file, errors=self.errors)
+                parser_metadata = parser.get_file_metadata(input_file)
+            else:
+                # do standard read
+                with open(input_file, "r", errors=self.errors) as f:
+                    data = f.read()
+            
+            # Calculate token count for this file
+            if isinstance(data, List):
+                file_tokens = sum(num_tokens_from_string(str(d)) for d in data)
+            else:
+                file_tokens = num_tokens_from_string(str(data))
+            
+            full_path = str(input_file.resolve())
+            self.file_token_counts[full_path] = file_tokens
+            
+            base_metadata = {
+                'title': input_file.name,
+                'token_count': file_tokens,
+            }
+            if parser_metadata:
+                base_metadata.update(parser_metadata)
+            
+            if hasattr(self, 'input_dir'):
+                try:
+                    relative_path = str(input_file.relative_to(self.input_dir))
+                    base_metadata['source'] = relative_path
+                except ValueError:
+                    base_metadata['source'] = str(input_file)
+            else:
+                base_metadata['source'] = str(input_file)
+
+            if self.file_metadata is not None:
+                custom_metadata = self.file_metadata(input_file.name)
+                base_metadata.update(custom_metadata)
+
+            if isinstance(data, List):
+                # Extend data_list with each item in the data list
+                data_list.extend([str(d) for d in data])
+                metadata_list.extend([base_metadata for _ in data])
+            else:
+                data_list.append(str(data))
+                metadata_list.append(base_metadata)
+
+            if progress_callback is not None:
+                try:
+                    progress_callback(file_index + 1, total_files)
+                except Exception:
+                    logging.warning(
+                        "load_data progress callback failed", exc_info=True
+                    )
+
+        # Build directory structure if input_dir is provided
+        if hasattr(self, 'input_dir'):
+            self.directory_structure = self.build_directory_structure(self.input_dir)
+            logging.info("Directory structure built successfully")
+        else:
+            self.directory_structure = {}
+
+        if concatenate:
+            return [Document("\n".join(data_list))]
+        elif self.file_metadata is not None:
+            return [Document(d, extra_info=m) for d, m in zip(data_list, metadata_list)]
+        else:
+            return [Document(d) for d in data_list]
+
+    def build_directory_structure(self, base_path):
+        """Build a dictionary representing the directory structure.
+
+        Args:
+            base_path: The base path to start building the structure from.
+
+        Returns:
+            dict: A nested dictionary representing the directory structure.
+        """
+        import mimetypes
+        
+        def build_tree(path):
+            """Helper function to recursively build the directory tree."""
+            result = {}
+            
+            for item in path.iterdir():
+                if self.exclude_hidden and item.name.startswith('.'):
+                    continue
+                    
+                if item.is_dir():
+                    subtree = build_tree(item)
+                    if subtree:
+                        result[item.name] = subtree
+                else:
+                    if self.required_exts is not None and item.suffix.lower() not in self.required_exts:
+                        continue
+                    
+                    full_path = str(item.resolve())
+                    file_size_bytes = item.stat().st_size
+                    mime_type = mimetypes.guess_type(item.name)[0] or "application/octet-stream"
+                    
+                    file_info = {
+                        "type": mime_type,
+                        "size_bytes": file_size_bytes
+                    }
+                    
+                    if hasattr(self, 'file_token_counts') and full_path in self.file_token_counts:
+                        file_info["token_count"] = self.file_token_counts[full_path]
+                        
+                    result[item.name] = file_info
+                    
+            return result
+        
+        return build_tree(Path(base_path))

application/parser/file/constants.py

@@ -0,0 +1,27 @@
+"""Shared file-extension constants for parsing and ingestion flows."""
+
+from application.stt.constants import SUPPORTED_AUDIO_EXTENSIONS
+
+
+SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS = (
+    ".rst",
+    ".md",
+    ".pdf",
+    ".txt",
+    ".docx",
+    ".csv",
+    ".epub",
+    ".html",
+    ".mdx",
+    ".json",
+    ".xlsx",
+    ".pptx",
+)
+
+SUPPORTED_SOURCE_IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg")
+
+SUPPORTED_SOURCE_EXTENSIONS = (
+    *SUPPORTED_SOURCE_DOCUMENT_EXTENSIONS,
+    *SUPPORTED_SOURCE_IMAGE_EXTENSIONS,
+    *SUPPORTED_AUDIO_EXTENSIONS,
+)

application/parser/file/docling_parser.py

@@ -0,0 +1,354 @@
+"""Docling parser.
+
+Uses docling library for advanced document parsing with layout detection,
+table structure recognition, and unified document representation.
+
+Supports: PDF, DOCX, PPTX, XLSX, HTML, XHTML, CSV, Markdown, AsciiDoc,
+images (PNG, JPEG, TIFF, BMP, WEBP), WebVTT, and specialized XML formats.
+"""
+import importlib.util
+import logging
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+from application.parser.file.base_parser import BaseParser
+
+logger = logging.getLogger(__name__)
+
+
+# Per-stage batch size for docling's threaded pipeline; 1 holds the
+# concurrent working set to a single page (see _apply_pipeline_caps).
+_PIPELINE_BATCH_SIZE = 1
+
+
+def _apply_pipeline_caps(pipeline_options) -> None:
+    """Cap docling's threaded-pipeline queue depth and batch sizes in place.
+
+    hasattr-guarded so docling builds without these knobs are unaffected.
+    """
+    from application.core.settings import settings
+
+    caps = {
+        "queue_max_size": max(1, settings.DOCLING_PIPELINE_QUEUE_MAX_SIZE),
+        "layout_batch_size": _PIPELINE_BATCH_SIZE,
+        "table_batch_size": _PIPELINE_BATCH_SIZE,
+        "ocr_batch_size": _PIPELINE_BATCH_SIZE,
+    }
+    for name, value in caps.items():
+        if hasattr(pipeline_options, name):
+            setattr(pipeline_options, name, value)
+
+
+class DoclingParser(BaseParser):
+    """Parser using docling for advanced document processing.
+
+    Docling provides:
+    - Advanced PDF layout analysis
+    - Table structure recognition
+    - Reading order detection
+    - OCR for scanned documents (supports RapidOCR)
+    - Unified DoclingDocument format
+    - Export to Markdown
+
+    Uses hybrid OCR approach by default:
+    - Text regions: Direct PDF text extraction (fast)
+    - Bitmap/image regions: OCR only these areas (smart)
+    """
+
+    def __init__(
+        self,
+        ocr_enabled: bool = True,
+        table_structure: bool = True,
+        export_format: str = "markdown",
+        use_rapidocr: bool = True,
+        ocr_languages: Optional[List[str]] = None,
+        force_full_page_ocr: bool = False,
+    ):
+        """Initialize DoclingParser.
+
+        Args:
+            ocr_enabled: Enable OCR for bitmap/image regions in documents
+            table_structure: Enable table structure recognition
+            export_format: Output format ('markdown', 'text', 'html')
+            use_rapidocr: Use RapidOCR engine (default True, works well in Docker)
+            ocr_languages: List of OCR languages (default: ['english'])
+            force_full_page_ocr: Force OCR on entire page (False = smart hybrid OCR)
+        """
+        super().__init__()
+        self.ocr_enabled = ocr_enabled
+        self.table_structure = table_structure
+        self.export_format = export_format
+        self.use_rapidocr = use_rapidocr
+        self.ocr_languages = ocr_languages or ["english"]
+        self.force_full_page_ocr = force_full_page_ocr
+        self._converter = None
+
+    def _create_converter(self):
+        """Create a docling converter with hybrid OCR configuration.
+
+        Uses smart OCR approach:
+        - When ocr_enabled=True and force_full_page_ocr=False (default):
+          Layout model detects text vs bitmap regions, OCR only runs on bitmaps
+        - When ocr_enabled=True and force_full_page_ocr=True:
+          OCR runs on entire page (for scanned documents/images)
+        - When ocr_enabled=False:
+          No OCR, only native text extraction
+
+        Returns:
+            DocumentConverter instance
+        """
+        from docling.document_converter import (
+            DocumentConverter,
+            ImageFormatOption,
+            InputFormat,
+            PdfFormatOption,
+        )
+        from docling.datamodel.pipeline_options import PdfPipelineOptions
+
+        pipeline_options = PdfPipelineOptions(
+            do_ocr=self.ocr_enabled,
+            do_table_structure=self.table_structure,
+        )
+        _apply_pipeline_caps(pipeline_options)
+
+        if self.ocr_enabled:
+            ocr_options = self._get_ocr_options()
+            if ocr_options is not None:
+                pipeline_options.ocr_options = ocr_options
+
+        return DocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_options=pipeline_options,
+                ),
+                InputFormat.IMAGE: ImageFormatOption(
+                    pipeline_options=pipeline_options,
+                ),
+            }
+        )
+
+    def _init_parser(self) -> Dict:
+        """Initialize the docling converter with hybrid OCR."""
+        logger.info("Initializing DoclingParser...")
+        logger.info(f"  ocr_enabled={self.ocr_enabled}")
+        logger.info(f"  force_full_page_ocr={self.force_full_page_ocr}")
+        logger.info(f"  use_rapidocr={self.use_rapidocr}")
+
+        if importlib.util.find_spec("docling.document_converter") is None:
+            raise ImportError(
+                "docling is required for DoclingParser. "
+                "Install it with: pip install docling"
+            )
+
+        # Create converter with hybrid OCR (smart: text direct, bitmaps OCR'd)
+        self._converter = self._create_converter()
+
+        logger.info("DoclingParser initialized successfully")
+        return {
+            "ocr_enabled": self.ocr_enabled,
+            "table_structure": self.table_structure,
+            "export_format": self.export_format,
+            "use_rapidocr": self.use_rapidocr,
+            "ocr_languages": self.ocr_languages,
+            "force_full_page_ocr": self.force_full_page_ocr,
+        }
+
+    def _get_ocr_options(self):
+        """Get OCR options based on configuration.
+
+        Returns RapidOcrOptions if use_rapidocr is True and available,
+        otherwise returns None to use docling defaults.
+        """
+        if not self.use_rapidocr:
+            return None
+
+        try:
+            from docling.datamodel.pipeline_options import RapidOcrOptions
+
+            return RapidOcrOptions(
+                lang=self.ocr_languages,
+                force_full_page_ocr=self.force_full_page_ocr,
+            )
+        except ImportError as e:
+            logger.warning(f"Failed to import RapidOcrOptions: {e}")
+            return None
+        except Exception as e:
+            logger.error(f"Error creating RapidOcrOptions: {e}")
+            return None
+
+    def _export_content(self, document) -> str:
+        """Export document content in the configured format.
+
+        Handles edge case where text is nested under picture elements (e.g., OCR'd
+        images). If the standard export returns minimal content but document.texts
+        contains extracted text, falls back to direct text extraction.
+        """
+        if self.export_format == "markdown":
+            content = document.export_to_markdown()
+        elif self.export_format == "html":
+            content = document.export_to_html()
+        else:
+            content = document.export_to_text()
+
+        # Handle case where text is nested under pictures (common with OCR'd images)
+        # Standard exports may return just "<!-- image -->" while actual text exists
+        stripped_content = content.strip()
+        is_minimal = len(stripped_content) < 50 or stripped_content == "<!-- image -->"
+
+        if is_minimal and hasattr(document, "texts") and document.texts:
+            # Extract text directly from document.texts
+            extracted_texts = [t.text for t in document.texts if t.text]
+            if extracted_texts:
+                logger.info(
+                    f"Standard export minimal ({len(stripped_content)} chars), "
+                    f"extracting {len(extracted_texts)} texts directly"
+                )
+                return "\n\n".join(extracted_texts)
+
+        return content
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file using docling with hybrid OCR.
+
+        Uses smart OCR approach where the layout model detects text vs bitmap
+        regions. Text is extracted directly, bitmaps are OCR'd only when needed.
+
+        Args:
+            file: Path to the file to parse
+            errors: Error handling mode (ignored, docling handles internally)
+
+        Returns:
+            Parsed document content as markdown string
+        """
+        logger.info(f"parse_file called for: {file}")
+
+        if self._converter is None:
+            self._init_parser()
+
+        try:
+            logger.info(f"Converting file with hybrid OCR: {file}")
+            result = self._converter.convert(str(file))
+            content = self._export_content(result.document)
+            logger.info(f"Parse complete, content length: {len(content)} chars")
+
+            return content
+
+        except Exception as e:
+            logger.error(f"Error parsing file with docling: {e}", exc_info=True)
+            if errors == "ignore":
+                return f"[Error parsing file with docling: {str(e)}]"
+            raise
+
+
+class DoclingPDFParser(DoclingParser):
+    """Docling-based PDF parser with advanced features and RapidOCR support.
+
+    Uses hybrid OCR approach by default:
+    - Text regions: Direct PDF text extraction (fast)
+    - Bitmap/image regions: OCR only these areas (smart)
+
+    Set force_full_page_ocr=True only for fully scanned documents.
+    """
+
+    def __init__(
+        self,
+        ocr_enabled: bool = True,
+        table_structure: bool = True,
+        use_rapidocr: bool = True,
+        ocr_languages: Optional[List[str]] = None,
+        force_full_page_ocr: bool = False,
+    ):
+        super().__init__(
+            ocr_enabled=ocr_enabled,
+            table_structure=table_structure,
+            export_format="markdown",
+            use_rapidocr=use_rapidocr,
+            ocr_languages=ocr_languages,
+            force_full_page_ocr=force_full_page_ocr,
+        )
+
+
+class DoclingDocxParser(DoclingParser):
+    """Docling-based DOCX parser."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")
+
+
+class DoclingPPTXParser(DoclingParser):
+    """Docling-based PPTX parser."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")
+
+
+class DoclingXLSXParser(DoclingParser):
+    """Docling-based XLSX parser with table structure."""
+
+    def __init__(self):
+        super().__init__(table_structure=True, export_format="markdown")
+
+
+class DoclingHTMLParser(DoclingParser):
+    """Docling-based HTML parser."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")
+
+
+class DoclingImageParser(DoclingParser):
+    """Docling-based image parser with OCR and RapidOCR support.
+
+    For images, force_full_page_ocr=True is used since images are entirely
+    visual and require full OCR to extract any text.
+    """
+
+    def __init__(
+        self,
+        ocr_enabled: bool = True,
+        use_rapidocr: bool = True,
+        ocr_languages: Optional[List[str]] = None,
+        force_full_page_ocr: bool = True,
+    ):
+        super().__init__(
+            ocr_enabled=ocr_enabled,
+            export_format="markdown",
+            use_rapidocr=use_rapidocr,
+            ocr_languages=ocr_languages,
+            force_full_page_ocr=force_full_page_ocr,
+        )
+
+
+class DoclingCSVParser(DoclingParser):
+    """Docling-based CSV parser."""
+
+    def __init__(self):
+        super().__init__(table_structure=True, export_format="markdown")
+
+
+class DoclingMarkdownParser(DoclingParser):
+    """Docling-based Markdown parser."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")
+
+
+class DoclingAsciiDocParser(DoclingParser):
+    """Docling-based AsciiDoc parser."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")
+
+
+class DoclingVTTParser(DoclingParser):
+    """Docling-based WebVTT (video text tracks) parser."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")
+
+
+class DoclingXMLParser(DoclingParser):
+    """Docling-based XML parser (USPTO, JATS)."""
+
+    def __init__(self):
+        super().__init__(export_format="markdown")

application/parser/file/docs_parser.py

@@ -0,0 +1,70 @@
+"""Docs parser.
+
+Contains parsers for docx, pdf files.
+
+"""
+from pathlib import Path
+from typing import Dict
+
+from application.parser.file.base_parser import BaseParser
+from application.core.settings import settings
+import requests
+
+class PDFParser(BaseParser):
+    """PDF parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        if settings.PARSE_PDF_AS_IMAGE:
+            doc2md_service = "https://llm.arc53.com/doc2md"
+            # alternatively you can use local vision capable LLM
+            with open(file, "rb") as file_loaded:
+                files = {'file': file_loaded}
+                response = requests.post(doc2md_service, files=files, timeout=100)
+                data = response.json()["markdown"]
+            return data
+
+        try:
+            from pypdf import PdfReader
+        except ImportError:
+            raise ValueError("pypdf is required to read PDF files.")
+        text_list = []
+        with open(file, "rb") as fp:
+            # Create a PDF object
+            pdf = PdfReader(fp)
+
+            # Get the number of pages in the PDF document
+            num_pages = len(pdf.pages)
+
+            # Iterate over every page
+            for page_index in range(num_pages):
+                # Extract the text from the page
+                page = pdf.pages[page_index]
+                page_text = page.extract_text()
+                text_list.append(page_text)
+        text = "\n".join(text_list)
+
+        return text
+
+
+class DocxParser(BaseParser):
+    """Docx parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            import docx2txt
+        except ImportError:
+            raise ValueError("docx2txt is required to read Microsoft Word files.")
+
+        text = docx2txt.process(file)
+
+        return text

application/parser/file/epub_parser.py

@@ -0,0 +1,28 @@
+"""Epub parser.
+
+Contains parsers for epub files.
+"""
+
+from pathlib import Path
+from typing import Dict
+
+from application.parser.file.base_parser import BaseParser
+
+
+class EpubParser(BaseParser):
+    """Epub Parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> str:
+        """Parse file."""
+        try:
+            from fast_ebook import epub
+        except ImportError:
+            raise ValueError("`fast-ebook` is required to read Epub files.")
+
+        book = epub.read_epub(file)
+        text = book.to_markdown()
+        return text

application/parser/file/html_parser.py

@@ -0,0 +1,24 @@
+"""HTML parser.
+
+Contains parser for html files.
+
+"""
+from pathlib import Path
+from typing import Dict, Union
+
+from application.parser.file.base_parser import BaseParser
+
+
+class HTMLParser(BaseParser):
+    """HTML parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        from langchain_community.document_loaders import BSHTMLLoader
+
+        loader = BSHTMLLoader(file)
+        data = loader.load()        
+        return data

application/parser/file/image_parser.py

@@ -0,0 +1,31 @@
+"""Image parser.
+
+Contains parser for .png, .jpg, .jpeg files.
+
+"""
+from pathlib import Path
+import requests
+from typing import Dict, Union
+
+from application.parser.file.base_parser import BaseParser
+from application.core.settings import settings
+
+
+class ImageParser(BaseParser):
+    """Image parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        if settings.PARSE_IMAGE_REMOTE:
+            doc2md_service = "https://llm.arc53.com/doc2md"
+            # alternatively you can use local vision capable LLM
+            with open(file, "rb") as file_loaded:
+                files = {'file': file_loaded}
+                response = requests.post(doc2md_service, files=files, timeout=100)   
+                data = response.json()["markdown"] 
+        else:
+            data = ""
+        return data

application/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from application.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

application/parser/file/markdown_parser.py

@@ -0,0 +1,145 @@
+"""Markdown parser.
+
+Contains parser for md files.
+
+"""
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union, cast
+
+from application.parser.file.base_parser import BaseParser
+from application.utils import num_tokens_from_string
+
+
+class MarkdownParser(BaseParser):
+    """Markdown parser.
+
+    Extract text from markdown files.
+    Returns dictionary with keys as headers and values as the text between headers.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            remove_hyperlinks: bool = True,
+            remove_images: bool = True,
+            max_tokens: int = 2048,
+            # remove_tables: bool = True,
+            **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._remove_hyperlinks = remove_hyperlinks
+        self._remove_images = remove_images
+        self._max_tokens = max_tokens
+        # self._remove_tables = remove_tables
+
+    def tups_chunk_append(self, tups: List[Tuple[Optional[str], str]], current_header: Optional[str],
+                          current_text: str):
+        """Append to tups chunk."""
+        num_tokens = num_tokens_from_string(current_text)
+        if num_tokens > self._max_tokens:
+            chunks = [current_text[i:i + self._max_tokens] for i in range(0, len(current_text), self._max_tokens)]
+            for chunk in chunks:
+                tups.append((current_header, chunk))
+        else:
+            tups.append((current_header, current_text))
+        return tups
+
+    def markdown_to_tups(self, markdown_text: str) -> List[Tuple[Optional[str], str]]:
+        """Convert a markdown file to a dictionary.
+
+        The keys are the headers and the values are the text under each header.
+
+        """
+        markdown_tups: List[Tuple[Optional[str], str]] = []
+        lines = markdown_text.split("\n")
+
+        current_header = None
+        current_text = ""
+
+        for line in lines:
+            header_match = re.match(r"^#+\s", line)
+            if header_match:
+                if current_header is not None:
+                    if current_text == "" or None:
+                        continue
+                    markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
+
+                current_header = line
+                current_text = ""
+            else:
+                current_text += line + "\n"
+        markdown_tups = self.tups_chunk_append(markdown_tups, current_header, current_text)
+
+        if current_header is not None:
+            # pass linting, assert keys are defined
+            markdown_tups = [
+                (re.sub(r"#", "", cast(str, key)).strip(), re.sub(r"<.*?>", "", value))
+                for key, value in markdown_tups
+            ]
+        else:
+            markdown_tups = [
+                (key, re.sub("\n", "", value)) for key, value in markdown_tups
+            ]
+
+        return markdown_tups
+
+    def remove_images(self, content: str) -> str:
+        """Get a dictionary of a markdown file from its path."""
+        pattern = r"!{1}\[\[(.*)\]\]"
+        content = re.sub(pattern, "", content)
+        return content
+
+    # def remove_tables(self, content: str) -> List[List[str]]:
+    #     """Convert markdown tables to nested lists."""
+    #     table_rows_pattern = r"((\r?\n){2}|^)([^\r\n]*\|[^\r\n]*(\r?\n)?)+(?=(\r?\n){2}|$)"
+    #     table_cells_pattern = r"([^\|\r\n]*)\|"
+    #
+    #     table_rows = re.findall(table_rows_pattern, content, re.MULTILINE)
+    #     table_lists = []
+    #     for row in table_rows:
+    #         cells = re.findall(table_cells_pattern, row[2])
+    #         cells = [cell.strip() for cell in cells if cell.strip()]
+    #         table_lists.append(cells)
+    #     return str(table_lists)
+
+    def remove_hyperlinks(self, content: str) -> str:
+        """Get a dictionary of a markdown file from its path."""
+        pattern = r"\[(.*?)\]\((.*?)\)"
+        content = re.sub(pattern, r"\1", content)
+        return content
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+        return {}
+
+    def parse_tups(
+            self, filepath: Path, errors: str = "ignore"
+    ) -> List[Tuple[Optional[str], str]]:
+        """Parse file into tuples."""
+        with open(filepath, "r") as f:
+            content = f.read()
+        if self._remove_hyperlinks:
+            content = self.remove_hyperlinks(content)
+        if self._remove_images:
+            content = self.remove_images(content)
+        # if self._remove_tables:
+        #     content = self.remove_tables(content)
+        markdown_tups = self.markdown_to_tups(content)
+        return markdown_tups
+
+    def parse_file(
+            self, filepath: Path, errors: str = "ignore"
+    ) -> Union[str, List[str]]:
+        """Parse file into string."""
+        tups = self.parse_tups(filepath, errors=errors)
+        results = []
+        # TODO: don't include headers right now
+        for header, value in tups:
+            if header is None:
+                results.append(value)
+            else:
+                results.append(f"\n\n{header}\n{value}")
+        return results

application/parser/file/openapi3_parser.py

@@ -0,0 +1,51 @@
+from urllib.parse import urlparse
+
+from openapi_parser import parse
+
+try:
+    from application.parser.file.base_parser import BaseParser
+except ModuleNotFoundError:
+    from base_parser import BaseParser
+
+
+class OpenAPI3Parser(BaseParser):
+    def init_parser(self) -> None:
+        return super().init_parser()
+
+    def get_base_urls(self, urls):
+        base_urls = []
+        for i in urls:
+            parsed_url = urlparse(i)
+            base_url = parsed_url.scheme + "://" + parsed_url.netloc
+            if base_url not in base_urls:
+                base_urls.append(base_url)
+        return base_urls
+
+    def get_info_from_paths(self, path):
+        info = ""
+        if path.operations:
+            for operation in path.operations:
+                info += (
+                    f"\n{operation.method.value}="
+                    f"{operation.responses[0].description}"
+                )
+        return info
+
+    def parse_file(self, file_path):
+        data = parse(file_path)
+        results = ""
+        base_urls = self.get_base_urls(link.url for link in data.servers)
+        base_urls = ",".join([base_url for base_url in base_urls])
+        results += f"Base URL:{base_urls}\n"
+        i = 1
+        for path in data.paths:
+            info = self.get_info_from_paths(path)
+            results += (
+                f"Path{i}: {path.url}\n"
+                f"description: {path.description}\n"
+                f"parameters: {path.parameters}\nmethods: {info}\n"
+            )
+            i += 1
+        with open("results.txt", "w") as f:
+            f.write(results)
+        return results

application/parser/file/pptx_parser.py

@@ -0,0 +1,75 @@
+"""PPT parser.
+Contains parsers for presentation (.pptx) files to extract slide text.
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from application.parser.file.base_parser import BaseParser
+
+class PPTXParser(BaseParser):
+    r"""PPTX (.pptx) parser for extracting text from PowerPoint slides.
+    Args:
+        concat_slides (bool): Specifies whether to concatenate all slide text into one document.
+            - If True, slide texts will be joined together as a single string.
+            - If False, each slide's text will be stored as a separate entry in a list.
+            Set to True by default.
+        slide_separator (str): Separator used to join slides' text content.
+            Only used when `concat_slides=True`. Default is "\n".
+        Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_slides: bool = True,
+        slide_separator: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_slides = concat_slides
+        self._slide_separator = slide_separator
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        r"""
+        Parse a .pptx file and extract text from each slide.
+        Args:
+            file (Path): Path to the .pptx file.
+            errors (str): Error handling policy ('ignore' by default).
+        Returns:
+            Union[str, List[str]]: Concatenated text if concat_slides is True,
+            otherwise a list of slide texts.
+        """
+
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError("pptx module is required to read .PPTX files.")
+
+        try:
+            presentation = Presentation(file)
+            slide_texts=[]
+
+            # Iterate over each slide in the presentation
+            for slide in presentation.slides:
+                slide_text=""
+
+                # Iterate over each shape in the slide
+                for shape in slide.shapes:
+                    # Check if the shape has a 'text' attribute and append that to the slide_text
+                    if hasattr(shape,"text"):
+                        slide_text+=shape.text
+
+                slide_texts.append(slide_text.strip())
+
+            if self._concat_slides:
+                return self._slide_separator.join(slide_texts)
+            else:
+                return slide_texts
+
+        except Exception as e:
+            raise e

application/parser/file/rst_parser.py

@@ -0,0 +1,201 @@
+"""reStructuredText parser.
+
+Contains parser for md files.
+
+"""
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+from application.parser.file.base_parser import BaseParser
+
+
+class RstParser(BaseParser):
+    """reStructuredText parser.
+
+    Extract text from .rst files.
+    Returns dictionary with keys as headers and values as the text between headers.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            remove_hyperlinks: bool = True,
+            remove_images: bool = True,
+            remove_table_excess: bool = True,
+            remove_interpreters: bool = True,
+            remove_directives: bool = True,
+            remove_whitespaces_excess: bool = True,
+            # Be careful with remove_characters_excess, might cause data loss
+            remove_characters_excess: bool = True,
+            **kwargs: Any,
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._remove_hyperlinks = remove_hyperlinks
+        self._remove_images = remove_images
+        self._remove_table_excess = remove_table_excess
+        self._remove_interpreters = remove_interpreters
+        self._remove_directives = remove_directives
+        self._remove_whitespaces_excess = remove_whitespaces_excess
+        self._remove_characters_excess = remove_characters_excess
+
+    def rst_to_tups(self, rst_text: str) -> List[Tuple[Optional[str], str]]:
+        """Convert a reStructuredText file to a dictionary.
+
+        The keys are the headers and the values are the text under each header.
+
+        """
+        rst_tups: List[Tuple[Optional[str], str]] = []
+        lines = rst_text.split("\n")
+
+        current_header = None
+        current_text = ""
+
+        for i, line in enumerate(lines):
+            header_match = re.match(r"^[^\S\n]*[-=]+[^\S\n]*$", line)
+            if header_match and i > 0 and (
+                    len(lines[i - 1].strip()) == len(header_match.group().strip()) or lines[i - 2] == lines[i - 2]):
+                if current_header is not None:
+                    if current_text == "" or None:
+                        continue
+                    # removes the next heading from current Document
+                    if current_text.endswith(lines[i - 1] + "\n"):
+                        current_text = current_text[:len(current_text) - len(lines[i - 1] + "\n")]
+                    rst_tups.append((current_header, current_text))
+
+                current_header = lines[i - 1]
+                current_text = ""
+            else:
+                current_text += line + "\n"
+
+        rst_tups.append((current_header, current_text))
+
+        # TODO: Format for rst
+        #
+        # if current_header is not None:
+        #     # pass linting, assert keys are defined
+        #     rst_tups = [
+        #         (re.sub(r"#", "", cast(str, key)).strip(), re.sub(r"<.*?>", "", value))
+        #         for key, value in rst_tups
+        #     ]
+        # else:
+        #     rst_tups = [
+        #         (key, re.sub("\n", "", value)) for key, value in rst_tups
+        #     ]
+
+        if current_header is None:
+            rst_tups = [
+                (key, re.sub("\n", "", value)) for key, value in rst_tups
+            ]
+        return rst_tups
+
+    def chunk_by_token_count(self, text: str, max_tokens: int = 100) -> List[str]:
+        """Chunk text by token count."""
+    
+        avg_token_length = 5
+    
+        chunk_size = max_tokens * avg_token_length
+
+        chunks = []
+        for i in range(0, len(text), chunk_size):
+            chunk = text[i:i+chunk_size]
+            if i + chunk_size < len(text):
+                last_space = chunk.rfind(' ')
+                if last_space != -1:
+                    chunk = chunk[:last_space]
+            
+            chunks.append(chunk.strip())
+        
+        return chunks
+    
+    def remove_images(self, content: str) -> str:
+        pattern = r"\.\. image:: (.*)"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_hyperlinks(self, content: str) -> str:
+        pattern = r"`(.*?) <(.*?)>`_"
+        content = re.sub(pattern, r"\1", content)
+        return content
+
+    def remove_directives(self, content: str) -> str:
+        """Removes reStructuredText Directives"""
+        pattern = r"`\.\.([^:]+)::"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_interpreters(self, content: str) -> str:
+        """Removes reStructuredText Interpreted Text Roles"""
+        pattern = r":(\w+):"
+        content = re.sub(pattern, "", content)
+        return content
+
+    def remove_table_excess(self, content: str) -> str:
+        """Pattern to remove grid table separators"""
+        pattern = r"^\+[-]+\+[-]+\+$"
+        content = re.sub(pattern, "", content, flags=re.MULTILINE)
+        return content
+
+    def remove_whitespaces_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Pattern to match 2 or more consecutive whitespaces"""
+        pattern = r"\s{2,}"
+        content = [(key, re.sub(pattern, "  ", value)) for key, value in content]
+        return content
+
+    def remove_characters_excess(self, content: List[Tuple[str, Any]]) -> List[Tuple[str, Any]]:
+        """Pattern to match 2 or more consecutive characters"""
+        pattern = r"(\S)\1{2,}"
+        content = [(key, re.sub(pattern, r"\1\1\1", value, flags=re.MULTILINE)) for key, value in content]
+        return content
+
+    def _init_parser(self) -> Dict:
+        """Initialize the parser with the config."""
+        return {}
+
+    def parse_tups(
+            self, filepath: Path, errors: str = "ignore",max_tokens: Optional[int] = 1000
+    ) -> List[Tuple[Optional[str], str]]:
+        """Parse file into tuples."""
+        with open(filepath, "r") as f:
+            content = f.read()
+        if self._remove_hyperlinks:
+            content = self.remove_hyperlinks(content)
+        if self._remove_images:
+            content = self.remove_images(content)
+        if self._remove_table_excess:
+            content = self.remove_table_excess(content)
+        if self._remove_directives:
+            content = self.remove_directives(content)
+        if self._remove_interpreters:
+            content = self.remove_interpreters(content)
+        rst_tups = self.rst_to_tups(content)
+        if self._remove_whitespaces_excess:
+            rst_tups = self.remove_whitespaces_excess(rst_tups)
+        if self._remove_characters_excess:
+            rst_tups = self.remove_characters_excess(rst_tups)
+
+        # Apply chunking if max_tokens is provided
+        if max_tokens is not None:
+            chunked_tups = []
+            for header, text in rst_tups:
+                chunks = self.chunk_by_token_count(text, max_tokens)
+                for idx, chunk in enumerate(chunks):
+                    chunked_tups.append((f"{header} - Chunk {idx + 1}", chunk))
+            return chunked_tups    
+        return rst_tups
+
+    def parse_file(
+            self, filepath: Path, errors: str = "ignore"
+    ) -> Union[str, List[str]]:
+        """Parse file into string."""
+        tups = self.parse_tups(filepath, errors=errors)
+        results = []
+        # TODO: don't include headers right now
+        for header, value in tups:
+            if header is None:
+                results.append(value)
+            else:
+                results.append(f"\n\n{header}\n{value}")
+        return results

application/parser/file/tabular_parser.py

@@ -0,0 +1,221 @@
+"""Tabular parser.
+
+Contains parsers for tabular data files.
+
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from application.parser.file.base_parser import BaseParser
+
+
+class CSVParser(BaseParser):
+    """CSV parser.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+    """
+
+    def __init__(self, *args: Any, concat_rows: bool = True, **kwargs: Any) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file.
+
+        Returns:
+            Union[str, List[str]]: a string or a List of strings.
+
+        """
+        try:
+            import csv
+        except ImportError:
+            raise ValueError("csv module is required to read CSV files.")
+        text_list = []
+        with open(file, "r") as fp:
+            csv_reader = csv.reader(fp)
+            for row in csv_reader:
+                text_list.append(", ".join(row))
+        if self._concat_rows:
+            return "\n".join(text_list)
+        else:
+            return text_list
+
+
+class PandasCSVParser(BaseParser):
+    r"""Pandas-based CSV parser.
+
+    Parses CSVs using the separator detection from Pandas `read_csv`function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_csv` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the separators, table head, etc. on its own.
+            
+        header_period (int): Controls how headers are included in output:
+            - 0: Headers only at the beginning
+            - 1: Headers in every row
+            - N > 1: Headers every N rows
+            
+        header_prefix (str): Prefix for header rows. Default is "HEADERS: ".
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            header_period: int = 20,
+            header_prefix: str = "HEADERS: ",
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+        self._header_period = header_period
+        self._header_prefix = header_prefix
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read CSV files.")
+
+        df = pd.read_csv(file, **self._pandas_config)
+        headers = df.columns.tolist()
+        header_row = f"{self._header_prefix}{self._col_joiner.join(headers)}"
+
+        if not self._concat_rows:
+            return df.apply(
+                lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+            ).tolist()
+        
+        text_list = []
+        if self._header_period != 1:
+            text_list.append(header_row)
+        
+        for i, row in df.iterrows():
+            if (self._header_period > 1 and i > 0 and i % self._header_period == 0):
+                text_list.append(header_row)
+            text_list.append(self._col_joiner.join(row.astype(str).tolist()))
+            if self._header_period == 1 and i < len(df) - 1:
+                text_list.append(header_row)
+
+        return self._row_joiner.join(text_list)
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+            
+        header_period (int): Controls how headers are included in output:
+            - 0: Headers only at the beginning (default)
+            - 1: Headers in every row
+            - N > 1: Headers every N rows
+            
+        header_prefix (str): Prefix for header rows. Default is "HEADERS: ".
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            header_period: int = 20,
+            header_prefix: str = "HEADERS: ",
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+        self._header_period = header_period
+        self._header_prefix = header_prefix
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+        headers = df.columns.tolist()
+        header_row = f"{self._header_prefix}{self._col_joiner.join(headers)}"
+        
+        if not self._concat_rows:
+            return df.apply(
+                lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+            ).tolist()
+        
+        text_list = []
+        if self._header_period != 1:
+            text_list.append(header_row)
+
+        for i, row in df.iterrows():
+            if (self._header_period > 1 and i > 0 and i % self._header_period == 0):
+                text_list.append(header_row)
+            text_list.append(self._col_joiner.join(row.astype(str).tolist()))
+            if self._header_period == 1 and i < len(df) - 1:
+                text_list.append(header_row)
+        return self._row_joiner.join(text_list)

application/parser/remote/base.py

@@ -0,0 +1,19 @@
+"""Base reader class."""
+from abc import abstractmethod
+from typing import Any, List
+
+from langchain_core.documents import Document as LCDocument
+from application.parser.schema.base import Document
+
+
+class BaseRemote:
+    """Utilities for loading data from a directory."""
+
+    @abstractmethod
+    def load_data(self, *args: Any, **load_kwargs: Any) -> List[Document]:
+        """Load data from the input directory."""
+
+    def load_langchain_documents(self, **load_kwargs: Any) -> List[LCDocument]:
+        """Load data in LangChain document format."""
+        docs = self.load_data(**load_kwargs)
+        return [d.to_langchain_format() for d in docs]

application/parser/remote/crawler_loader.py

@@ -0,0 +1,95 @@
+import logging
+import os
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin, urlparse
+
+from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+from application.core.url_validation import validate_url, SSRFError
+from application.security.safe_url import pinned_request
+
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10):
+        self.limit = limit  # Set the limit for the number of pages to scrape
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            url = validate_url(url)
+        except SSRFError as e:
+            logging.error(f"URL validation failed: {e}")
+            return []
+
+        visited_urls = set()
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
+        urls_to_visit = [url]
+        loaded_content = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop(0)
+            visited_urls.add(current_url)
+
+            try:
+                response = pinned_request("GET", current_url, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                loaded_content.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info={
+                            "source": current_url,
+                            "file_path": self._url_to_virtual_path(current_url),
+                        },
+                    )
+                )
+            except Exception as e:
+                logging.error(f"Error processing URL {current_url}: {e}", exc_info=True)
+                continue
+
+            # Parse the HTML content to extract all links
+            all_links = [
+                urljoin(current_url, a['href'])
+                for a in soup.find_all('a', href=True)
+                if base_url in urljoin(current_url, a['href'])
+            ]
+
+            # Add new links to the list of URLs to visit if they haven't been visited yet
+            urls_to_visit.extend([link for link in all_links if link not in visited_urls])
+            urls_to_visit = list(set(urls_to_visit))
+
+            # Stop crawling if the limit of pages to scrape is reached
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return loaded_content
+
+    def _url_to_virtual_path(self, url):
+        """
+        Convert a URL to a virtual file path ending with .md.
+
+        Examples:
+            https://docs.docsgpt.cloud/ -> index.md
+            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
+            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
+            https://example.com/page.html -> page.md
+        """
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+
+        if not path:
+            return "index.md"
+
+        # Remove common file extensions and add .md
+        base, ext = os.path.splitext(path)
+        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
+            path = base
+
+        if not path.endswith(".md"):
+            path = f"{path}.md"
+
+        return path

application/parser/remote/crawler_markdown.py

@@ -0,0 +1,181 @@
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from application.parser.remote.base import BaseRemote
+from application.core.url_validation import validate_url, SSRFError
+from application.security.safe_url import UnsafeUserUrlError, pinned_request
+import re
+from markdownify import markdownify
+from application.parser.schema.base import Document
+import tldextract
+import os
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10, allow_subdomains=False):
+        """
+        Given a URL crawl web pages up to `self.limit`,
+        convert HTML content to Markdown, and returning a list of Document objects.
+
+        :param limit: The maximum number of pages to crawl.
+        :param allow_subdomains: If True, crawl pages on subdomains of the base domain.
+        """
+        self.limit = limit
+        self.allow_subdomains = allow_subdomains
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            url = validate_url(url)
+        except SSRFError as e:
+            print(f"URL validation failed: {e}")
+            return []
+
+        # Keep track of visited URLs to avoid revisiting the same page
+        visited_urls = set()
+
+        # Determine the base domain for link filtering using tldextract
+        base_domain = self._get_base_domain(url)
+        urls_to_visit = {url}
+        documents = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop()
+
+            # Skip if already visited
+            if current_url in visited_urls:
+                continue
+            visited_urls.add(current_url)
+
+            # Fetch the page content
+            html_content = self._fetch_page(current_url)
+            if html_content is None:
+                continue
+
+            # Convert the HTML to Markdown for cleaner text formatting
+            title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)
+            if processed_markdown:
+                # Generate virtual file path from URL for consistent file-like matching
+                virtual_path = self._url_to_virtual_path(current_url)
+                
+                # Create a Document for each visited page
+                documents.append(
+                    Document(
+                        processed_markdown,  # content
+                        None,  # doc_id
+                        None,  # embedding
+                        {
+                            "source": current_url,
+                            "title": title,
+                            "language": language,
+                            "file_path": virtual_path,
+                        },  # extra_info
+                    )
+                )
+
+            # Extract links and filter them according to domain rules
+            new_links = self._extract_links(html_content, current_url)
+            filtered_links = self._filter_links(new_links, base_domain)
+
+            # Add any new, not-yet-visited links to the queue
+            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+
+            # If we've reached the limit, stop crawling
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return documents
+
+    def _fetch_page(self, url):
+        try:
+            response = pinned_request("GET", url, timeout=10)
+            response.raise_for_status()
+            return response.text
+        except UnsafeUserUrlError as e:
+            print(f"URL validation failed for {url}: {e}")
+            return None
+        except Exception as e:
+            print(f"Error fetching URL {url}: {e}")
+            return None
+
+    def _process_html_to_markdown(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title_tag = soup.find('title')
+        title = title_tag.text.strip() if title_tag else "No Title"
+
+        # Extract language
+        language_tag = soup.find('html')
+        language = language_tag.get('lang', 'en') if language_tag else "en"
+
+        markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+        # Reduce sequences of more than two newlines to exactly three
+        markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified)
+        return title, language, markdownified
+
+    def _extract_links(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        links = []
+        for a in soup.find_all('a', href=True):
+            full_url = urljoin(current_url, a['href'])
+            links.append((full_url, a.text.strip()))
+        return links
+
+    def _get_base_domain(self, url):
+        extracted = tldextract.extract(url)
+        # Reconstruct the domain as domain.suffix
+        base_domain = f"{extracted.domain}.{extracted.suffix}"
+        return base_domain
+
+    def _filter_links(self, links, base_domain):
+        """
+        Filter the extracted links to only include those that match the crawling criteria:
+        - If allow_subdomains is True, allow any link whose domain ends with the base_domain.
+        - If allow_subdomains is False, only allow exact matches of the base_domain.
+        """
+        filtered = []
+        for link, _ in links:
+            parsed_link = urlparse(link)
+            if not parsed_link.netloc:
+                continue
+
+            extracted = tldextract.extract(parsed_link.netloc)
+            link_base = f"{extracted.domain}.{extracted.suffix}"
+
+            if self.allow_subdomains:
+                # For subdomains: sub.example.com ends with example.com
+                if link_base == base_domain or link_base.endswith("." + base_domain):
+                    filtered.append(link)
+            else:
+                # Exact domain match
+                if link_base == base_domain:
+                    filtered.append(link)
+        return filtered
+
+    def _url_to_virtual_path(self, url):
+        """
+        Convert a URL to a virtual file path ending with .md.
+
+        Examples:
+            https://docs.docsgpt.cloud/ -> index.md
+            https://docs.docsgpt.cloud/guides/setup -> guides/setup.md
+            https://docs.docsgpt.cloud/guides/setup/ -> guides/setup.md
+            https://example.com/page.html -> page.md
+        """
+        parsed = urlparse(url)
+        path = parsed.path.strip("/")
+
+        if not path:
+            return "index.md"
+
+        # Remove common file extensions and add .md
+        base, ext = os.path.splitext(path)
+        if ext.lower() in [".html", ".htm", ".php", ".asp", ".aspx", ".jsp"]:
+            path = base
+
+        # Ensure path ends with .md
+        if not path.endswith(".md"):
+            path = path + ".md"
+
+        return path

application/parser/remote/github_loader.py

@@ -0,0 +1,158 @@
+import base64
+import requests
+import time
+from typing import List, Optional
+from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+import mimetypes
+from application.core.settings import settings
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = settings.GITHUB_ACCESS_TOKEN
+        self.headers = {
+            "Authorization": f"token {self.access_token}",
+            "Accept": "application/vnd.github.v3+json"
+        } if self.access_token else {
+            "Accept": "application/vnd.github.v3+json"
+        }
+        return
+
+    def is_text_file(self, file_path: str) -> bool:
+        """Determine if a file is a text file based on extension."""
+        # Common text file extensions
+        text_extensions = {
+            '.txt', '.md', '.markdown', '.rst', '.json', '.xml', '.yaml', '.yml',
+            '.py', '.js', '.ts', '.jsx', '.tsx', '.java', '.c', '.cpp', '.h', '.hpp',
+            '.cs', '.go', '.rs', '.rb', '.php', '.swift', '.kt', '.scala',
+            '.html', '.css', '.scss', '.sass', '.less',
+            '.sh', '.bash', '.zsh', '.fish',
+            '.sql', '.r', '.m', '.mat',
+            '.ini', '.cfg', '.conf', '.config', '.env',
+            '.gitignore', '.dockerignore', '.editorconfig',
+            '.log', '.csv', '.tsv'
+        }
+
+        # Get file extension
+        file_lower = file_path.lower()
+        for ext in text_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        # Also check MIME type
+        mime_type, _ = mimetypes.guess_type(file_path)
+        if mime_type and (mime_type.startswith("text") or mime_type in ["application/json", "application/xml"]):
+            return True
+
+        return False
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> Optional[str]:
+        """Fetch file content. Returns None if file should be skipped (binary files or empty files)."""
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = self._make_request(url)
+
+        content = response.json()
+
+        if content.get("encoding") == "base64":
+            if self.is_text_file(file_path):  # Handle only text files
+                try:
+                    decoded_content = base64.b64decode(content["content"]).decode("utf-8").strip()
+                    # Skip empty files
+                    if not decoded_content:
+                        return None
+                    return decoded_content
+                except Exception:
+                    # If decoding fails, it's probably a binary file
+                    return None
+            else:
+                # Skip binary files by returning None
+                return None
+        else:
+            file_content = content['content'].strip()
+            # Skip empty files
+            if not file_content:
+                return None
+            return file_content
+
+    def _make_request(self, url: str, max_retries: int = 3) -> requests.Response:
+        """Make a request with retry logic for rate limiting"""
+        for attempt in range(max_retries):
+            response = requests.get(url, headers=self.headers, timeout=100)
+
+            if response.status_code == 200:
+                return response
+            elif response.status_code == 403:
+                # Check if it's a rate limit issue
+                try:
+                    error_data = response.json()
+                    error_msg = error_data.get("message", "")
+
+                    # Check rate limit headers
+                    remaining = response.headers.get("X-RateLimit-Remaining", "unknown")
+                    reset_time = response.headers.get("X-RateLimit-Reset", "unknown")
+
+                    print(f"GitHub API 403 Error: {error_msg}")
+                    print(f"Rate limit remaining: {remaining}, Reset time: {reset_time}")
+
+                    if "rate limit" in error_msg.lower():
+                        if attempt < max_retries - 1:
+                            wait_time = 2 ** attempt  # Exponential backoff
+                            print(f"Rate limit hit, waiting {wait_time} seconds before retry...")
+                            time.sleep(wait_time)
+                            continue
+
+                    # Provide helpful error message
+                    if remaining == "0":
+                        raise Exception(f"GitHub API rate limit exceeded. Please set GITHUB_ACCESS_TOKEN environment variable. Reset time: {reset_time}")
+                    else:
+                        raise Exception(f"GitHub API error: {error_msg}. This may require authentication - set GITHUB_ACCESS_TOKEN environment variable.")
+                except Exception as e:
+                    if isinstance(e, Exception) and "GitHub API" in str(e):
+                        raise
+                    # If we can't parse the response, raise the original error
+                    response.raise_for_status()
+            else:
+                response.raise_for_status()
+
+        return response
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = self._make_request(url)
+
+        contents = response.json()
+
+        # Handle error responses from GitHub API
+        if isinstance(contents, dict) and "message" in contents:
+            raise Exception(f"GitHub API error: {contents.get('message')}")
+
+        # Ensure contents is a list
+        if not isinstance(contents, list):
+            raise TypeError(f"Expected list from GitHub API, got {type(contents).__name__}: {contents}")
+
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            # Skip binary files (content is None)
+            if content is None:
+                continue
+            documents.append(Document(
+                text=content,
+                doc_id=file_path,
+                extra_info={
+                    "title": file_path,
+                    "source": f"https://github.com/{repo_name}/blob/main/{file_path}"
+                }
+            ))
+        return documents

application/parser/remote/reddit_loader.py

@@ -0,0 +1,35 @@
+from application.parser.remote.base import BaseRemote
+from langchain_community.document_loaders import RedditPostsLoader
+import json
+
+
+class RedditPostsLoaderRemote(BaseRemote):
+    def load_data(self, inputs):
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
+        client_id = data.get("client_id")
+        client_secret = data.get("client_secret")
+        user_agent = data.get("user_agent")
+        categories = data.get("categories", ["new", "hot"])
+        mode = data.get("mode", "subreddit")
+        search_queries = data.get("search_queries")
+        number_posts = data.get("number_posts", 10)
+        self.loader = RedditPostsLoader(
+            client_id=client_id,
+            client_secret=client_secret,
+            user_agent=user_agent,
+            categories=categories,
+            mode=mode,
+            search_queries=search_queries,
+            number_posts=number_posts,
+        )
+        documents = self.loader.load()
+        print(f"Loaded {len(documents)} documents from Reddit")
+        return documents

application/parser/remote/remote_creator.py

@@ -0,0 +1,92 @@
+import json
+
+from application.parser.remote.sitemap_loader import SitemapLoader
+from application.parser.remote.crawler_loader import CrawlerLoader
+from application.parser.remote.web_loader import WebLoader
+from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader
+from application.parser.remote.s3_loader import S3Loader
+
+
+class RemoteCreator:
+    """
+    Factory class for creating remote content loaders.
+
+    These loaders fetch content from remote web sources like URLs,
+    sitemaps, web crawlers, social media platforms, etc.
+
+    For external knowledge base connectors (like Google Drive),
+    use ConnectorCreator instead.
+    """
+
+    loaders = {
+        "url": WebLoader,
+        "sitemap": SitemapLoader,
+        "crawler": CrawlerLoader,
+        "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
+        "s3": S3Loader,
+    }
+
+    @classmethod
+    def create_loader(cls, type, *args, **kwargs):
+        loader_class = cls.loaders.get(type.lower())
+        if not loader_class:
+            raise ValueError(f"No loader class found for type {type}")
+        return loader_class(*args, **kwargs)
+
+
+# Loader types whose load_data expects a URL string, not a config dict.
+_URL_LOADER_TYPES = {"url", "crawler", "sitemap", "github"}
+
+# Keys a remote_data dict may hold the URL under (``raw`` is the legacy shape).
+_URL_DATA_KEYS = ("url", "urls", "repo_url", "raw")
+
+
+def normalize_remote_data(source_type, remote_data):
+    """Convert a stored ``sources.remote_data`` JSONB value into the
+    ``source_data`` shape the matching loader expects.
+
+    Args:
+        source_type: The ``sources.type`` value (the loader name).
+        remote_data: The stored ``remote_data`` (dict, list, str, or None).
+
+    Returns:
+        Loader input: a URL string or list for url/crawler/sitemap/github,
+        a JSON string for reddit, a dict for s3; ``None`` when the row has
+        nothing syncable.
+    """
+    if remote_data is None:
+        return None
+
+    # Some legacy rows stored the JSON itself as a string.
+    if isinstance(remote_data, str):
+        stripped = remote_data.strip()
+        if stripped[:1] in ("{", "["):
+            try:
+                remote_data = json.loads(stripped)
+            except json.JSONDecodeError:
+                # Not actually JSON — leave remote_data as the original
+                # string; the per-loader branches below handle a string.
+                pass
+
+    loader = (source_type or "").lower()
+
+    if loader in _URL_LOADER_TYPES:
+        if isinstance(remote_data, dict):
+            for key in _URL_DATA_KEYS:
+                value = remote_data.get(key)
+                if value:
+                    return value
+            # No URL key — None keeps the loader off the dict-crash path.
+            return None
+        return remote_data
+
+    if loader == "reddit":
+        # reddit's loader runs json.loads() on its input — needs a string.
+        if isinstance(remote_data, (dict, list)):
+            return json.dumps(remote_data)
+        return remote_data
+
+    # s3's loader accepts a dict or JSON string; pass it through unchanged.
+    return remote_data

application/parser/remote/s3_loader.py

@@ -0,0 +1,433 @@
+import json
+import logging
+import os
+import tempfile
+import mimetypes
+from typing import List, Optional
+from application.core.url_validation import SSRFError, validate_url
+from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError, NoCredentialsError
+except ImportError:
+    boto3 = None
+
+logger = logging.getLogger(__name__)
+
+
+class S3Loader(BaseRemote):
+    """Load documents from an AWS S3 bucket."""
+
+    def __init__(self):
+        if boto3 is None:
+            raise ImportError(
+                "boto3 is required for S3Loader. Install it with: pip install boto3"
+            )
+        self.s3_client = None
+
+    def _normalize_endpoint_url(self, endpoint_url: str, bucket: str) -> tuple[str, str]:
+        """
+        Normalize endpoint URL for S3-compatible services.
+
+        Detects common mistakes like using bucket-prefixed URLs and extracts
+        the correct endpoint and bucket name.
+
+        Args:
+            endpoint_url: The provided endpoint URL
+            bucket: The provided bucket name
+
+        Returns:
+            Tuple of (normalized_endpoint_url, bucket_name)
+        """
+        import re
+        from urllib.parse import urlparse
+
+        if not endpoint_url:
+            return endpoint_url, bucket
+
+        parsed = urlparse(endpoint_url)
+        host = parsed.netloc or parsed.path
+
+        # Check for DigitalOcean Spaces bucket-prefixed URL pattern
+        # e.g., https://mybucket.nyc3.digitaloceanspaces.com
+        do_match = re.match(r"^([^.]+)\.([a-z0-9]+)\.digitaloceanspaces\.com$", host)
+        if do_match:
+            extracted_bucket = do_match.group(1)
+            region = do_match.group(2)
+            correct_endpoint = f"https://{region}.digitaloceanspaces.com"
+            logger.warning(
+                f"Detected bucket-prefixed DigitalOcean Spaces URL. "
+                f"Extracted bucket '{extracted_bucket}' from endpoint. "
+                f"Using endpoint: {correct_endpoint}"
+            )
+            # If bucket wasn't provided or differs, use extracted one
+            if not bucket or bucket != extracted_bucket:
+                logger.info(f"Using extracted bucket name: '{extracted_bucket}' (was: '{bucket}')")
+                bucket = extracted_bucket
+            return correct_endpoint, bucket
+
+        # Check for just "digitaloceanspaces.com" without region
+        if host == "digitaloceanspaces.com":
+            logger.error(
+                "Invalid DigitalOcean Spaces endpoint: missing region. "
+                "Use format: https://<region>.digitaloceanspaces.com (e.g., https://lon1.digitaloceanspaces.com)"
+            )
+
+        return endpoint_url, bucket
+
+    def _init_client(
+        self,
+        aws_access_key_id: str,
+        aws_secret_access_key: str,
+        region_name: str = "us-east-1",
+        endpoint_url: Optional[str] = None,
+        bucket: Optional[str] = None,
+    ) -> Optional[str]:
+        """
+        Initialize the S3 client with credentials.
+
+        Returns:
+            The potentially corrected bucket name if endpoint URL was normalized
+        """
+        from botocore.config import Config
+
+        client_kwargs = {
+            "aws_access_key_id": aws_access_key_id,
+            "aws_secret_access_key": aws_secret_access_key,
+            "region_name": region_name,
+        }
+
+        logger.info(f"Initializing S3 client with region: {region_name}")
+
+        corrected_bucket = bucket
+        if endpoint_url:
+            # Normalize the endpoint URL and potentially extract bucket name
+            normalized_endpoint, corrected_bucket = self._normalize_endpoint_url(endpoint_url, bucket)
+            logger.info(f"Original endpoint URL: {endpoint_url}")
+            logger.info(f"Normalized endpoint URL: {normalized_endpoint}")
+            logger.info(f"Bucket name: '{corrected_bucket}'")
+
+            try:
+                normalized_endpoint = validate_url(normalized_endpoint)
+            except SSRFError as e:
+                raise ValueError(f"Invalid S3 endpoint_url: {e}") from e
+
+            client_kwargs["endpoint_url"] = normalized_endpoint
+            # Use path-style addressing for S3-compatible services
+            # (DigitalOcean Spaces, MinIO, etc.)
+            client_kwargs["config"] = Config(s3={"addressing_style": "path"})
+        else:
+            logger.info("Using default AWS S3 endpoint")
+
+        self.s3_client = boto3.client("s3", **client_kwargs)
+        logger.info("S3 client initialized successfully")
+
+        return corrected_bucket
+
+    def is_text_file(self, file_path: str) -> bool:
+        """Determine if a file is a text file based on extension."""
+        text_extensions = {
+            ".txt",
+            ".md",
+            ".markdown",
+            ".rst",
+            ".json",
+            ".xml",
+            ".yaml",
+            ".yml",
+            ".py",
+            ".js",
+            ".ts",
+            ".jsx",
+            ".tsx",
+            ".java",
+            ".c",
+            ".cpp",
+            ".h",
+            ".hpp",
+            ".cs",
+            ".go",
+            ".rs",
+            ".rb",
+            ".php",
+            ".swift",
+            ".kt",
+            ".scala",
+            ".html",
+            ".css",
+            ".scss",
+            ".sass",
+            ".less",
+            ".sh",
+            ".bash",
+            ".zsh",
+            ".fish",
+            ".sql",
+            ".r",
+            ".m",
+            ".mat",
+            ".ini",
+            ".cfg",
+            ".conf",
+            ".config",
+            ".env",
+            ".gitignore",
+            ".dockerignore",
+            ".editorconfig",
+            ".log",
+            ".csv",
+            ".tsv",
+        }
+
+        file_lower = file_path.lower()
+        for ext in text_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        mime_type, _ = mimetypes.guess_type(file_path)
+        if mime_type and (
+            mime_type.startswith("text")
+            or mime_type in ["application/json", "application/xml"]
+        ):
+            return True
+
+        return False
+
+    def is_supported_document(self, file_path: str) -> bool:
+        """Check if file is a supported document type for parsing."""
+        document_extensions = {
+            ".pdf",
+            ".docx",
+            ".doc",
+            ".xlsx",
+            ".xls",
+            ".pptx",
+            ".ppt",
+            ".epub",
+            ".odt",
+            ".rtf",
+        }
+
+        file_lower = file_path.lower()
+        for ext in document_extensions:
+            if file_lower.endswith(ext):
+                return True
+
+        return False
+
+    def list_objects(self, bucket: str, prefix: str = "") -> List[str]:
+        """
+        List all objects in the bucket with the given prefix.
+
+        Args:
+            bucket: S3 bucket name
+            prefix: Optional path prefix to filter objects
+
+        Returns:
+            List of object keys
+        """
+        objects = []
+        paginator = self.s3_client.get_paginator("list_objects_v2")
+
+        logger.info(f"Listing objects in bucket: '{bucket}' with prefix: '{prefix}'")
+        logger.debug(f"S3 client endpoint: {self.s3_client.meta.endpoint_url}")
+
+        try:
+            page_count = 0
+            for page in paginator.paginate(Bucket=bucket, Prefix=prefix):
+                page_count += 1
+                logger.debug(f"Processing page {page_count}, keys in response: {list(page.keys())}")
+                if "Contents" in page:
+                    for obj in page["Contents"]:
+                        key = obj["Key"]
+                        if not key.endswith("/"):
+                            objects.append(key)
+                            logger.debug(f"Found object: {key}")
+                else:
+                    logger.info(f"Page {page_count} has no 'Contents' key - bucket may be empty or prefix not found")
+
+            logger.info(f"Found {len(objects)} objects in bucket '{bucket}'")
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            error_message = e.response.get("Error", {}).get("Message", "")
+            logger.error(f"ClientError listing objects - Code: {error_code}, Message: {error_message}")
+            logger.error(f"Full error response: {e.response}")
+            logger.error(f"Bucket: '{bucket}', Prefix: '{prefix}', Endpoint: {self.s3_client.meta.endpoint_url}")
+
+            if error_code == "NoSuchBucket":
+                raise Exception(f"S3 bucket '{bucket}' does not exist")
+            elif error_code == "AccessDenied":
+                raise Exception(
+                    f"Access denied to S3 bucket '{bucket}'. Check your credentials and permissions."
+                )
+            elif error_code == "NoSuchKey":
+                # This is unusual for ListObjectsV2 - may indicate endpoint/bucket configuration issue
+                logger.error(
+                    "NoSuchKey error on ListObjectsV2 - this may indicate the bucket name "
+                    "is incorrect or the endpoint URL format is wrong. "
+                    "For DigitalOcean Spaces, the endpoint should be like: "
+                    "https://<region>.digitaloceanspaces.com and bucket should be just the space name."
+                )
+                raise Exception(
+                    f"S3 error: {e}. For S3-compatible services, verify: "
+                    f"1) Endpoint URL format (e.g., https://nyc3.digitaloceanspaces.com), "
+                    f"2) Bucket name is just the space/bucket name without region prefix"
+                )
+            else:
+                raise Exception(f"S3 error: {e}")
+        except NoCredentialsError:
+            raise Exception(
+                "AWS credentials not found. Please provide valid credentials."
+            )
+
+        return objects
+
+    def get_object_content(self, bucket: str, key: str) -> Optional[str]:
+        """
+        Get the content of an S3 object as text.
+
+        Args:
+            bucket: S3 bucket name
+            key: Object key
+
+        Returns:
+            File content as string, or None if file should be skipped
+        """
+        if not self.is_text_file(key) and not self.is_supported_document(key):
+            return None
+
+        try:
+            response = self.s3_client.get_object(Bucket=bucket, Key=key)
+            content = response["Body"].read()
+
+            if self.is_text_file(key):
+                try:
+                    decoded_content = content.decode("utf-8").strip()
+                    if not decoded_content:
+                        return None
+                    return decoded_content
+                except UnicodeDecodeError:
+                    return None
+            elif self.is_supported_document(key):
+                return self._process_document(content, key)
+
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "")
+            if error_code == "NoSuchKey":
+                return None
+            elif error_code == "AccessDenied":
+                print(f"Access denied to object: {key}")
+                return None
+            else:
+                print(f"Error fetching object {key}: {e}")
+                return None
+
+        return None
+
+    def _process_document(self, content: bytes, key: str) -> Optional[str]:
+        """
+        Process a document file (PDF, DOCX, etc.) and extract text.
+
+        Args:
+            content: File content as bytes
+            key: Object key (filename)
+
+        Returns:
+            Extracted text content
+        """
+        ext = os.path.splitext(key)[1].lower()
+
+        with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file:
+            tmp_file.write(content)
+            tmp_path = tmp_file.name
+
+        try:
+            from application.parser.file.bulk import SimpleDirectoryReader
+
+            reader = SimpleDirectoryReader(input_files=[tmp_path])
+            documents = reader.load_data()
+            if documents:
+                return "\n\n".join(doc.text for doc in documents if doc.text)
+            return None
+        except Exception as e:
+            print(f"Error processing document {key}: {e}")
+            return None
+        finally:
+            if os.path.exists(tmp_path):
+                os.unlink(tmp_path)
+
+    def load_data(self, inputs) -> List[Document]:
+        """
+        Load documents from an S3 bucket.
+
+        Args:
+            inputs: JSON string or dict containing:
+                - aws_access_key_id: AWS access key ID
+                - aws_secret_access_key: AWS secret access key
+                - bucket: S3 bucket name
+                - prefix: Optional path prefix to filter objects
+                - region: AWS region (default: us-east-1)
+                - endpoint_url: Custom S3 endpoint URL (for MinIO, R2, etc.)
+
+        Returns:
+            List of Document objects
+        """
+        if isinstance(inputs, str):
+            try:
+                data = json.loads(inputs)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Invalid JSON input: {e}")
+        else:
+            data = inputs
+
+        required_fields = ["aws_access_key_id", "aws_secret_access_key", "bucket"]
+        missing_fields = [field for field in required_fields if not data.get(field)]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
+
+        aws_access_key_id = data["aws_access_key_id"]
+        aws_secret_access_key = data["aws_secret_access_key"]
+        bucket = data["bucket"]
+        prefix = data.get("prefix", "")
+        region = data.get("region", "us-east-1")
+        endpoint_url = data.get("endpoint_url", "")
+
+        logger.info(f"Loading data from S3 - Bucket: '{bucket}', Prefix: '{prefix}', Region: '{region}'")
+        if endpoint_url:
+            logger.info(f"Custom endpoint URL provided: '{endpoint_url}'")
+
+        corrected_bucket = self._init_client(
+            aws_access_key_id, aws_secret_access_key, region, endpoint_url or None, bucket
+        )
+
+        # Use the corrected bucket name if endpoint URL normalization extracted one
+        if corrected_bucket and corrected_bucket != bucket:
+            logger.info(f"Using corrected bucket name: '{corrected_bucket}' (original: '{bucket}')")
+            bucket = corrected_bucket
+
+        objects = self.list_objects(bucket, prefix)
+        documents = []
+
+        for key in objects:
+            content = self.get_object_content(bucket, key)
+            if content is None:
+                continue
+
+            documents.append(
+                Document(
+                    text=content,
+                    doc_id=key,
+                    extra_info={
+                        "title": os.path.basename(key),
+                        "source": f"s3://{bucket}/{key}",
+                        "bucket": bucket,
+                        "key": key,
+                    },
+                )
+            )
+
+        logger.info(f"Loaded {len(documents)} documents from S3 bucket '{bucket}'")
+        return documents

application/parser/remote/sitemap_loader.py

@@ -0,0 +1,111 @@
+import logging
+import re
+
+import defusedxml.ElementTree as ET
+from bs4 import BeautifulSoup
+
+from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+from application.core.url_validation import validate_url, SSRFError
+from application.security.safe_url import UnsafeUserUrlError, pinned_request
+
+class SitemapLoader(BaseRemote):
+    def __init__(self, limit=20):
+        self.limit = limit  # Adding limit to control the number of URLs to process
+
+    def load_data(self, inputs):
+        sitemap_url= inputs
+        # Check if the input is a list and if it is, use the first element
+        if isinstance(sitemap_url, list) and sitemap_url:
+            sitemap_url = sitemap_url[0]
+
+        # Validate URL to prevent SSRF attacks
+        try:
+            sitemap_url = validate_url(sitemap_url)
+        except SSRFError as e:
+            logging.error(f"URL validation failed: {e}")
+            return []
+
+        urls = self._extract_urls(sitemap_url)
+        if not urls:
+            print(f"No URLs found in the sitemap: {sitemap_url}")
+            return []
+
+        # Load content of extracted URLs
+        documents = []
+        processed_urls = 0  # Counter for processed URLs
+        for url in urls:
+            if self.limit is not None and processed_urls >= self.limit:
+                break  # Stop processing if the limit is reached
+
+            try:
+                url = validate_url(url)
+            except SSRFError as e:
+                logging.error(f"URL validation failed for sitemap entry {url}: {e}")
+                continue
+            try:
+                response = pinned_request("GET", url, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                documents.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info={"source": url},
+                    )
+                )
+                processed_urls += 1  # Increment the counter after processing each URL
+            except Exception as e:
+                logging.error(f"Error processing URL {url}: {e}", exc_info=True)
+                continue
+
+        return documents
+
+    def _extract_urls(self, sitemap_url):
+        try:
+            response = pinned_request("GET", sitemap_url, timeout=30)
+            response.raise_for_status()
+        except UnsafeUserUrlError as e:
+            print(f"URL validation failed for sitemap: {sitemap_url}. Error: {e}")
+            return []
+        except Exception as e:
+            print(f"Failed to fetch sitemap: {sitemap_url}. Error: {e}")
+            return []
+
+        # Determine if this is a sitemap or a URL
+        if self._is_sitemap(response):
+            # It's a sitemap, so parse it and extract URLs
+            return self._parse_sitemap(response.content)
+        else:
+            # It's not a sitemap, return the URL itself
+            return [sitemap_url]
+
+    def _is_sitemap(self, response):
+        content_type = response.headers.get('Content-Type', '')
+        if 'xml' in content_type or response.url.endswith('.xml'):
+            return True
+
+        if '<sitemapindex' in response.text or '<urlset' in response.text:
+            return True
+
+        return False
+
+    def _parse_sitemap(self, sitemap_content):
+        # Remove namespaces
+        sitemap_content = re.sub(' xmlns="[^"]+"', '', sitemap_content.decode('utf-8'), count=1)
+
+        root = ET.fromstring(sitemap_content)
+
+        urls = []
+        for loc in root.findall('.//url/loc'):
+            if not loc.text:
+                continue
+            urls.append(loc.text)
+
+        # Check for nested sitemaps
+        for sitemap in root.findall('.//sitemap/loc'):
+            nested_sitemap_url = sitemap.text
+            if not nested_sitemap_url:
+                continue
+            urls.extend(self._extract_urls(nested_sitemap_url))
+
+        return urls

application/parser/remote/telegram.py

@@ -0,0 +1,11 @@
+from langchain.document_loader import TelegramChatApiLoader
+from application.parser.remote.base import BaseRemote
+
+class TelegramChatApiRemote(BaseRemote):
+    def _init_parser(self, *args, **load_kwargs):
+        self.loader = TelegramChatApiLoader(**load_kwargs)
+        return {}
+
+    def parse_file(self, *args, **load_kwargs):
+
+        return 

application/parser/remote/web_loader.py

@@ -0,0 +1,57 @@
+import logging
+
+from bs4 import BeautifulSoup
+
+from application.core.url_validation import SSRFError, validate_url
+from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+from application.security.safe_url import pinned_request
+
+headers = {
+    "User-Agent": "Mozilla/5.0",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*"
+    ";q=0.8",
+    "Accept-Language": "en-US,en;q=0.5",
+    "Referer": "https://www.google.com/",
+    "DNT": "1",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+}
+
+
+class WebLoader(BaseRemote):
+    def load_data(self, inputs):
+        urls = inputs
+        if isinstance(urls, str):
+            urls = [urls]
+        documents = []
+        for url in urls:
+            try:
+                url = validate_url(url)
+            except SSRFError as e:
+                logging.warning(
+                    f"Skipping URL due to SSRF validation failure: {url} - {e}"
+                )
+                continue
+            try:
+                response = pinned_request("GET", url, headers=headers, timeout=30)
+                response.raise_for_status()
+                soup = BeautifulSoup(response.text, "html.parser")
+                metadata = {"source": url}
+                if soup.title:
+                    title = soup.title.get_text(strip=True)
+                    if title:
+                        metadata["title"] = title
+                html_tag = soup.find("html")
+                if html_tag and html_tag.get("lang"):
+                    metadata["language"] = html_tag.get("lang")
+                documents.append(
+                    Document(
+                        soup.get_text(separator="\n", strip=True),
+                        extra_info=metadata,
+                    )
+                )
+            except Exception as e:
+                logging.error(f"Error processing URL {url}: {e}", exc_info=True)
+                continue
+        return documents

application/parser/schema/__init__.py

@@ -0,0 +1 @@
+

application/parser/schema/base.py

@@ -0,0 +1,34 @@
+"""Base schema for readers."""
+from dataclasses import dataclass
+
+from langchain_core.documents import Document as LCDocument
+from application.parser.schema.schema import BaseDocument
+
+
+@dataclass
+class Document(BaseDocument):
+    """Generic interface for a data document.
+
+    This document connects to data sources.
+
+    """
+
+    def __post_init__(self) -> None:
+        """Post init."""
+        if self.text is None:
+            raise ValueError("text field not set.")
+
+    @classmethod
+    def get_type(cls) -> str:
+        """Get Document type."""
+        return "Document"
+
+    def to_langchain_format(self) -> LCDocument:
+        """Convert struct to LangChain document format."""
+        metadata = self.extra_info or {}
+        return LCDocument(page_content=self.text, metadata=metadata)
+
+    @classmethod
+    def from_langchain_format(cls, doc: LCDocument) -> "Document":
+        """Convert struct from LangChain document format."""
+        return cls(text=doc.page_content, extra_info=doc.metadata)

application/parser/schema/schema.py

@@ -0,0 +1,64 @@
+"""Base schema for data structures."""
+from abc import abstractmethod
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from dataclasses_json import DataClassJsonMixin
+
+
+@dataclass
+class BaseDocument(DataClassJsonMixin):
+    """Base document.
+
+    Generic abstract interfaces that captures both index structs
+    as well as documents.
+
+    """
+
+    # TODO: consolidate fields from Document/IndexStruct into base class
+    text: Optional[str] = None
+    doc_id: Optional[str] = None
+    embedding: Optional[List[float]] = None
+
+    # extra fields
+    extra_info: Optional[Dict[str, Any]] = None
+
+    @classmethod
+    @abstractmethod
+    def get_type(cls) -> str:
+        """Get Document type."""
+
+    def get_text(self) -> str:
+        """Get text."""
+        if self.text is None:
+            raise ValueError("text field not set.")
+        return self.text
+
+    def get_doc_id(self) -> str:
+        """Get doc_id."""
+        if self.doc_id is None:
+            raise ValueError("doc_id not set.")
+        return self.doc_id
+
+    @property
+    def is_doc_id_none(self) -> bool:
+        """Check if doc_id is None."""
+        return self.doc_id is None
+
+    def get_embedding(self) -> List[float]:
+        """Get embedding.
+
+        Errors if embedding is None.
+
+        """
+        if self.embedding is None:
+            raise ValueError("embedding not set.")
+        return self.embedding
+
+    @property
+    def extra_info_str(self) -> Optional[str]:
+        """Extra info string."""
+        if self.extra_info is None:
+            return None
+
+        return "\n".join([f"{k}: {str(v)}" for k, v in self.extra_info.items()])