chore: import upstream snapshot with attribution

application/vectorstore/base.py

@@ -0,0 +1,312 @@
+import logging
+import os
+from abc import ABC, abstractmethod
+
+import requests
+from langchain_openai import OpenAIEmbeddings
+
+from application.core.settings import settings
+from application.utils import get_encoding
+
+
+class RemoteEmbeddings:
+    """
+    Wrapper for remote embeddings API (OpenAI-compatible).
+    Used when EMBEDDINGS_BASE_URL is configured.
+    Sends requests to {base_url}/v1/embeddings in OpenAI format.
+    """
+
+    def __init__(self, api_url: str, model_name: str, api_key: str = None):
+        self.api_url = api_url.rstrip("/")
+        self.model_name = model_name
+        self.headers = {"Content-Type": "application/json"}
+        if api_key:
+            self.headers["Authorization"] = f"Bearer {api_key}"
+        self.dimension = 768
+
+    def _truncate_inputs(self, inputs):
+        """Clip each input to ``EMBEDDINGS_MAX_INPUT_TOKENS`` tokens.
+
+        The remote server (e.g. llama.cpp) hard-rejects any single input
+        larger than its physical batch size with a 500. When the setting is
+        configured, each input is truncated to that many tokens before the
+        request and the overflow is dropped (lossy by design). Token counts
+        use the shared tiktoken encoding, which differs from the server's
+        tokenizer, so set the limit with headroom under the server's true
+        limit to absorb tokenizer skew.
+
+        Args:
+            inputs: A single string or a list of strings to embed.
+
+        Returns:
+            The inputs with each string clipped to the token limit, or the
+            inputs unchanged when the limit is unset or non-positive.
+        """
+        limit = settings.EMBEDDINGS_MAX_INPUT_TOKENS
+        if not limit or limit <= 0:
+            return inputs
+
+        encoding = get_encoding()
+
+        def clip(text):
+            if not isinstance(text, str):
+                return text
+            tokens = encoding.encode(text)
+            if len(tokens) <= limit:
+                return text
+            logging.warning(
+                "Truncating remote embeddings input from %d to %d tokens (%d dropped)",
+                len(tokens),
+                limit,
+                len(tokens) - limit,
+            )
+            return encoding.decode(tokens[:limit])
+
+        if isinstance(inputs, list):
+            return [clip(text) for text in inputs]
+        return clip(inputs)
+
+    def _embed(self, inputs):
+        """Send embedding request to remote API in OpenAI-compatible format."""
+        inputs = self._truncate_inputs(inputs)
+        payload = {"input": inputs}
+        if self.model_name:
+            payload["model"] = self.model_name
+
+        url = f"{self.api_url}/v1/embeddings"
+        response = requests.post(url, headers=self.headers, json=payload, timeout=180)
+        response.raise_for_status()
+        result = response.json()
+
+        # Handle OpenAI-compatible response format
+        if isinstance(result, dict):
+            if "error" in result:
+                raise ValueError(f"Remote embeddings API error: {result['error']}")
+            if "data" in result:
+                # Sort by index to ensure correct order
+                data = sorted(result["data"], key=lambda x: x.get("index", 0))
+                return [item["embedding"] for item in data]
+            raise ValueError(
+                f"Unexpected response format from remote embeddings API: {result}"
+            )
+        else:
+            raise ValueError(
+                f"Unexpected response format from remote embeddings API: {result}"
+            )
+
+    def embed_query(self, query: str):
+        """Embed a single query string."""
+        embeddings_list = self._embed(query)
+        if (
+            isinstance(embeddings_list, list)
+            and len(embeddings_list) == 1
+            and isinstance(embeddings_list[0], list)
+        ):
+            if self.dimension is None:
+                self.dimension = len(embeddings_list[0])
+            return embeddings_list[0]
+        raise ValueError(
+            f"Unexpected result structure after embedding query: {embeddings_list}"
+        )
+
+    def embed_documents(self, documents: list):
+        """Embed a list of documents."""
+        if not documents:
+            return []
+        embeddings_list = self._embed(documents)
+        if self.dimension is None and embeddings_list:
+            self.dimension = len(embeddings_list[0])
+        return embeddings_list
+
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.embed_query(text)
+        elif isinstance(text, list):
+            return self.embed_documents(text)
+        else:
+            raise ValueError("Input must be a string or a list of strings")
+
+
+def _get_embeddings_wrapper():
+    """Lazy import of EmbeddingsWrapper to avoid loading SentenceTransformer when using remote embeddings."""
+    from application.vectorstore.embeddings_local import EmbeddingsWrapper
+
+    return EmbeddingsWrapper
+
+
+class EmbeddingsSingleton:
+    _instances = {}
+
+    @staticmethod
+    def _remote_instance(embeddings_name, embeddings_key=None):
+        """Return a cached ``RemoteEmbeddings`` for the configured remote API.
+
+        Centralizes the ``EMBEDDINGS_BASE_URL`` dispatch so every caller —
+        including code that calls :meth:`get_instance` directly (GraphRAG,
+        semantic chunking) rather than via
+        :meth:`BaseVectorStore._get_embeddings` — routes to the remote
+        embeddings server instead of attempting a local model download.
+
+        Args:
+            embeddings_name: Model name forwarded to the remote API.
+            embeddings_key: Optional API key; falls back to
+                ``settings.EMBEDDINGS_KEY`` when not provided.
+
+        Returns:
+            RemoteEmbeddings: Shared instance keyed by base URL and model name.
+        """
+        api_key = embeddings_key if embeddings_key is not None else settings.EMBEDDINGS_KEY
+        cache_key = f"remote_{settings.EMBEDDINGS_BASE_URL}_{embeddings_name}"
+        if cache_key not in EmbeddingsSingleton._instances:
+            EmbeddingsSingleton._instances[cache_key] = RemoteEmbeddings(
+                api_url=settings.EMBEDDINGS_BASE_URL,
+                model_name=embeddings_name,
+                api_key=api_key,
+            )
+        return EmbeddingsSingleton._instances[cache_key]
+
+    @staticmethod
+    def get_instance(embeddings_name, *args, **kwargs):
+        if settings.EMBEDDINGS_BASE_URL:
+            return EmbeddingsSingleton._remote_instance(embeddings_name)
+        if embeddings_name not in EmbeddingsSingleton._instances:
+            EmbeddingsSingleton._instances[embeddings_name] = (
+                EmbeddingsSingleton._create_instance(embeddings_name, *args, **kwargs)
+            )
+        return EmbeddingsSingleton._instances[embeddings_name]
+
+    @staticmethod
+    def _create_instance(embeddings_name, *args, **kwargs):
+        if embeddings_name == "openai_text-embedding-ada-002":
+            return OpenAIEmbeddings(*args, **kwargs)
+
+        # Lazy import EmbeddingsWrapper only when needed (avoids loading SentenceTransformer)
+        EmbeddingsWrapper = _get_embeddings_wrapper()
+
+        embeddings_factory = {
+            "huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper(
+                "sentence-transformers/all-mpnet-base-v2"
+            ),
+            "huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper(
+                "sentence-transformers/all-mpnet-base-v2"
+            ),
+            "huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper(
+                "hkunlp/instructor-large"
+            ),
+        }
+
+        if embeddings_name in embeddings_factory:
+            return embeddings_factory[embeddings_name](*args, **kwargs)
+        else:
+            return EmbeddingsWrapper(embeddings_name, *args, **kwargs)
+
+
+class BaseVectorStore(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def search(self, *args, **kwargs):
+        """Search for similar documents/chunks in the vectorstore"""
+        pass
+
+    def keyword_search(self, question, k=10):
+        """Keyword/full-text search.
+
+        Default returns no results so hybrid retrieval degrades to vector-only
+        on stores without keyword support. Override in stores that support it.
+        """
+        return []
+
+    @abstractmethod
+    def add_texts(self, texts, metadatas=None, *args, **kwargs):
+        """Add texts with their embeddings to the vectorstore"""
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        """Delete the entire index/collection"""
+        pass
+
+    def save_local(self, *args, **kwargs):
+        """Save vectorstore to local storage"""
+        pass
+
+    def get_chunks(self, *args, **kwargs):
+        """Get all chunks from the vectorstore"""
+        pass
+
+    def add_chunk(self, text, metadata=None, *args, **kwargs):
+        """Add a single chunk to the vectorstore"""
+        pass
+
+    def delete_chunk(self, chunk_id, *args, **kwargs):
+        """Delete a specific chunk from the vectorstore"""
+        pass
+
+    def delete_chunks_by_source_path(self, path) -> int:
+        """Delete every chunk whose ``metadata.source`` equals ``path``.
+
+        Default implementation iterates ``get_chunks()`` and deletes the
+        matches via ``delete_chunk()`` — works for any store. Override with a
+        single targeted statement where the store supports it. Returns the
+        number of chunks deleted.
+        """
+        deleted = 0
+        for chunk in self.get_chunks() or []:
+            if (chunk.get("metadata") or {}).get("source") == path:
+                if self.delete_chunk(chunk.get("doc_id")):
+                    deleted += 1
+        return deleted
+
+    def is_azure_configured(self):
+        return (
+            settings.OPENAI_API_BASE
+            and settings.OPENAI_API_VERSION
+            and settings.AZURE_DEPLOYMENT_NAME
+        )
+
+    def _get_embeddings(self, embeddings_name, embeddings_key=None):
+        # Check for remote embeddings first
+        if settings.EMBEDDINGS_BASE_URL:
+            logging.info(
+                f"Using remote embeddings API at: {settings.EMBEDDINGS_BASE_URL}"
+            )
+            return EmbeddingsSingleton._remote_instance(embeddings_name, embeddings_key)
+
+        if embeddings_name == "openai_text-embedding-ada-002":
+            if self.is_azure_configured():
+                os.environ["OPENAI_API_TYPE"] = "azure"
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name, model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME
+                )
+            else:
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name, openai_api_key=embeddings_key
+                )
+        elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
+            possible_paths = [
+                "/app/models/all-mpnet-base-v2",  # Docker absolute path
+                "./models/all-mpnet-base-v2",  # Relative path
+            ]
+            local_model_path = None
+            for path in possible_paths:
+                if os.path.exists(path):
+                    local_model_path = path
+                    logging.info(f"Found local model at path: {path}")
+                    break
+                else:
+                    logging.info(f"Path does not exist: {path}")
+            if local_model_path:
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    local_model_path,
+                )
+            else:
+                logging.warning(
+                    f"Local model not found in any of the paths: {possible_paths}. Falling back to HuggingFace download."
+                )
+                embedding_instance = EmbeddingsSingleton.get_instance(
+                    embeddings_name,
+                )
+        else:
+            embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)
+        return embedding_instance

application/vectorstore/document_class.py

@@ -0,0 +1,8 @@
+class Document(str):
+    """Class for storing a piece of text and associated metadata."""
+
+    def __new__(cls, page_content: str, metadata: dict):
+        instance = super().__new__(cls, page_content)
+        instance.page_content = page_content
+        instance.metadata = metadata
+        return instance

application/vectorstore/elasticsearch.py

@@ -0,0 +1,207 @@
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+from application.vectorstore.document_class import Document
+
+
+class ElasticsearchStore(BaseVectorStore):
+    _es_connection = None  # Class attribute to hold the Elasticsearch connection
+
+    def __init__(self, source_id, embeddings_key, index_name=settings.ELASTIC_INDEX):
+        super().__init__()
+        self.source_id = source_id.replace("application/indexes/", "").rstrip("/")
+        self.embeddings_key = embeddings_key
+        self.index_name = index_name
+        
+        if ElasticsearchStore._es_connection is None:
+            connection_params = {}
+            if settings.ELASTIC_URL:
+                connection_params["hosts"] = [settings.ELASTIC_URL]
+                connection_params["http_auth"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD)
+            elif settings.ELASTIC_CLOUD_ID:
+                connection_params["cloud_id"] = settings.ELASTIC_CLOUD_ID
+                connection_params["basic_auth"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD)
+            else:
+                raise ValueError("Please provide either elasticsearch_url or cloud_id.")
+
+            import elasticsearch
+            ElasticsearchStore._es_connection = elasticsearch.Elasticsearch(**connection_params)
+            
+        self.docsearch = ElasticsearchStore._es_connection
+
+    def connect_to_elasticsearch(
+        *,
+        es_url = None,
+        cloud_id = None,
+        api_key = None,
+        username = None,
+        password = None,
+    ):
+        try:
+            import elasticsearch
+        except ImportError:
+            raise ImportError(
+                "Could not import elasticsearch python package. "
+                "Please install it with `pip install elasticsearch`."
+            )
+
+        if es_url and cloud_id:
+            raise ValueError(
+                "Both es_url and cloud_id are defined. Please provide only one."
+            )
+
+        connection_params = {}
+
+        if es_url:
+            connection_params["hosts"] = [es_url]
+        elif cloud_id:
+            connection_params["cloud_id"] = cloud_id
+        else:
+            raise ValueError("Please provide either elasticsearch_url or cloud_id.")
+
+        if api_key:
+            connection_params["api_key"] = api_key
+        elif username and password:
+            connection_params["basic_auth"] = (username, password)
+
+        es_client = elasticsearch.Elasticsearch(
+            **connection_params,
+        )
+        try:
+            es_client.info()
+        except Exception as e:
+            raise e
+
+        return es_client
+
+    def search(self, question, k=2, index_name=settings.ELASTIC_INDEX, *args, **kwargs):
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+        vector = embeddings.embed_query(question)
+        knn = {
+            "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
+            "field": "vector",
+            "k": k,
+            "num_candidates": 100,
+            "query_vector": vector,
+        }
+        full_query = {
+            "knn": knn,
+            "query": {
+                "bool": {
+                    "must": [
+                        {
+                            "match": {
+                                "text": {
+                                    "query": question,
+                                }
+                            }
+                        }
+                    ],
+                    "filter": [{"match": {"metadata.source_id.keyword": self.source_id}}],
+                }
+            },
+            "rank": {"rrf": {}},
+        }
+        resp = self.docsearch.search(index=self.index_name, query=full_query['query'], size=k, knn=full_query['knn'])
+        # create Documents objects from the results page_content ['_source']['text'], metadata ['_source']['metadata']
+        doc_list = []
+        for hit in resp['hits']['hits']:
+            
+            doc_list.append(Document(page_content = hit['_source']['text'], metadata = hit['_source']['metadata']))
+        return doc_list
+
+    def _create_index_if_not_exists(
+            self, index_name, dims_length
+        ):
+
+        if self._es_connection.indices.exists(index=index_name):
+            print(f"Index {index_name} already exists.")
+
+        else:
+
+            indexSettings = self.index(
+                dims_length=dims_length,
+            )
+            self._es_connection.indices.create(index=index_name, **indexSettings)
+
+    def index(
+            self,
+            dims_length,
+        ):
+        return {
+            "mappings": {
+                "properties": {
+                    "vector": {
+                        "type": "dense_vector",
+                        "dims": dims_length,
+                        "index": True,
+                        "similarity": "cosine",
+                    },
+                }
+            }
+        }
+
+    def add_texts(
+        self,
+        texts,
+        metadatas = None,
+        ids = None,
+        refresh_indices = True,
+        create_index_if_not_exists = True,
+        bulk_kwargs = None,
+        **kwargs,
+        ):
+        
+        bulk_kwargs = bulk_kwargs or {}
+        import uuid
+        embeddings = []
+        ids = ids or [str(uuid.uuid4()) for _ in texts]
+        requests = []
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+
+        vectors = embeddings.embed_documents(list(texts))
+
+        dims_length = len(vectors[0])
+
+        if create_index_if_not_exists:
+            self._create_index_if_not_exists(
+                index_name=self.index_name, dims_length=dims_length
+            )
+
+        for i, (text, vector) in enumerate(zip(texts, vectors)):
+            metadata = metadatas[i] if metadatas else {}
+
+            requests.append(
+                {
+                    "_op_type": "index",
+                    "_index": self.index_name,
+                    "text": text,
+                    "vector": vector,
+                    "metadata": metadata,
+                    "_id": ids[i],
+                }
+            )
+
+
+        if len(requests) > 0:
+            from elasticsearch.helpers import BulkIndexError, bulk
+            try:
+                success, failed = bulk(
+                    self._es_connection,
+                    requests,
+                    stats_only=True,
+                    refresh=refresh_indices,
+                    **bulk_kwargs,
+                )
+                return ids
+            except BulkIndexError as e:
+                print(f"Error adding texts: {e}")
+                firstError = e.errors[0].get("index", {}).get("error", {})
+                print(f"First error reason: {firstError.get('reason')}")
+                raise e
+
+        else:
+            return []
+
+    def delete_index(self):
+        self._es_connection.delete_by_query(index=self.index_name, query={"match": {
+                                      "metadata.source_id.keyword": self.source_id}},)

application/vectorstore/embeddings_local.py

@@ -0,0 +1,48 @@
+"""
+Local embeddings using SentenceTransformer.
+This module is only imported when EMBEDDINGS_BASE_URL is not set,
+to avoid loading SentenceTransformer into memory when using remote embeddings.
+"""
+
+import logging
+
+from sentence_transformers import SentenceTransformer
+
+
+class EmbeddingsWrapper:
+    def __init__(self, model_name, *args, **kwargs):
+        logging.info(f"Initializing EmbeddingsWrapper with model: {model_name}")
+        try:
+            kwargs.setdefault("trust_remote_code", True)
+            self.model = SentenceTransformer(
+                model_name,
+                config_kwargs={"allow_dangerous_deserialization": True},
+                *args,
+                **kwargs,
+            )
+            if self.model is None or self.model._first_module() is None:
+                raise ValueError(
+                    f"SentenceTransformer model failed to load properly for: {model_name}"
+                )
+            self.dimension = self.model.get_sentence_embedding_dimension()
+            logging.info(f"Successfully loaded model with dimension: {self.dimension}")
+        except Exception as e:
+            logging.error(
+                f"Failed to initialize SentenceTransformer with model {model_name}: {str(e)}",
+                exc_info=True,
+            )
+            raise
+
+    def embed_query(self, query: str):
+        return self.model.encode(query).tolist()
+
+    def embed_documents(self, documents: list):
+        return self.model.encode(documents).tolist()
+
+    def __call__(self, text):
+        if isinstance(text, str):
+            return self.embed_query(text)
+        elif isinstance(text, list):
+            return self.embed_documents(text)
+        else:
+            raise ValueError("Input must be a string or a list of strings")

application/vectorstore/faiss.py

@@ -0,0 +1,174 @@
+import os
+import tempfile
+import io
+
+from langchain_community.vectorstores import FAISS
+
+from application.core.settings import settings
+from application.parser.schema.base import Document
+from application.vectorstore.base import BaseVectorStore
+from application.storage.storage_creator import StorageCreator
+
+
+def get_vectorstore(path: str) -> str:
+    """Build a safe local path for a FAISS index.
+
+    Args:
+        path: Source identifier provided by the caller.
+
+    Returns:
+        The validated vectorstore path rooted under ``indexes``.
+
+    Raises:
+        ValueError: If ``path`` escapes the ``indexes`` directory.
+    """
+    base_dir = "indexes"
+    if not path:
+        return base_dir
+
+    normalized = str(path).strip()
+    if "\\" in normalized:
+        raise ValueError("Invalid source_id path")
+
+    candidate = os.path.normpath(os.path.join(base_dir, normalized))
+    base_abs = os.path.abspath(base_dir)
+    candidate_abs = os.path.abspath(candidate)
+
+    if not candidate_abs.startswith(base_abs + os.sep) and candidate_abs != base_abs:
+        raise ValueError("Invalid source_id path")
+
+    return candidate
+
+
+class FaissStore(BaseVectorStore):
+    def __init__(self, source_id: str, embeddings_key: str, docs_init=None):
+        super().__init__()
+        self.source_id = source_id
+        self.path = get_vectorstore(source_id)
+        self.embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
+        self.storage = StorageCreator.get_storage()
+
+        try:
+            if docs_init:
+                self.docsearch = FAISS.from_documents(docs_init, self.embeddings)
+            else:
+                with tempfile.TemporaryDirectory() as temp_dir:
+                    faiss_path = f"{self.path}/index.faiss"
+                    pkl_path = f"{self.path}/index.pkl"
+
+                    if not self.storage.file_exists(
+                        faiss_path
+                    ) or not self.storage.file_exists(pkl_path):
+                        raise FileNotFoundError(
+                            f"Index files not found in storage at {self.path}"
+                        )
+
+                    faiss_file = self.storage.get_file(faiss_path)
+                    pkl_file = self.storage.get_file(pkl_path)
+
+                    local_faiss_path = os.path.join(temp_dir, "index.faiss")
+                    local_pkl_path = os.path.join(temp_dir, "index.pkl")
+
+                    with open(local_faiss_path, "wb") as f:
+                        f.write(faiss_file.read())
+
+                    with open(local_pkl_path, "wb") as f:
+                        f.write(pkl_file.read())
+
+                    self.docsearch = FAISS.load_local(
+                        temp_dir, self.embeddings, allow_dangerous_deserialization=True
+                    )
+        except Exception as e:
+            raise Exception(f"Error loading FAISS index: {str(e)}")
+
+        self.assert_embedding_dimensions(self.embeddings)
+
+    def search(self, *args, **kwargs):
+        # FAISS has no relevance-threshold knob; drop it so the per-source
+        # score_threshold is safely ignored rather than crashing the forward.
+        kwargs.pop("score_threshold", None)
+        return self.docsearch.similarity_search(*args, **kwargs)
+
+    def add_texts(self, *args, **kwargs):
+        return self.docsearch.add_texts(*args, **kwargs)
+
+    def _save_to_storage(self):
+        """
+        Save the FAISS index to storage using temporary directory pattern.
+        Works consistently for both local and S3 storage.
+        """
+        with tempfile.TemporaryDirectory() as temp_dir:
+            self.docsearch.save_local(temp_dir)
+
+            faiss_path = os.path.join(temp_dir, "index.faiss")
+            pkl_path = os.path.join(temp_dir, "index.pkl")
+
+            with open(faiss_path, "rb") as f_faiss:
+                faiss_data = f_faiss.read()
+
+            with open(pkl_path, "rb") as f_pkl:
+                pkl_data = f_pkl.read()
+
+            storage_path = get_vectorstore(self.source_id)
+            self.storage.save_file(io.BytesIO(faiss_data), f"{storage_path}/index.faiss")
+            self.storage.save_file(io.BytesIO(pkl_data), f"{storage_path}/index.pkl")
+
+        return True
+
+    def save_local(self, path=None):
+        if path:
+            os.makedirs(path, exist_ok=True)
+            self.docsearch.save_local(path)
+
+        self._save_to_storage()
+
+        return True
+
+    def delete_index(self, *args, **kwargs):
+        return self.docsearch.delete(*args, **kwargs)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used."""
+        if (
+            settings.EMBEDDINGS_NAME
+            == "huggingface_sentence-transformers/all-mpnet-base-v2"
+        ):
+            word_embedding_dimension = getattr(embeddings, "dimension", None)
+            if word_embedding_dimension is None:
+                raise AttributeError(
+                    "'dimension' attribute not found in embeddings instance."
+                )
+
+            docsearch_index_dimension = self.docsearch.index.d
+            if word_embedding_dimension != docsearch_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})"
+                )
+
+    def get_chunks(self):
+        chunks = []
+        if self.docsearch:
+            for doc_id, doc in self.docsearch.docstore._dict.items():
+                chunk_data = {
+                    "doc_id": doc_id,
+                    "text": doc.page_content,
+                    "metadata": doc.metadata,
+                }
+                chunks.append(chunk_data)
+        return chunks
+
+    def add_chunk(self, text, metadata=None):
+        """Add a new chunk and save to storage."""
+        metadata = metadata or {}
+        doc = Document(text=text, extra_info=metadata).to_langchain_format()
+        doc_id = self.docsearch.add_documents([doc])
+        self._save_to_storage()
+        return doc_id
+
+
+
+    def delete_chunk(self, chunk_id):
+        """Delete a chunk and save to storage."""
+        self.delete_index([chunk_id])
+        self._save_to_storage()
+        return True

application/vectorstore/lancedb.py

@@ -0,0 +1,119 @@
+from typing import List, Optional
+import importlib
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+
+class LanceDBVectorStore(BaseVectorStore):
+    """Class for LanceDB Vector Store integration."""
+
+    def __init__(self, path: str = settings.LANCEDB_PATH,
+                 table_name_prefix: str = settings.LANCEDB_TABLE_NAME,
+                 source_id: str = None,
+                 embeddings_key: str = "embeddings"):
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self.path = path
+        self.table_name = f"{table_name_prefix}_{source_id}" if source_id else table_name_prefix
+        self.embeddings_key = embeddings_key
+        self._lance_db = None
+        self.docsearch = None
+        self._pa = None  # PyArrow (pa) will be lazy loaded
+
+    @property
+    def pa(self):
+        """Lazy load pyarrow module."""
+        if self._pa is None:
+            self._pa = importlib.import_module("pyarrow")
+        return self._pa
+
+    @property
+    def lancedb(self):
+        """Lazy load lancedb module."""
+        if not hasattr(self, "_lancedb_module"):
+            self._lancedb_module = importlib.import_module("lancedb")
+        return self._lancedb_module
+
+    @property
+    def lance_db(self):
+        """Lazy load the LanceDB connection."""
+        if self._lance_db is None:
+            self._lance_db = self.lancedb.connect(self.path)
+        return self._lance_db
+
+    @property
+    def table(self):
+        """Lazy load the LanceDB table."""
+        if self.docsearch is None:
+            if self.table_name in self.lance_db.table_names():
+                self.docsearch = self.lance_db.open_table(self.table_name)
+            else:
+                self.docsearch = None
+        return self.docsearch
+
+    def ensure_table_exists(self):
+        """Ensure the table exists before performing operations."""
+        if self.table is None:
+            embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+            schema = self.pa.schema([
+                self.pa.field("vector", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),
+                self.pa.field("text", self.pa.string()),
+                self.pa.field("metadata", self.pa.struct([
+                    self.pa.field("key", self.pa.string()),
+                    self.pa.field("value", self.pa.string())
+                ]))
+            ])
+            self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):
+        """Add texts with metadata and their embeddings to the LanceDB table."""
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)
+        vectors = []
+        for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):
+            if source_id:
+                metadata["source_id"] = source_id
+            metadata_struct = [{"key": k, "value": str(v)} for k, v in metadata.items()]
+            vectors.append({
+                "vector": embedding,
+                "text": text,
+                "metadata": metadata_struct
+            })
+        self.ensure_table_exists()
+        self.docsearch.add(vectors)
+
+    def search(self, query: str, k: int = 2, *args, **kwargs):
+        """Search LanceDB for the top k most similar vectors."""
+        self.ensure_table_exists()
+        query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)
+        results = self.docsearch.search(query_embedding).limit(k).to_list()
+        return [(result["_distance"], result["text"], result["metadata"]) for result in results]
+
+    def delete_index(self):
+        """Delete the entire LanceDB index (table)."""
+        if self.table:
+            self.lance_db.drop_table(self.table_name)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Ensure that embedding dimensions match the table index dimensions."""
+        word_embedding_dimension = embeddings.dimension
+        if self.table:
+            table_index_dimension = len(self.docsearch.schema["vector"].type.value_type)
+            if word_embedding_dimension != table_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) "
+                    f"!= table index dimension ({table_index_dimension})"
+                )
+
+    def filter_documents(self, filter_condition: dict) -> List[dict]:
+        """Filter documents based on certain conditions."""
+        self.ensure_table_exists()
+
+        # Ensure source_id exists in the filter condition
+        if 'source_id' not in filter_condition:
+            raise ValueError("filter_condition must contain 'source_id'")
+
+        source_id = filter_condition["source_id"]
+
+        # Use LanceDB's native filtering if supported, otherwise filter manually
+        filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get("source_id") == source_id).to_list()
+
+        return filtered_data

application/vectorstore/milvus.py

@@ -0,0 +1,41 @@
+from typing import List, Optional
+from uuid import uuid4
+
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+
+
+class MilvusStore(BaseVectorStore):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
+        super().__init__()
+        from langchain_milvus import Milvus
+
+        connection_args = {
+            "uri": settings.MILVUS_URI,
+            "token": settings.MILVUS_TOKEN,
+        }
+        self._docsearch = Milvus(
+            embedding_function=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key),
+            collection_name=settings.MILVUS_COLLECTION_NAME,
+            connection_args=connection_args,
+        )
+        self._source_id = source_id
+
+    def search(self, question, k=2, *args, **kwargs):
+        # Drop the per-source score_threshold (unsupported here) so it is safely
+        # ignored instead of being forwarded into the langchain call.
+        kwargs.pop("score_threshold", None)
+        expr = f"source_id == '{self._source_id}'"
+        return self._docsearch.similarity_search(query=question, k=k, expr=expr, *args, **kwargs)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
+        ids = [str(uuid4()) for _ in range(len(texts))]
+
+        return self._docsearch.add_texts(texts=texts, metadatas=metadatas, ids=ids, *args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        pass

application/vectorstore/mongodb.py

@@ -0,0 +1,206 @@
+import logging
+from functools import cached_property
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+from application.vectorstore.document_class import Document
+
+
+def _lazy_import_pymongo():
+    """Lazy import of pymongo so installations that don't use the MongoDB vectorstore don't need it."""
+    try:
+        import pymongo
+    except ImportError as exc:
+        raise ImportError(
+            "Could not import pymongo python package. "
+            "Please install it with `pip install pymongo`."
+        ) from exc
+    return pymongo
+
+
+class MongoDBVectorStore(BaseVectorStore):
+    def __init__(
+        self,
+        source_id: str = "",
+        embeddings_key: str = "embeddings",
+        collection: str = "documents",
+        index_name: str = "vector_search_index",
+        text_key: str = "text",
+        embedding_key: str = "embedding",
+        database: str = "docsgpt",
+    ):
+        self._index_name = index_name
+        self._text_key = text_key
+        self._embedding_key = embedding_key
+        self._embeddings_key = embeddings_key
+        self._mongo_uri = settings.MONGO_URI
+        self._database_name = database
+        self._collection_name = collection
+        self._source_id = source_id.replace("application/indexes/", "").rstrip("/")
+        self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
+
+    @cached_property
+    def _client(self):
+        pymongo = _lazy_import_pymongo()
+        return pymongo.MongoClient(self._mongo_uri)
+
+    @cached_property
+    def _database(self):
+        return self._client[self._database_name]
+
+    @cached_property
+    def _collection(self):
+        return self._database[self._collection_name]
+
+    def search(self, question, k=2, *args, score_threshold=None, **kwargs):
+        """Search via Atlas ``$vectorSearch``.
+
+        Args:
+            question: The query string.
+            k: Maximum number of results.
+            score_threshold: Optional ``vectorSearchScore`` floor in ``[0, 1]``;
+                results scoring below it are dropped.
+        """
+        query_vector = self._embedding.embed_query(question)
+
+        pipeline = [
+            {
+                "$vectorSearch": {
+                    "queryVector": query_vector,
+                    "path": self._embedding_key,
+                    "limit": k,
+                    "numCandidates": k * 10,
+                    "index": self._index_name,
+                    "filter": {"source_id": {"$eq": self._source_id}},
+                }
+            }
+        ]
+        if score_threshold is not None:
+            pipeline.append({"$addFields": {"_score": {"$meta": "vectorSearchScore"}}})
+            pipeline.append({"$match": {"_score": {"$gte": float(score_threshold)}}})
+
+        cursor = self._collection.aggregate(pipeline)
+
+        results = []
+        for doc in cursor:
+            text = doc[self._text_key]
+            doc.pop("_id")
+            doc.pop(self._text_key)
+            doc.pop(self._embedding_key)
+            doc.pop("_score", None)
+            metadata = doc
+            results.append(Document(text, metadata))
+        return results
+
+    def _insert_texts(self, texts, metadatas):
+        if not texts:
+            return []
+        embeddings = self._embedding.embed_documents(texts)
+
+        to_insert = [
+            {self._text_key: t, self._embedding_key: embedding, **m}
+            for t, m, embedding in zip(texts, metadatas, embeddings)
+        ]
+
+        insert_result = self._collection.insert_many(to_insert)
+        return insert_result.inserted_ids
+
+    def add_texts(
+        self,
+        texts,
+        metadatas=None,
+        ids=None,
+        refresh_indices=True,
+        create_index_if_not_exists=True,
+        bulk_kwargs=None,
+        **kwargs,
+    ):
+
+        # dims = self._embedding.client[1].word_embedding_dimension
+        # # check if index exists
+        # if create_index_if_not_exists:
+        #     # check if index exists
+        #     info = self._collection.index_information()
+        #     if self._index_name not in info:
+        #         index_mongo = {
+        #         "fields": [{
+        #             "type": "vector",
+        #             "path": self._embedding_key,
+        #             "numDimensions": dims,
+        #             "similarity": "cosine",
+        #         },
+        #         {
+        #             "type": "filter",
+        #             "path": "store"
+        #         }]
+        #         }
+        #         self._collection.create_index(self._index_name, index_mongo)
+
+        batch_size = 100
+        _metadatas = metadatas or ({} for _ in texts)
+        texts_batch = []
+        metadatas_batch = []
+        result_ids = []
+        for i, (text, metadata) in enumerate(zip(texts, _metadatas)):
+            texts_batch.append(text)
+            metadatas_batch.append(metadata)
+            if (i + 1) % batch_size == 0:
+                result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
+                texts_batch = []
+                metadatas_batch = []
+        if texts_batch:
+            result_ids.extend(self._insert_texts(texts_batch, metadatas_batch))
+        return result_ids
+
+    def delete_index(self, *args, **kwargs):
+        self._collection.delete_many({"source_id": self._source_id})
+
+    def get_chunks(self):
+        try:
+            chunks = []
+            cursor = self._collection.find({"source_id": self._source_id})
+            for doc in cursor:
+                doc_id = str(doc.get("_id"))
+                text = doc.get(self._text_key)
+                metadata = {
+                    k: v
+                    for k, v in doc.items()
+                    if k
+                    not in ["_id", self._text_key, self._embedding_key, "source_id"]
+                }
+
+                if text:
+                    chunks.append(
+                        {"doc_id": doc_id, "text": text, "metadata": metadata}
+                    )
+
+            return chunks
+        except Exception as e:
+            logging.error(f"Error getting chunks: {e}", exc_info=True)
+            return []
+
+    def add_chunk(self, text, metadata=None):
+        metadata = metadata or {}
+        embeddings = self._embedding.embed_documents([text])
+        if not embeddings:
+            raise ValueError("Could not generate embedding for chunk")
+
+        chunk_data = {
+            self._text_key: text,
+            self._embedding_key: embeddings[0],
+            "source_id": self._source_id,
+            **metadata,
+        }
+        result = self._collection.insert_one(chunk_data)
+        return str(result.inserted_id)
+
+    def delete_chunk(self, chunk_id):
+        try:
+            from bson.objectid import ObjectId
+
+            object_id = ObjectId(chunk_id)
+            result = self._collection.delete_one({"_id": object_id})
+            return result.deleted_count > 0
+        except Exception as e:
+            logging.error(f"Error deleting chunk: {e}", exc_info=True)
+            return False

application/vectorstore/pgvector.py

@@ -0,0 +1,406 @@
+import logging
+from typing import List, Optional, Any, Dict
+
+from psycopg.types.json import Jsonb
+
+from application.core.settings import settings
+from application.vectorstore.base import BaseVectorStore
+from application.vectorstore.document_class import Document
+
+
+class PGVectorStore(BaseVectorStore):
+    def __init__(
+        self,
+        source_id: str = "",
+        embeddings_key: str = "embeddings",
+        table_name: str = "documents",
+        decoded_token: Optional[str] = None,
+        vector_column: str = "embedding",
+        text_column: str = "text",
+        metadata_column: str = "metadata",
+        connection_string: str = None,
+    ):
+        super().__init__()
+        # Store the source_id for use in add_chunk
+        self._source_id = str(source_id).replace("application/indexes/", "").rstrip("/")
+        self._embeddings_key = embeddings_key
+        self._table_name = table_name
+        self._vector_column = vector_column
+        self._text_column = text_column
+        self._metadata_column = metadata_column
+        self._embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
+        
+        # Use provided connection string or fall back to settings.
+        # If PGVECTOR_CONNECTION_STRING is not set but POSTGRES_URI is,
+        # reuse the same cluster — normalize from SQLAlchemy dialect to libpq form.
+        self._connection_string = connection_string or getattr(settings, 'PGVECTOR_CONNECTION_STRING', None)
+
+        if not self._connection_string and getattr(settings, 'POSTGRES_URI', None):
+            from application.core.db_uri import normalize_pgvector_connection_string
+            self._connection_string = normalize_pgvector_connection_string(settings.POSTGRES_URI)
+
+        if not self._connection_string:
+            raise ValueError(
+                "PostgreSQL connection string is required. "
+                "Set PGVECTOR_CONNECTION_STRING or POSTGRES_URI in settings, "
+                "or pass connection_string parameter."
+            )
+
+        try:
+            import psycopg
+            from pgvector.psycopg import register_vector
+        except ImportError:
+            raise ImportError(
+                "Could not import required packages. "
+                "Please install with `pip install 'psycopg[binary,pool]' pgvector`."
+            )
+
+        self._psycopg = psycopg
+        self._register_vector = register_vector
+        self._connection = None
+        self._ensure_table_exists()
+
+    def _get_connection(self):
+        """Get or create database connection"""
+        if self._connection is None or self._connection.closed:
+            self._connection = self._psycopg.connect(self._connection_string)
+            # Register pgvector types
+            self._register_vector(self._connection)
+        return self._connection
+
+    def _ensure_table_exists(self):
+        """Create table and enable pgvector extension if they don't exist"""
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        
+        try:
+            # Enable pgvector extension
+            cursor.execute("CREATE EXTENSION IF NOT EXISTS vector;")
+            
+            embedding_dim = getattr(self._embedding, 'dimension', 768)
+            
+            # Create table with vector column
+            create_table_query = f"""
+            CREATE TABLE IF NOT EXISTS {self._table_name} (
+                id SERIAL PRIMARY KEY,
+                {self._text_column} TEXT NOT NULL,
+                {self._vector_column} vector({embedding_dim}),
+                {self._metadata_column} JSONB,
+                source_id TEXT NOT NULL,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            );
+            """
+            cursor.execute(create_table_query)
+            
+            # Create index for vector similarity search
+            index_query = f"""
+            CREATE INDEX IF NOT EXISTS {self._table_name}_{self._vector_column}_idx 
+            ON {self._table_name} USING ivfflat ({self._vector_column} vector_cosine_ops)
+            WITH (lists = 100);
+            """
+            cursor.execute(index_query)
+            
+            # Create index for source_id filtering
+            source_index_query = f"""
+            CREATE INDEX IF NOT EXISTS {self._table_name}_source_id_idx
+            ON {self._table_name} (source_id);
+            """
+            cursor.execute(source_index_query)
+
+            # Functional GIN index backing keyword_search full-text queries.
+            fts_index_query = f"""
+            CREATE INDEX IF NOT EXISTS {self._table_name}_text_fts_idx
+            ON {self._table_name} USING gin(to_tsvector('english', {self._text_column}));
+            """
+            cursor.execute(fts_index_query)
+
+            conn.commit()
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Error creating table: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    def search(
+        self,
+        question: str,
+        k: int = 2,
+        *args,
+        score_threshold: float = None,
+        **kwargs,
+    ) -> List[Document]:
+        """Search for similar documents using vector similarity.
+
+        Args:
+            question: The query string.
+            k: Maximum number of results.
+            score_threshold: Optional cosine-similarity floor in ``[0, 1]``.
+                Cosine distance = ``1 - similarity``; rows with similarity below
+                the threshold (distance above ``1 - threshold``) are dropped.
+        """
+        query_vector = self._embedding.embed_query(question)
+
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        try:
+            # Use cosine distance for similarity search with proper vector formatting
+            search_query = f"""
+            SELECT {self._text_column}, {self._metadata_column},
+                   ({self._vector_column} <=> %s::vector) as distance
+            FROM {self._table_name}
+            WHERE source_id = %s
+            ORDER BY {self._vector_column} <=> %s::vector
+            LIMIT %s;
+            """
+
+            cursor.execute(search_query, (query_vector, self._source_id, query_vector, k))
+            results = cursor.fetchall()
+
+            max_distance = None
+            if score_threshold is not None:
+                max_distance = 1.0 - float(score_threshold)
+
+            documents = []
+            for text, metadata, distance in results:
+                if max_distance is not None and distance is not None and distance > max_distance:
+                    continue
+                metadata = metadata or {}
+                documents.append(Document(page_content=text, metadata=metadata))
+
+            return documents
+            
+        except Exception as e:
+            logging.error(f"Error searching documents: {e}", exc_info=True)
+            return []
+        finally:
+            cursor.close()
+
+    def keyword_search(self, question: str, k: int = 10) -> List[Document]:
+        """Full-text keyword search using Postgres ``websearch_to_tsquery``.
+
+        Returns the same ``Document`` shape as :meth:`search`. The question is
+        bound as a query parameter (never interpolated) to prevent injection.
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        try:
+            keyword_query = f"""
+            SELECT {self._text_column}, {self._metadata_column},
+                   ts_rank(
+                       to_tsvector('english', {self._text_column}),
+                       websearch_to_tsquery('english', %s)
+                   ) AS rank
+            FROM {self._table_name}
+            WHERE source_id = %s
+              AND to_tsvector('english', {self._text_column})
+                  @@ websearch_to_tsquery('english', %s)
+            ORDER BY rank DESC
+            LIMIT %s;
+            """
+
+            cursor.execute(
+                keyword_query, (question, self._source_id, question, k)
+            )
+            results = cursor.fetchall()
+
+            documents = []
+            for text, metadata, _rank in results:
+                metadata = metadata or {}
+                documents.append(Document(page_content=text, metadata=metadata))
+
+            return documents
+
+        except Exception as e:
+            logging.error(f"Error in keyword search: {e}", exc_info=True)
+            return []
+        finally:
+            cursor.close()
+
+    def add_texts(
+        self,
+        texts: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+        *args,
+        **kwargs,
+    ) -> List[str]:
+        """Add texts with their embeddings to the vector store"""
+        if not texts:
+            return []
+
+        embeddings = self._embedding.embed_documents(texts)
+        metadatas = metadatas or [{}] * len(texts)
+        
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        
+        try:
+            insert_query = f"""
+            INSERT INTO {self._table_name} ({self._text_column}, {self._vector_column}, {self._metadata_column}, source_id)
+            VALUES (%s, %s, %s, %s)
+            RETURNING id;
+            """
+            
+            inserted_ids = []
+            for text, embedding, metadata in zip(texts, embeddings, metadatas):
+                cursor.execute(
+                    insert_query,
+                    (text, embedding, Jsonb(metadata), self._source_id)
+                )
+                inserted_id = cursor.fetchone()[0]
+                inserted_ids.append(str(inserted_id))
+            
+            conn.commit()
+            return inserted_ids
+            
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Error adding texts: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    def delete_index(self, *args, **kwargs):
+        """Delete all documents for this source_id"""
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        
+        try:
+            delete_query = f"DELETE FROM {self._table_name} WHERE source_id = %s;"
+            cursor.execute(delete_query, (self._source_id,))
+            conn.commit()
+            
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Error deleting index: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    def save_local(self, *args, **kwargs):
+        """No-op for PostgreSQL - data is already persisted"""
+        pass
+
+    def get_chunks(self) -> List[Dict[str, Any]]:
+        """Get all chunks for this source_id"""
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        
+        try:
+            select_query = f"""
+            SELECT id, {self._text_column}, {self._metadata_column}
+            FROM {self._table_name}
+            WHERE source_id = %s;
+            """
+            cursor.execute(select_query, (self._source_id,))
+            results = cursor.fetchall()
+            
+            chunks = []
+            for doc_id, text, metadata in results:
+                chunks.append({
+                    "doc_id": str(doc_id),
+                    "text": text,
+                    "metadata": metadata or {}
+                })
+            
+            return chunks
+            
+        except Exception as e:
+            logging.error(f"Error getting chunks: {e}")
+            return []
+        finally:
+            cursor.close()
+
+    def add_chunk(self, text: str, metadata: Optional[Dict[str, Any]] = None) -> str:
+        """Add a single chunk to the vector store"""
+        metadata = metadata or {}
+
+        final_metadata = metadata.copy()
+
+        final_metadata["source_id"] = self._source_id
+
+        embeddings = self._embedding.embed_documents([text])
+
+        if not embeddings:
+            raise ValueError("Could not generate embedding for chunk")
+        
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        
+        try:
+            insert_query = f"""
+            INSERT INTO {self._table_name} ({self._text_column}, {self._vector_column}, {self._metadata_column}, source_id)
+            VALUES (%s, %s, %s, %s)
+            RETURNING id;
+            """
+            
+            cursor.execute(
+                insert_query,
+                (text, embeddings[0], Jsonb(final_metadata), self._source_id)
+            )
+            inserted_id = cursor.fetchone()[0]
+            conn.commit()
+            
+            return str(inserted_id)
+            
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Error adding chunk: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    def delete_chunk(self, chunk_id: str) -> bool:
+        """Delete a specific chunk by its ID"""
+        conn = self._get_connection()
+        cursor = conn.cursor()
+        
+        try:
+            delete_query = f"DELETE FROM {self._table_name} WHERE id = %s AND source_id = %s;"
+            cursor.execute(delete_query, (int(chunk_id), self._source_id))
+            deleted_count = cursor.rowcount
+            conn.commit()
+            
+            return deleted_count > 0
+            
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Error deleting chunk: {e}")
+            return False
+        finally:
+            cursor.close()
+
+    def delete_chunks_by_source_path(self, path: str) -> int:
+        """Delete this source's chunks whose ``metadata.source`` equals ``path``.
+
+        One targeted statement instead of the base loop+scan. The path is bound
+        as a query parameter (never interpolated); only the internal table name
+        is f-string interpolated. Returns the number of rows deleted.
+        """
+        conn = self._get_connection()
+        cursor = conn.cursor()
+
+        try:
+            delete_query = (
+                f"DELETE FROM {self._table_name} "
+                f"WHERE source_id = %s AND {self._metadata_column}->>'source' = %s;"
+            )
+            cursor.execute(delete_query, (self._source_id, path))
+            deleted_count = cursor.rowcount
+            conn.commit()
+
+            return deleted_count
+
+        except Exception as e:
+            conn.rollback()
+            logging.error(f"Error deleting chunks by source path: {e}")
+            raise
+        finally:
+            cursor.close()
+
+    def __del__(self):
+        """Close database connection when object is destroyed"""
+        if hasattr(self, '_connection') and self._connection and not self._connection.closed:
+            self._connection.close()

application/vectorstore/qdrant.py

@@ -0,0 +1,139 @@
+import logging
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+from application.vectorstore.document_class import Document
+
+
+class QdrantStore(BaseVectorStore):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
+        from qdrant_client import models
+        from langchain_community.vectorstores.qdrant import Qdrant
+
+        # Store the source_id for use in add_chunk
+        self._source_id = str(source_id).replace("application/indexes/", "").rstrip("/")
+        
+        self._filter = models.Filter(
+            must=[
+                models.FieldCondition(
+                    key="metadata.source_id",
+                    match=models.MatchValue(value=self._source_id),
+                )
+            ]
+        )
+
+        embedding=self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
+        self._docsearch = Qdrant.construct_instance(
+            ["TEXT_TO_OBTAIN_EMBEDDINGS_DIMENSION"],
+            embedding=embedding,
+            collection_name=settings.QDRANT_COLLECTION_NAME,
+            location=settings.QDRANT_LOCATION,
+            url=settings.QDRANT_URL,
+            port=settings.QDRANT_PORT,
+            grpc_port=settings.QDRANT_GRPC_PORT,
+            https=settings.QDRANT_HTTPS,
+            prefer_grpc=settings.QDRANT_PREFER_GRPC,
+            api_key=settings.QDRANT_API_KEY,
+            prefix=settings.QDRANT_PREFIX,
+            timeout=settings.QDRANT_TIMEOUT,
+            path=settings.QDRANT_PATH,
+            distance_func=settings.QDRANT_DISTANCE_FUNC,
+        )
+        try:
+            collections = self._docsearch.client.get_collections()
+            collection_exists = settings.QDRANT_COLLECTION_NAME in [
+                collection.name for collection in collections.collections
+            ]
+            
+            if not collection_exists:
+                self._docsearch.client.recreate_collection(
+                    collection_name=settings.QDRANT_COLLECTION_NAME,
+                    vectors_config=models.VectorParams(size=embedding.client[1].word_embedding_dimension, distance=models.Distance.COSINE),
+                )
+            
+            # Ensure the required index exists for metadata.source_id
+            try:
+                self._docsearch.client.create_payload_index(
+                    collection_name=settings.QDRANT_COLLECTION_NAME,
+                    field_name="metadata.source_id",
+                    field_schema=models.PayloadSchemaType.KEYWORD,
+                )
+            except Exception as index_error:
+                # Index might already exist, which is fine
+                if "already exists" not in str(index_error).lower():
+                    logging.warning(f"Could not create index for metadata.source_id: {index_error}")
+                    
+        except Exception as e:
+            logging.warning(f"Could not check for collection: {e}")
+
+    def search(self, *args, **kwargs):
+        # Drop the per-source score_threshold (unsupported here) so it is safely
+        # ignored instead of being forwarded into the langchain call.
+        kwargs.pop("score_threshold", None)
+        return self._docsearch.similarity_search(filter=self._filter, *args, **kwargs)
+
+    def add_texts(self, *args, **kwargs):
+        return self._docsearch.add_texts(*args, **kwargs)
+
+    def save_local(self, *args, **kwargs):
+        pass
+
+    def delete_index(self, *args, **kwargs):
+        return self._docsearch.client.delete(
+            collection_name=settings.QDRANT_COLLECTION_NAME, points_selector=self._filter
+        )
+
+    def get_chunks(self):
+        try:
+
+            chunks = []
+            offset = None
+            while True:
+                records, offset = self._docsearch.client.scroll(
+                    collection_name=settings.QDRANT_COLLECTION_NAME,
+                    scroll_filter=self._filter,
+                    limit=10,
+                    with_payload=True,
+                    with_vectors=False,
+                    offset=offset,
+                )
+                for record in records:
+                    doc_id = record.id
+                    text = record.payload.get("page_content")
+                    metadata = record.payload.get("metadata")
+                    chunks.append(
+                        {"doc_id": doc_id, "text": text, "metadata": metadata}
+                    )
+                if offset is None:
+                    break
+            return chunks
+        except Exception as e:
+            logging.error(f"Error getting chunks: {e}", exc_info=True)
+            return []
+
+    def add_chunk(self, text, metadata=None):
+        import uuid
+        metadata = metadata or {}
+        
+        # Create a copy to avoid modifying the original metadata
+        final_metadata = metadata.copy()
+        
+        # Ensure the source_id is in the metadata so the chunk can be found by filters
+        final_metadata["source_id"] = self._source_id
+        
+        doc = Document(page_content=text, metadata=final_metadata)
+        # Generate a unique ID for the document
+        doc_id = str(uuid.uuid4())
+        doc.id = doc_id
+        doc_ids = self._docsearch.add_documents([doc])
+        return doc_ids[0] if doc_ids else doc_id
+
+    def delete_chunk(self, chunk_id):
+        try:
+            self._docsearch.client.delete(
+                collection_name=settings.QDRANT_COLLECTION_NAME,
+                points_selector=[chunk_id],
+            )
+            return True
+        except Exception as e:
+            logging.error(f"Error deleting chunk: {e}", exc_info=True)
+            return False

application/vectorstore/vector_creator.py

@@ -0,0 +1,24 @@
+from application.vectorstore.faiss import FaissStore
+from application.vectorstore.elasticsearch import ElasticsearchStore
+from application.vectorstore.milvus import MilvusStore
+from application.vectorstore.mongodb import MongoDBVectorStore
+from application.vectorstore.qdrant import QdrantStore
+from application.vectorstore.pgvector import PGVectorStore
+
+
+class VectorCreator:
+    vectorstores = {
+        "faiss": FaissStore,
+        "elasticsearch": ElasticsearchStore,
+        "mongodb": MongoDBVectorStore,
+        "qdrant": QdrantStore,
+        "milvus": MilvusStore,
+        "pgvector": PGVectorStore
+    }
+
+    @classmethod
+    def create_vectorstore(cls, type, *args, **kwargs):
+        vectorstore_class = cls.vectorstores.get(type.lower())
+        if not vectorstore_class:
+            raise ValueError(f"No vectorstore class found for type {type}")
+        return vectorstore_class(*args, **kwargs)