microsoft--graphrag/packages/graphrag-storage/graphrag_storage/tables/cosmos_table.py

# Copyright (c) 2024 Microsoft Corporation.
# Licensed under the MIT License

"""Cosmos DB implementation of the Table abstraction for streaming row access."""

from __future__ import annotations

import inspect
import logging
from typing import TYPE_CHECKING, Any

from azure.cosmos.exceptions import CosmosResourceNotFoundError

from graphrag_storage.tables.table import RowTransformer, Table

if TYPE_CHECKING:
    from collections.abc import AsyncIterator

    from azure.cosmos.aio import ContainerProxy

logger = logging.getLogger(__name__)

_DEFAULT_PAGE_SIZE = 100
_DEFAULT_BATCH_SIZE = 50


def _identity(row: dict[str, Any]) -> Any:
    """Return row unchanged (default transformer)."""
    return row


def _apply_transformer(transformer: RowTransformer, row: dict[str, Any]) -> Any:
    """Apply transformer, handling both callables and classes (e.g. Pydantic models)."""
    if inspect.isclass(transformer):
        return transformer(**row)
    return transformer(row)


# Cosmos system properties to strip from returned rows.
_COSMOS_SYSTEM_KEYS = frozenset({
    "_rid",
    "_self",
    "_etag",
    "_attachments",
    "_ts",
    "namespace",
    "table_name",
})


def _strip_cosmos_metadata(doc: dict[str, Any]) -> dict[str, Any]:
    """Remove Cosmos system properties and restore original id."""
    result = {k: v for k, v in doc.items() if k not in _COSMOS_SYSTEM_KEYS}
    # Restore the pipeline's original id from row_id if present.
    if "row_id" in result:
        result["id"] = result.pop("row_id")
    return result


class CosmosTable(Table):
    """Streaming table interface backed by Cosmos DB.

    Reads page through query_items() using the async SDK, yielding rows
    one at a time.  Writes accumulate in memory and are bulk-upserted on
    close().
    """

    def __init__(
        self,
        container: ContainerProxy,
        table_name: str,
        namespace: str,
        transformer: RowTransformer | None = None,
        truncate: bool = True,
        page_size: int = _DEFAULT_PAGE_SIZE,
        batch_size: int = _DEFAULT_BATCH_SIZE,
    ) -> None:
        """Initialise with a Cosmos container and table/namespace identifiers."""
        self._container = container
        self._table_name = table_name
        self._namespace = namespace
        self._transformer = transformer or _identity
        self._truncate = truncate
        self._page_size = page_size
        self._batch_size = batch_size
        self._write_rows: list[dict[str, Any]] = []

    # ------------------------------------------------------------------
    # Read
    # ------------------------------------------------------------------

    def __aiter__(self) -> AsyncIterator[Any]:
        """Iterate rows asynchronously."""
        return self._aiter_impl()

    async def _aiter_impl(self) -> AsyncIterator[Any]:
        """Page through Cosmos query results, yielding transformed rows."""
        query = "SELECT * FROM c WHERE c.table_name = @table_name"
        parameters: list[dict[str, Any]] = [
            {"name": "@table_name", "value": self._table_name},
        ]
        async for page in self._container.query_items(
            query=query,
            parameters=parameters,
            partition_key=self._namespace,
            max_item_count=self._page_size,
        ).by_page():
            async for doc in page:
                row = _strip_cosmos_metadata(doc)
                yield _apply_transformer(self._transformer, row)

    async def length(self) -> int:
        """Return the number of rows in the table (single-partition COUNT)."""
        query = "SELECT VALUE COUNT(1) FROM c WHERE c.table_name = @table_name"
        parameters: list[dict[str, Any]] = [
            {"name": "@table_name", "value": self._table_name},
        ]
        results: list[Any] = []
        async for item in self._container.query_items(
            query=query,
            parameters=parameters,
            partition_key=self._namespace,
        ):
            results.append(item)  # noqa: PERF401
        return int(results[0]) if results else 0

    async def has(self, row_id: str) -> bool:
        """Check if a row with the given ID exists (point-read)."""
        cosmos_id = f"{self._table_name}:{row_id}"
        try:
            await self._container.read_item(
                item=cosmos_id, partition_key=self._namespace
            )
        except CosmosResourceNotFoundError:
            return False
        else:
            return True

    # ------------------------------------------------------------------
    # Write
    # ------------------------------------------------------------------

    async def write(self, row: dict[str, Any]) -> None:
        """Accumulate a row for batch upsert on close()."""
        self._write_rows.append(row)

    async def close(self) -> None:
        """Flush accumulated rows to Cosmos DB.

        If truncate=True, existing documents for this table in the namespace
        are deleted before writing.
        """
        if self._truncate and self._write_rows:
            await self._delete_table_docs()

        docs = []
        for index, row in enumerate(self._write_rows):
            doc = dict(row)
            row_key = doc.pop("id", index)
            doc["id"] = f"{self._table_name}:{row_key}"
            doc["namespace"] = self._namespace
            doc["table_name"] = self._table_name
            doc["row_id"] = row_key
            docs.append(doc)

        if docs:
            from graphrag_storage.tables.cosmos_table_provider import (
                _batch_upsert,
            )

            await _batch_upsert(
                self._container, docs, self._namespace, self._batch_size
            )

        self._write_rows = []

    async def _delete_table_docs(self) -> None:
        """Delete all documents for this table in the current namespace."""
        query = "SELECT c.id FROM c WHERE c.table_name = @table_name"
        parameters: list[dict[str, Any]] = [
            {"name": "@table_name", "value": self._table_name},
        ]
        async for page in self._container.query_items(
            query=query,
            parameters=parameters,
            partition_key=self._namespace,
        ).by_page():
            async for doc in page:
                await self._container.delete_item(
                    item=doc["id"], partition_key=self._namespace
                )