chore: import upstream snapshot with attribution

packages/python-lean-ctx/lean_ctx/__init__.py

@@ -0,0 +1,30 @@
+"""lean-ctx SDK — context compression for AI agents and frameworks.
+
+Drop-in usage::
+
+    from lean_ctx import compress
+    messages = compress(messages, model="claude-sonnet-4")
+"""
+
+from lean_ctx.client import LeanCtxClient
+from lean_ctx.errors import LeanCtxAuthError, LeanCtxConnectionError, LeanCtxError
+from lean_ctx.langchain import LeanCtxRetriever, compress_messages
+from lean_ctx.litellm import LeanCtxLiteLLMHandler, compress_request_data
+from lean_ctx.llamaindex import LeanCtxNodeParser
+from lean_ctx.proxy import CompressResult, ProxyClient, compress
+
+__version__ = "0.3.0"
+__all__ = [
+    "compress",
+    "ProxyClient",
+    "CompressResult",
+    "LeanCtxClient",
+    "LeanCtxRetriever",
+    "compress_messages",
+    "LeanCtxLiteLLMHandler",
+    "compress_request_data",
+    "LeanCtxNodeParser",
+    "LeanCtxError",
+    "LeanCtxConnectionError",
+    "LeanCtxAuthError",
+]

packages/python-lean-ctx/lean_ctx/client.py

@@ -0,0 +1,63 @@
+"""Core client for interacting with the lean-ctx daemon."""
+
+import json
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+
+class LeanCtxClient:
+    """Thin wrapper around the lean-ctx CLI for programmatic access."""
+
+    def __init__(self, binary: str = "lean-ctx", project_root: Optional[str] = None):
+        self.binary = binary
+        self.project_root = project_root or str(Path.cwd())
+
+    def read(self, path: str, mode: str = "auto") -> str:
+        return self._run(["read", path, "--mode", mode])
+
+    def search(self, pattern: str, path: Optional[str] = None) -> str:
+        args = ["grep", pattern]
+        if path:
+            args.append(path)
+        return self._run(args)
+
+    def shell(self, command: str) -> str:
+        return self._run(["-c", command])
+
+    def gain(self) -> dict:
+        output = self._run(["gain", "--json"])
+        try:
+            return json.loads(output)
+        except json.JSONDecodeError:
+            return {"raw": output}
+
+    def benchmark(self, path: Optional[str] = None, json_output: bool = True) -> dict:
+        args = ["benchmark", "eval"]
+        if path:
+            args.append(path)
+        if json_output:
+            args.append("--json")
+        output = self._run(args)
+        try:
+            return json.loads(output)
+        except json.JSONDecodeError:
+            return {"raw": output}
+
+    def _run(self, args: list[str]) -> str:
+        try:
+            result = subprocess.run(
+                [self.binary] + args,
+                capture_output=True,
+                text=True,
+                cwd=self.project_root,
+                timeout=30,
+            )
+            return result.stdout.strip()
+        except FileNotFoundError:
+            raise RuntimeError(
+                f"lean-ctx binary not found at '{self.binary}'. "
+                "Install: curl -fsSL https://leanctx.com/install.sh | sh"
+            )
+        except subprocess.TimeoutExpired:
+            raise RuntimeError("lean-ctx command timed out after 30s")

packages/python-lean-ctx/lean_ctx/discovery.py

@@ -0,0 +1,113 @@
+"""Zero-dependency discovery of the local lean-ctx proxy endpoint.
+
+Mirrors the daemon's own resolution so ``compress()`` works out of the box once
+``lean-ctx proxy enable`` has run, while every step stays overridable via the
+same environment variables the CLI honours:
+
+* URL    — ``LEAN_CTX_PROXY_URL`` else ``http://127.0.0.1:<port>``
+* Port   — ``LEAN_CTX_PROXY_PORT`` → ``config.toml`` ``proxy_port`` → UID-derived
+* Token  — ``LEAN_CTX_PROXY_TOKEN`` → ``<data_dir>/session_token``
+* Dirs   — ``LEAN_CTX_DATA_DIR`` / XDG, matching the Rust ``data_dir`` rules
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List, Optional
+
+# Base port the daemon derives per-UID (see proxy_setup::uid_based_port).
+_DEFAULT_PORT = 4444
+
+
+def _candidate_dirs() -> List[Path]:
+    """Ordered data/config directories the daemon may have written to."""
+    dirs: List[Path] = []
+    env = os.environ.get("LEAN_CTX_DATA_DIR", "").strip()
+    if env:
+        dirs.append(Path(env))
+
+    home = Path.home()
+    dirs.append(home / ".lean-ctx")
+
+    xdg_data = os.environ.get("XDG_DATA_HOME", "").strip()
+    dirs.append(Path(xdg_data) / "lean-ctx" if xdg_data else home / ".local" / "share" / "lean-ctx")
+
+    xdg_config = os.environ.get("XDG_CONFIG_HOME", "").strip()
+    dirs.append(Path(xdg_config) / "lean-ctx" if xdg_config else home / ".config" / "lean-ctx")
+
+    # De-duplicate while preserving order.
+    seen = set()
+    unique: List[Path] = []
+    for d in dirs:
+        if d not in seen:
+            seen.add(d)
+            unique.append(d)
+    return unique
+
+
+def _uid_port() -> int:
+    """Replicate proxy_setup::uid_based_port (UID 1000 → 4444, +offset, base for <1000)."""
+    getuid = getattr(os, "getuid", None)
+    if getuid is None:  # Windows
+        return _DEFAULT_PORT
+    uid = getuid()
+    offset = (uid - 1000) % 1000 if uid >= 1000 else 0
+    return _DEFAULT_PORT + offset
+
+
+def _config_port() -> Optional[int]:
+    """Read a top-level ``proxy_port`` from the first config.toml found."""
+    for directory in _candidate_dirs():
+        try:
+            text = (directory / "config.toml").read_text(encoding="utf-8")
+        except OSError:
+            continue
+        for line in text.splitlines():
+            stripped = line.strip()
+            if not stripped.startswith("proxy_port"):
+                continue
+            try:
+                value = stripped.split("=", 1)[1].strip().strip("\"'")
+                return int(value)
+            except (IndexError, ValueError):
+                break
+    return None
+
+
+def resolve_port() -> int:
+    env = os.environ.get("LEAN_CTX_PROXY_PORT", "").strip()
+    if env:
+        try:
+            return int(env)
+        except ValueError:
+            pass
+    cfg = _config_port()
+    if cfg is not None:
+        return cfg
+    return _uid_port()
+
+
+def resolve_base_url(base_url: Optional[str] = None) -> str:
+    if base_url:
+        return base_url.rstrip("/")
+    env = os.environ.get("LEAN_CTX_PROXY_URL", "").strip()
+    if env:
+        return env.rstrip("/")
+    return f"http://127.0.0.1:{resolve_port()}"
+
+
+def resolve_token(token: Optional[str] = None) -> Optional[str]:
+    if token:
+        return token
+    env = os.environ.get("LEAN_CTX_PROXY_TOKEN", "").strip()
+    if env:
+        return env
+    for directory in _candidate_dirs():
+        try:
+            value = (directory / "session_token").read_text(encoding="utf-8").strip()
+        except OSError:
+            continue
+        if value:
+            return value
+    return None

packages/python-lean-ctx/lean_ctx/errors.py

@@ -0,0 +1,21 @@
+"""Exception hierarchy for the lean-ctx SDK."""
+
+
+class LeanCtxError(Exception):
+    """Base class for every error raised by the lean-ctx SDK."""
+
+
+class LeanCtxConnectionError(LeanCtxError):
+    """The local lean-ctx proxy could not be reached.
+
+    Usually means the daemon is not running (``lean-ctx proxy enable``) or the
+    discovered host/port is wrong (override with ``LEAN_CTX_PROXY_URL``).
+    """
+
+
+class LeanCtxAuthError(LeanCtxError):
+    """The proxy rejected the request (missing or invalid session token).
+
+    Provide the token explicitly, or export ``LEAN_CTX_PROXY_TOKEN`` so the SDK
+    can authenticate against the loopback proxy.
+    """

packages/python-lean-ctx/lean_ctx/langchain.py

@@ -0,0 +1,92 @@
+"""LangChain integration for lean-ctx."""
+
+from typing import Any, List, Optional
+
+from lean_ctx.client import LeanCtxClient
+from lean_ctx.proxy import compress as _compress_dicts
+
+
+def _reattach_content(originals: List[Any], compressed: List[Any]) -> List[Any]:
+    """Return clones of ``originals`` with ``content`` taken from ``compressed``.
+
+    Pydantic ``model_copy`` preserves each message's type and metadata; only the
+    textual content is swapped. If the conversion changed the message count (e.g.
+    tool-call splitting) the mapping is ambiguous, so the originals are returned
+    unchanged rather than risk corrupting the transcript.
+    """
+    if len(originals) != len(compressed):
+        return list(originals)
+    out: List[Any] = []
+    for original, comp in zip(originals, compressed):
+        new_content = comp.get("content") if isinstance(comp, dict) else None
+        if new_content is None or not hasattr(original, "model_copy"):
+            out.append(original)
+        else:
+            out.append(original.model_copy(update={"content": new_content}))
+    return out
+
+
+def compress_messages(messages: List[Any], model: Optional[str] = None, **kwargs: Any) -> List[Any]:
+    """Compress the textual content of a list of LangChain ``BaseMessage`` objects.
+
+    Converts to the OpenAI wire shape, runs the deterministic proxy compression,
+    and returns new messages with only their ``content`` rewritten. Requires
+    ``langchain-core``. Extra keyword arguments (``base_url``, ``token``,
+    ``timeout``) are forwarded to :func:`lean_ctx.compress`.
+    """
+    try:
+        from langchain_core.messages import convert_to_openai_messages
+    except ImportError as exc:  # pragma: no cover - optional dependency
+        raise ImportError("langchain-core is required: pip install langchain-core") from exc
+
+    openai_messages = convert_to_openai_messages(messages)
+    compressed = _compress_dicts(openai_messages, model, **kwargs)
+    return _reattach_content(messages, compressed)
+
+try:
+    from langchain_core.retrievers import BaseRetriever
+    from langchain_core.documents import Document
+    from langchain_core.callbacks import CallbackManagerForRetrieverRun
+
+    class LeanCtxRetriever(BaseRetriever):
+        """LangChain retriever backed by lean-ctx hybrid search."""
+
+        client: LeanCtxClient = None
+        top_k: int = 10
+
+        def __init__(self, project_root: Optional[str] = None, top_k: int = 10, **kwargs):
+            super().__init__(**kwargs)
+            self.client = LeanCtxClient(project_root=project_root)
+            self.top_k = top_k
+
+        def _get_relevant_documents(
+            self, query: str, *, run_manager: CallbackManagerForRetrieverRun
+        ) -> list[Document]:
+            result = self.client.search(query)
+            documents = []
+            for line in result.split("\n"):
+                if not line.strip():
+                    continue
+                parts = line.split(":", 2)
+                if len(parts) >= 3:
+                    file_path, line_num, content = parts[0], parts[1], parts[2]
+                    documents.append(
+                        Document(
+                            page_content=content.strip(),
+                            metadata={"source": file_path, "line": line_num},
+                        )
+                    )
+                else:
+                    documents.append(Document(page_content=line.strip()))
+
+            return documents[: self.top_k]
+
+except ImportError:
+
+    class LeanCtxRetriever:
+        """Stub: install langchain-core for full integration."""
+
+        def __init__(self, **kwargs):
+            raise ImportError(
+                "langchain-core is required: pip install langchain-core"
+            )

packages/python-lean-ctx/lean_ctx/litellm.py

@@ -0,0 +1,105 @@
+"""LiteLLM integration for lean-ctx.
+
+Compresses the ``messages`` of an outbound LiteLLM request through the local
+proxy before it is sent to the provider. Two entry points:
+
+* :func:`compress_request_data` — framework-agnostic helper that rewrites a
+  request ``dict`` in place (testable without LiteLLM installed).
+* :class:`LeanCtxLiteLLMHandler` — a LiteLLM ``CustomLogger`` that hooks
+  ``async_pre_call_hook`` for use with LiteLLM Proxy or ``litellm.callbacks``.
+
+Register programmatically::
+
+    import litellm
+    from lean_ctx.litellm import LeanCtxLiteLLMHandler
+
+    litellm.callbacks = [LeanCtxLiteLLMHandler(model="gpt-4o")]
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, Optional
+
+from .errors import LeanCtxError
+from .proxy import ProxyClient
+
+_PRECALL_TYPES = ("completion", "text_completion")
+
+
+def compress_request_data(
+    data: Dict[str, Any],
+    *,
+    client: Optional[ProxyClient] = None,
+    model: Optional[str] = None,
+    raise_on_error: bool = False,
+) -> Dict[str, Any]:
+    """Compress ``data["messages"]`` in place and return ``data``.
+
+    Mirrors the LiteLLM/OpenAI request shape. On a proxy failure the messages are
+    left untouched — a compaction hiccup must never block the LLM call — unless
+    ``raise_on_error`` is set.
+    """
+    messages = data.get("messages")
+    if not isinstance(messages, list) or not messages:
+        return data
+    proxy = client or ProxyClient()
+    try:
+        data["messages"] = proxy.compress(messages, model=model or data.get("model")).messages
+    except LeanCtxError:
+        if raise_on_error:
+            raise
+    return data
+
+
+try:  # pragma: no cover - import wiring, exercised by presence/absence of litellm
+    from litellm.integrations.custom_logger import CustomLogger
+
+    _BASE: type = CustomLogger
+    _LITELLM_AVAILABLE = True
+except Exception:  # noqa: BLE001 - any litellm import failure means "not available"
+    _BASE = object
+    _LITELLM_AVAILABLE = False
+
+
+class LeanCtxLiteLLMHandler(_BASE):  # type: ignore[misc,valid-type]
+    """LiteLLM ``CustomLogger`` that compresses requests in ``async_pre_call_hook``.
+
+    The synchronous (urllib) :class:`ProxyClient` call is dispatched to a thread
+    so it never blocks the proxy event loop.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: Optional[str] = None,
+        raise_on_error: bool = False,
+        base_url: Optional[str] = None,
+        token: Optional[str] = None,
+    ) -> None:
+        if not _LITELLM_AVAILABLE:
+            raise ImportError("litellm is required: pip install litellm")
+        super().__init__()
+        self._client = ProxyClient(base_url=base_url, token=token)
+        self._model = model
+        self._raise_on_error = raise_on_error
+
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: Any,
+        cache: Any,
+        data: Dict[str, Any],
+        call_type: str,
+    ) -> Dict[str, Any]:
+        if call_type in _PRECALL_TYPES and isinstance(data, dict):
+            loop = asyncio.get_running_loop()
+            await loop.run_in_executor(
+                None,
+                lambda: compress_request_data(
+                    data,
+                    client=self._client,
+                    model=self._model,
+                    raise_on_error=self._raise_on_error,
+                ),
+            )
+        return data

packages/python-lean-ctx/lean_ctx/llamaindex.py

@@ -0,0 +1,47 @@
+"""LlamaIndex integration for lean-ctx."""
+
+from typing import Optional
+
+from lean_ctx.client import LeanCtxClient
+
+try:
+    from llama_index.core.node_parser import NodeParser
+    from llama_index.core.schema import BaseNode, TextNode, Document
+
+    class LeanCtxNodeParser(NodeParser):
+        """LlamaIndex node parser using lean-ctx compression modes."""
+
+        client: LeanCtxClient = None
+        mode: str = "map"
+
+        def __init__(self, project_root: Optional[str] = None, mode: str = "map", **kwargs):
+            super().__init__(**kwargs)
+            self.client = LeanCtxClient(project_root=project_root)
+            self.mode = mode
+
+        def _parse_nodes(self, nodes: list[BaseNode], **kwargs) -> list[BaseNode]:
+            result_nodes = []
+            for node in nodes:
+                if isinstance(node, Document) and hasattr(node, "metadata"):
+                    file_path = node.metadata.get("file_path", "")
+                    if file_path:
+                        compressed = self.client.read(file_path, mode=self.mode)
+                        result_nodes.append(
+                            TextNode(
+                                text=compressed,
+                                metadata={**node.metadata, "compression_mode": self.mode},
+                            )
+                        )
+                        continue
+                result_nodes.append(node)
+            return result_nodes
+
+except ImportError:
+
+    class LeanCtxNodeParser:
+        """Stub: install llama-index-core for full integration."""
+
+        def __init__(self, **kwargs):
+            raise ImportError(
+                "llama-index-core is required: pip install llama-index-core"
+            )

packages/python-lean-ctx/lean_ctx/proxy.py

@@ -0,0 +1,164 @@
+"""Drop-in ``compress(messages, model)`` over the local lean-ctx proxy.
+
+Posts a chat-style ``messages`` array to the daemon's deterministic
+``POST /v1/compress`` endpoint and returns the rewritten messages. Only text
+payloads are compressed; images, tool-call blocks and ids pass through
+untouched, and the output is byte-stable for provider prompt caching.
+
+Stdlib-only (``urllib``) so ``pip install lean-ctx-sdk`` pulls in no transitive
+dependencies.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from . import discovery
+from .errors import LeanCtxAuthError, LeanCtxConnectionError, LeanCtxError
+
+Message = Dict[str, Any]
+
+_DEFAULT_TIMEOUT = 30.0
+
+
+@dataclass
+class CompressResult:
+    """Result of a ``/v1/compress`` call: rewritten messages plus savings."""
+
+    messages: List[Message]
+    stats: Dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def original_tokens(self) -> int:
+        return int(self.stats.get("original_tokens", 0))
+
+    @property
+    def compressed_tokens(self) -> int:
+        return int(self.stats.get("compressed_tokens", 0))
+
+    @property
+    def saved_tokens(self) -> int:
+        return int(self.stats.get("saved_tokens", 0))
+
+    @property
+    def saved_pct(self) -> float:
+        return float(self.stats.get("saved_pct", 0.0))
+
+
+class ProxyClient:
+    """Reusable client for the local lean-ctx proxy ``/v1/compress`` endpoint.
+
+    Endpoint and token are auto-discovered (env → config → UID/data-dir) and may
+    be overridden explicitly for CI or remote proxies.
+    """
+
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        token: Optional[str] = None,
+        timeout: float = _DEFAULT_TIMEOUT,
+    ) -> None:
+        self.base_url = discovery.resolve_base_url(base_url)
+        self.token = discovery.resolve_token(token)
+        self.timeout = timeout
+
+    def compress(
+        self,
+        messages: List[Message],
+        model: Optional[str] = None,
+    ) -> CompressResult:
+        """Compress ``messages`` and return the rewritten list plus stats."""
+        if not isinstance(messages, list):
+            raise TypeError("messages must be a list of chat-message dicts")
+
+        payload: Dict[str, Any] = {"messages": messages}
+        if model:
+            payload["model"] = model
+
+        data = self._post("/v1/compress", payload)
+        out = data.get("messages")
+        if not isinstance(out, list):
+            raise LeanCtxError("malformed /v1/compress response: 'messages' missing")
+        stats = data.get("stats")
+        return CompressResult(messages=out, stats=stats if isinstance(stats, dict) else {})
+
+    def resolve_reference(self, reference_id: str) -> str:
+        """Return the original content behind a lean-ctx reference id.
+
+        lean-ctx replaces oversized omitted content with a durable reference; this
+        fetches it back via ``GET /v1/references/{id}``. Raises :class:`LeanCtxError`
+        when the reference is unknown or expired.
+        """
+        if not reference_id:
+            raise ValueError("reference_id must be a non-empty string")
+        quoted = urllib.parse.quote(reference_id, safe="")
+        request = urllib.request.Request(f"{self.base_url}/v1/references/{quoted}", method="GET")
+        if self.token:
+            request.add_header("Authorization", f"Bearer {self.token}")
+        return self._send(request).decode("utf-8")
+
+    def _post(self, path: str, payload: Dict[str, Any]) -> Dict[str, Any]:
+        body = json.dumps(payload).encode("utf-8")
+        request = urllib.request.Request(f"{self.base_url}{path}", data=body, method="POST")
+        request.add_header("Content-Type", "application/json")
+        if self.token:
+            request.add_header("Authorization", f"Bearer {self.token}")
+        raw = self._send(request)
+        try:
+            return json.loads(raw.decode("utf-8"))
+        except (ValueError, TypeError) as exc:
+            raise LeanCtxError(f"invalid JSON response from {request.full_url}: {exc}") from exc
+
+    def _send(self, request: urllib.request.Request) -> bytes:
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as response:
+                return response.read()
+        except urllib.error.HTTPError as exc:
+            detail = exc.read().decode("utf-8", "replace").strip()
+            if exc.code in (401, 403):
+                raise LeanCtxAuthError(
+                    f"proxy rejected the request (HTTP {exc.code}). "
+                    "Set LEAN_CTX_PROXY_TOKEN or pass token=…"
+                ) from exc
+            if exc.code == 404:
+                raise LeanCtxError(
+                    f"{request.full_url} not found (HTTP 404): {detail}"
+                ) from exc
+            raise LeanCtxError(
+                f"{request.get_method()} {request.full_url} failed (HTTP {exc.code}): {detail}"
+            ) from exc
+        except urllib.error.URLError as exc:
+            raise LeanCtxConnectionError(
+                f"could not reach the lean-ctx proxy at {self.base_url} ({exc.reason}). "
+                "Is the daemon running? Try: lean-ctx proxy enable"
+            ) from exc
+
+
+def compress(
+    messages: List[Message],
+    model: Optional[str] = None,
+    *,
+    base_url: Optional[str] = None,
+    token: Optional[str] = None,
+    timeout: float = _DEFAULT_TIMEOUT,
+) -> List[Message]:
+    """Compress a chat ``messages`` array, returning the rewritten messages.
+
+    Drop-in parity with library-style gateways::
+
+        from lean_ctx import compress
+        messages = compress(messages, model="claude-sonnet-4")
+
+    For token-savings stats, use :class:`ProxyClient` directly::
+
+        from lean_ctx import ProxyClient
+        result = ProxyClient().compress(messages)
+        print(result.saved_pct)
+    """
+    client = ProxyClient(base_url=base_url, token=token, timeout=timeout)
+    return client.compress(messages, model=model).messages