chore: import upstream snapshot with attribution
CI / changes (push) Has been cancelled
CI / cd libs/checkpoint (push) Has been cancelled
CI / cd libs/checkpoint-conformance (push) Has been cancelled
CI / cd libs/checkpoint-postgres (push) Has been cancelled
CI / cd libs/checkpoint-sqlite (push) Has been cancelled
CI / cd libs/cli (push) Has been cancelled
CI / cd libs/prebuilt (push) Has been cancelled
CI / cd libs/sdk-py (push) Has been cancelled
CI / cd libs/langgraph (push) Has been cancelled
CI / Check SDK methods matching (push) Has been cancelled
CI / Check CLI schema hasn't changed #3.13 (push) Has been cancelled
CI / CLI integration test (push) Has been cancelled
CI / sdk-py integration test (push) Has been cancelled
CI / CI Success (push) Has been cancelled
baseline / benchmark (push) Has been cancelled
Deploy Redirects to GitHub Pages / deploy (push) Has been cancelled
CI / changes (push) Has been cancelled
CI / cd libs/checkpoint (push) Has been cancelled
CI / cd libs/checkpoint-conformance (push) Has been cancelled
CI / cd libs/checkpoint-postgres (push) Has been cancelled
CI / cd libs/checkpoint-sqlite (push) Has been cancelled
CI / cd libs/cli (push) Has been cancelled
CI / cd libs/prebuilt (push) Has been cancelled
CI / cd libs/sdk-py (push) Has been cancelled
CI / cd libs/langgraph (push) Has been cancelled
CI / Check SDK methods matching (push) Has been cancelled
CI / Check CLI schema hasn't changed #3.13 (push) Has been cancelled
CI / CLI integration test (push) Has been cancelled
CI / sdk-py integration test (push) Has been cancelled
CI / CI Success (push) Has been cancelled
baseline / benchmark (push) Has been cancelled
Deploy Redirects to GitHub Pages / deploy (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
services:
|
||||
postgres-test:
|
||||
image: pgvector/pgvector:pg${POSTGRES_VERSION:-16}
|
||||
ports:
|
||||
- "5441:5432"
|
||||
environment:
|
||||
POSTGRES_DB: postgres
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
command: ["postgres", "-c", "shared_preload_libraries=vector"]
|
||||
healthcheck:
|
||||
test: pg_isready -U postgres
|
||||
start_period: 10s
|
||||
timeout: 1s
|
||||
retries: 5
|
||||
interval: 60s
|
||||
start_interval: 1s
|
||||
@@ -0,0 +1,44 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import pytest
|
||||
from psycopg import AsyncConnection
|
||||
from psycopg.errors import UndefinedTable
|
||||
from psycopg.rows import DictRow, dict_row
|
||||
|
||||
from tests.embed_test_utils import CharacterEmbeddings
|
||||
|
||||
DEFAULT_POSTGRES_URI = "postgres://postgres:postgres@localhost:5441/"
|
||||
DEFAULT_URI = "postgres://postgres:postgres@localhost:5441/postgres?sslmode=disable"
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
async def conn() -> AsyncIterator[AsyncConnection[DictRow]]:
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_URI, autocommit=True, prepare_threshold=0, row_factory=dict_row
|
||||
) as conn:
|
||||
yield conn
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", autouse=True)
|
||||
async def clear_test_db(conn: AsyncConnection[DictRow]) -> None:
|
||||
"""Delete all tables before each test."""
|
||||
try:
|
||||
await conn.execute("DELETE FROM checkpoints")
|
||||
await conn.execute("DELETE FROM checkpoint_blobs")
|
||||
await conn.execute("DELETE FROM checkpoint_writes")
|
||||
await conn.execute("DELETE FROM checkpoint_migrations")
|
||||
except UndefinedTable:
|
||||
pass
|
||||
try:
|
||||
await conn.execute("DELETE FROM store_migrations")
|
||||
await conn.execute("DELETE FROM store")
|
||||
except UndefinedTable:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_embeddings() -> CharacterEmbeddings:
|
||||
return CharacterEmbeddings(dims=500)
|
||||
|
||||
|
||||
VECTOR_TYPES = ["vector", "halfvec"]
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Embedding utilities for testing."""
|
||||
|
||||
import math
|
||||
import random
|
||||
from collections import Counter, defaultdict
|
||||
from typing import Any
|
||||
|
||||
from langchain_core.embeddings import Embeddings
|
||||
|
||||
|
||||
class CharacterEmbeddings(Embeddings):
|
||||
"""Simple character-frequency based embeddings using random projections."""
|
||||
|
||||
def __init__(self, dims: int = 50, seed: int = 42):
|
||||
"""Initialize with embedding dimensions and random seed."""
|
||||
self._rng = random.Random(seed)
|
||||
self.dims = dims
|
||||
# Create projection vector for each character lazily
|
||||
self._char_projections: defaultdict[str, list[float]] = defaultdict(
|
||||
lambda: [
|
||||
self._rng.gauss(0, 1 / math.sqrt(self.dims)) for _ in range(self.dims)
|
||||
]
|
||||
)
|
||||
|
||||
def _embed_one(self, text: str) -> list[float]:
|
||||
"""Embed a single text."""
|
||||
counts = Counter(text)
|
||||
total = sum(counts.values())
|
||||
|
||||
if total == 0:
|
||||
return [0.0] * self.dims
|
||||
|
||||
embedding = [0.0] * self.dims
|
||||
for char, count in counts.items():
|
||||
weight = count / total
|
||||
char_proj = self._char_projections[char]
|
||||
for i, proj in enumerate(char_proj):
|
||||
embedding[i] += weight * proj
|
||||
|
||||
norm = math.sqrt(sum(x * x for x in embedding))
|
||||
if norm > 0:
|
||||
embedding = [x / norm for x in embedding]
|
||||
|
||||
return embedding
|
||||
|
||||
def embed_documents(self, texts: list[str]) -> list[list[float]]:
|
||||
"""Embed a list of documents."""
|
||||
return [self._embed_one(text) for text in texts]
|
||||
|
||||
def embed_query(self, text: str) -> list[float]:
|
||||
"""Embed a query string."""
|
||||
return self._embed_one(text)
|
||||
|
||||
def __eq__(self, other: Any) -> bool:
|
||||
return isinstance(other, CharacterEmbeddings) and self.dims == other.dims
|
||||
@@ -0,0 +1,417 @@
|
||||
# type: ignore
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
from langgraph.checkpoint.base import (
|
||||
EXCLUDED_METADATA_KEYS,
|
||||
Checkpoint,
|
||||
CheckpointMetadata,
|
||||
create_checkpoint,
|
||||
empty_checkpoint,
|
||||
)
|
||||
from langgraph.checkpoint.serde.types import TASKS
|
||||
from psycopg import AsyncConnection
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
|
||||
from langgraph.checkpoint.postgres.aio import (
|
||||
AsyncPostgresSaver,
|
||||
AsyncShallowPostgresSaver,
|
||||
)
|
||||
from tests.conftest import DEFAULT_POSTGRES_URI
|
||||
|
||||
|
||||
def _exclude_keys(config: dict[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in config.items() if k not in EXCLUDED_METADATA_KEYS}
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _pool_saver():
|
||||
"""Fixture for pool mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
# yield checkpointer
|
||||
async with AsyncConnectionPool(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
max_size=10,
|
||||
kwargs={"autocommit": True, "row_factory": dict_row},
|
||||
) as pool:
|
||||
checkpointer = AsyncPostgresSaver(pool)
|
||||
await checkpointer.setup()
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _pipe_saver():
|
||||
"""Fixture for pipeline mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
autocommit=True,
|
||||
prepare_threshold=0,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
checkpointer = AsyncPostgresSaver(conn)
|
||||
await checkpointer.setup()
|
||||
async with conn.pipeline() as pipe:
|
||||
checkpointer = AsyncPostgresSaver(conn, pipe=pipe)
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _base_saver():
|
||||
"""Fixture for regular connection mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
autocommit=True,
|
||||
prepare_threshold=0,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
checkpointer = AsyncPostgresSaver(conn)
|
||||
await checkpointer.setup()
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _shallow_saver():
|
||||
"""Fixture for shallow connection mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
autocommit=True,
|
||||
prepare_threshold=0,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
checkpointer = AsyncShallowPostgresSaver(conn)
|
||||
await checkpointer.setup()
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
async with await AsyncConnection.connect(
|
||||
DEFAULT_POSTGRES_URI, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _saver(name: str):
|
||||
if name == "base":
|
||||
async with _base_saver() as saver:
|
||||
yield saver
|
||||
elif name == "shallow":
|
||||
async with _shallow_saver() as saver:
|
||||
yield saver
|
||||
elif name == "pool":
|
||||
async with _pool_saver() as saver:
|
||||
yield saver
|
||||
elif name == "pipe":
|
||||
async with _pipe_saver() as saver:
|
||||
yield saver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_data():
|
||||
"""Fixture providing test data for checkpoint tests."""
|
||||
config_1: RunnableConfig = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-1",
|
||||
"checkpoint_id": "1",
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
config_2: RunnableConfig = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_id": "2",
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
config_3: RunnableConfig = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_id": "2-inner",
|
||||
"checkpoint_ns": "inner",
|
||||
}
|
||||
}
|
||||
|
||||
chkpnt_1: Checkpoint = empty_checkpoint()
|
||||
chkpnt_2: Checkpoint = create_checkpoint(chkpnt_1, {}, 1)
|
||||
chkpnt_3: Checkpoint = empty_checkpoint()
|
||||
|
||||
metadata_1: CheckpointMetadata = {
|
||||
"source": "input",
|
||||
"step": 2,
|
||||
"score": 1,
|
||||
}
|
||||
metadata_2: CheckpointMetadata = {
|
||||
"source": "loop",
|
||||
"step": 1,
|
||||
"score": None,
|
||||
}
|
||||
metadata_3: CheckpointMetadata = {}
|
||||
|
||||
return {
|
||||
"configs": [config_1, config_2, config_3],
|
||||
"checkpoints": [chkpnt_1, chkpnt_2, chkpnt_3],
|
||||
"metadata": [metadata_1, metadata_2, metadata_3],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe", "shallow"])
|
||||
async def test_combined_metadata(saver_name: str, test_data) -> None:
|
||||
async with _saver(saver_name) as saver:
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_ns": "",
|
||||
"__super_private_key": "super_private_value",
|
||||
},
|
||||
"metadata": {"run_id": "my_run_id"},
|
||||
}
|
||||
chkpnt: Checkpoint = create_checkpoint(empty_checkpoint(), {}, 1)
|
||||
metadata: CheckpointMetadata = {
|
||||
"source": "loop",
|
||||
"step": 1,
|
||||
"score": None,
|
||||
}
|
||||
await saver.aput(config, chkpnt, metadata, {})
|
||||
checkpoint = await saver.aget_tuple(config)
|
||||
assert checkpoint.metadata == {
|
||||
**metadata,
|
||||
"run_id": "my_run_id",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe", "shallow"])
|
||||
async def test_asearch(saver_name: str, test_data) -> None:
|
||||
async with _saver(saver_name) as saver:
|
||||
configs = test_data["configs"]
|
||||
checkpoints = test_data["checkpoints"]
|
||||
metadata = test_data["metadata"]
|
||||
|
||||
await saver.aput(configs[0], checkpoints[0], metadata[0], {})
|
||||
await saver.aput(configs[1], checkpoints[1], metadata[1], {})
|
||||
await saver.aput(configs[2], checkpoints[2], metadata[2], {})
|
||||
|
||||
# call method / assertions
|
||||
query_1 = {"source": "input"} # search by 1 key
|
||||
query_2 = {
|
||||
"step": 1,
|
||||
} # search by multiple keys
|
||||
query_3: dict[str, Any] = {} # search by no keys, return all checkpoints
|
||||
query_4 = {"source": "update", "step": 1} # no match
|
||||
|
||||
search_results_1 = [c async for c in saver.alist(None, filter=query_1)]
|
||||
assert len(search_results_1) == 1
|
||||
assert search_results_1[0].metadata == {
|
||||
**_exclude_keys(configs[0]["configurable"]),
|
||||
**metadata[0],
|
||||
}
|
||||
|
||||
search_results_2 = [c async for c in saver.alist(None, filter=query_2)]
|
||||
assert len(search_results_2) == 1
|
||||
assert search_results_2[0].metadata == {
|
||||
**_exclude_keys(configs[1]["configurable"]),
|
||||
**metadata[1],
|
||||
}
|
||||
|
||||
search_results_3 = [c async for c in saver.alist(None, filter=query_3)]
|
||||
assert len(search_results_3) == 3
|
||||
|
||||
search_results_4 = [c async for c in saver.alist(None, filter=query_4)]
|
||||
assert len(search_results_4) == 0
|
||||
|
||||
# search by config (defaults to checkpoints across all namespaces)
|
||||
search_results_5 = [
|
||||
c async for c in saver.alist({"configurable": {"thread_id": "thread-2"}})
|
||||
]
|
||||
assert len(search_results_5) == 2
|
||||
assert {
|
||||
search_results_5[0].config["configurable"]["checkpoint_ns"],
|
||||
search_results_5[1].config["configurable"]["checkpoint_ns"],
|
||||
} == {"", "inner"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe", "shallow"])
|
||||
async def test_null_chars(saver_name: str, test_data) -> None:
|
||||
async with _saver(saver_name) as saver:
|
||||
config = await saver.aput(
|
||||
test_data["configs"][0],
|
||||
test_data["checkpoints"][0],
|
||||
{"my_key": "\x00abc"},
|
||||
{},
|
||||
)
|
||||
assert (await saver.aget_tuple(config)).metadata["my_key"] == "abc" # type: ignore
|
||||
assert [c async for c in saver.alist(None, filter={"my_key": "abc"})][
|
||||
0
|
||||
].metadata["my_key"] == "abc"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe"])
|
||||
async def test_pending_sends_migration(saver_name: str) -> None:
|
||||
async with _saver(saver_name) as saver:
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-1",
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
|
||||
# create the first checkpoint
|
||||
# and put some pending sends
|
||||
checkpoint_0 = empty_checkpoint()
|
||||
config = await saver.aput(config, checkpoint_0, {}, {})
|
||||
await saver.aput_writes(
|
||||
config, [(TASKS, "send-1"), (TASKS, "send-2")], task_id="task-1"
|
||||
)
|
||||
await saver.aput_writes(config, [(TASKS, "send-3")], task_id="task-2")
|
||||
|
||||
# check that fetching checkpoint_0 doesn't attach pending sends
|
||||
# (they should be attached to the next checkpoint)
|
||||
tuple_0 = await saver.aget_tuple(config)
|
||||
assert tuple_0.checkpoint["channel_values"] == {}
|
||||
assert tuple_0.checkpoint["channel_versions"] == {}
|
||||
|
||||
# create the second checkpoint
|
||||
checkpoint_1 = create_checkpoint(checkpoint_0, {}, 1)
|
||||
config = await saver.aput(config, checkpoint_1, {}, {})
|
||||
|
||||
# check that pending sends are attached to checkpoint_1
|
||||
tuple_1 = await saver.aget_tuple(config)
|
||||
assert tuple_1.checkpoint["channel_values"] == {
|
||||
TASKS: ["send-1", "send-2", "send-3"]
|
||||
}
|
||||
assert TASKS in tuple_1.checkpoint["channel_versions"]
|
||||
|
||||
# check that list also applies the migration
|
||||
search_results = [
|
||||
c async for c in saver.alist({"configurable": {"thread_id": "thread-1"}})
|
||||
]
|
||||
assert len(search_results) == 2
|
||||
assert search_results[-1].checkpoint["channel_values"] == {}
|
||||
assert search_results[-1].checkpoint["channel_versions"] == {}
|
||||
assert search_results[0].checkpoint["channel_values"] == {
|
||||
TASKS: ["send-1", "send-2", "send-3"]
|
||||
}
|
||||
assert TASKS in search_results[0].checkpoint["channel_versions"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe"])
|
||||
async def test_get_checkpoint_no_channel_values(
|
||||
monkeypatch, saver_name: str, test_data
|
||||
) -> None:
|
||||
"""Backwards compatibility test that verifies a checkpoint with no channel_values key can be retrieved without throwing an error."""
|
||||
async with _saver(saver_name) as saver:
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_ns": "",
|
||||
"__super_private_key": "super_private_value",
|
||||
},
|
||||
"metadata": {"run_id": "my_run_id"},
|
||||
}
|
||||
chkpnt: Checkpoint = create_checkpoint(empty_checkpoint(), {}, 1)
|
||||
await saver.aput(config, chkpnt, {}, {})
|
||||
|
||||
load_checkpoint_tuple = saver._load_checkpoint_tuple
|
||||
|
||||
async def patched_load_checkpoint_tuple(value):
|
||||
value["checkpoint"].pop("channel_values", None)
|
||||
return await load_checkpoint_tuple(value)
|
||||
|
||||
monkeypatch.setattr(
|
||||
saver, "_load_checkpoint_tuple", patched_load_checkpoint_tuple
|
||||
)
|
||||
|
||||
checkpoint = await saver.aget_tuple(config)
|
||||
assert checkpoint.checkpoint["channel_values"] == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe"])
|
||||
async def test_delta_channel_chain_reconstruction(saver_name: str) -> None:
|
||||
"""AsyncPostgresSaver reconstructs DeltaChannel chain via point-lookup traversal."""
|
||||
pytest.importorskip(
|
||||
"langgraph.channels.delta", reason="langgraph core not installed"
|
||||
)
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langgraph.channels.delta import DeltaChannel
|
||||
from langgraph.graph import START, StateGraph
|
||||
from langgraph.graph.message import _messages_delta_reducer
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
class State(TypedDict):
|
||||
messages: Annotated[list, DeltaChannel(_messages_delta_reducer)]
|
||||
|
||||
def respond(state: State) -> dict:
|
||||
n = len(state["messages"])
|
||||
return {"messages": [AIMessage(content=f"reply-{n}", id=f"ai-{n}")]}
|
||||
|
||||
builder = StateGraph(State)
|
||||
builder.add_node("respond", respond)
|
||||
builder.add_edge(START, "respond")
|
||||
|
||||
async with _saver(saver_name) as saver:
|
||||
graph = builder.compile(checkpointer=saver)
|
||||
config = {"configurable": {"thread_id": "diff-channel-test-1"}}
|
||||
|
||||
await graph.ainvoke({"messages": [HumanMessage(content="hi", id="h1")]}, config)
|
||||
await graph.ainvoke(
|
||||
{"messages": [HumanMessage(content="there", id="h2")]}, config
|
||||
)
|
||||
|
||||
state = await graph.aget_state(config)
|
||||
msgs = state.values["messages"]
|
||||
assert len(msgs) == 4, f"expected 4, got {len(msgs)}: {msgs}"
|
||||
assert msgs[0].content == "hi"
|
||||
assert msgs[1].content == "reply-1"
|
||||
assert msgs[2].content == "there"
|
||||
assert msgs[3].content == "reply-3"
|
||||
@@ -0,0 +1,741 @@
|
||||
# type: ignore
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from langchain_core.embeddings import Embeddings
|
||||
from langgraph.store.base import (
|
||||
GetOp,
|
||||
Item,
|
||||
ListNamespacesOp,
|
||||
PutOp,
|
||||
SearchOp,
|
||||
)
|
||||
from psycopg import AsyncConnection
|
||||
|
||||
from langgraph.checkpoint.postgres import _ainternal
|
||||
from langgraph.store.postgres import AsyncPostgresStore
|
||||
from tests.conftest import (
|
||||
DEFAULT_URI,
|
||||
VECTOR_TYPES,
|
||||
CharacterEmbeddings,
|
||||
)
|
||||
|
||||
TTL_SECONDS = 6
|
||||
TTL_MINUTES = TTL_SECONDS / 60
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", params=["default", "pipe", "pool"])
|
||||
async def store(request) -> AsyncIterator[AsyncPostgresStore]:
|
||||
database = f"test_{uuid.uuid4().hex[:16]}"
|
||||
uri_parts = DEFAULT_URI.split("/")
|
||||
uri_base = "/".join(uri_parts[:-1])
|
||||
query_params = ""
|
||||
if "?" in uri_parts[-1]:
|
||||
db_name, query_params = uri_parts[-1].split("?", 1)
|
||||
query_params = "?" + query_params
|
||||
|
||||
conn_string = f"{uri_base}/{database}{query_params}"
|
||||
admin_conn_string = DEFAULT_URI
|
||||
ttl_config = {
|
||||
"default_ttl": TTL_MINUTES,
|
||||
"refresh_on_read": True,
|
||||
"sweep_interval_minutes": TTL_MINUTES / 2,
|
||||
}
|
||||
async with await AsyncConnection.connect(
|
||||
admin_conn_string, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
async with AsyncPostgresStore.from_conn_string(
|
||||
conn_string, ttl=ttl_config
|
||||
) as store:
|
||||
store.MIGRATIONS = [
|
||||
(
|
||||
mig.replace("ttl_minutes INT;", "ttl_minutes FLOAT;")
|
||||
if isinstance(mig, str)
|
||||
else mig
|
||||
)
|
||||
for mig in store.MIGRATIONS
|
||||
]
|
||||
await store.setup()
|
||||
async with store._cursor() as cur:
|
||||
# drop the migration index
|
||||
await cur.execute("DROP TABLE IF EXISTS store_migrations")
|
||||
await store.setup() # Will fail if migrations aren't idempotent
|
||||
|
||||
if request.param == "pipe":
|
||||
async with AsyncPostgresStore.from_conn_string(
|
||||
conn_string, pipeline=True, ttl=ttl_config
|
||||
) as store:
|
||||
await store.start_ttl_sweeper()
|
||||
yield store
|
||||
await store.stop_ttl_sweeper()
|
||||
elif request.param == "pool":
|
||||
async with AsyncPostgresStore.from_conn_string(
|
||||
conn_string, pool_config={"min_size": 1, "max_size": 10}, ttl=ttl_config
|
||||
) as store:
|
||||
await store.start_ttl_sweeper()
|
||||
yield store
|
||||
await store.stop_ttl_sweeper()
|
||||
else: # default
|
||||
async with AsyncPostgresStore.from_conn_string(
|
||||
conn_string, ttl=ttl_config
|
||||
) as store:
|
||||
await store.start_ttl_sweeper()
|
||||
yield store
|
||||
await store.stop_ttl_sweeper()
|
||||
finally:
|
||||
async with await AsyncConnection.connect(
|
||||
admin_conn_string, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
async def test_no_running_loop(store: AsyncPostgresStore) -> None:
|
||||
with pytest.raises(asyncio.InvalidStateError):
|
||||
store.put(("foo", "bar"), "baz", {"val": "baz"})
|
||||
with pytest.raises(asyncio.InvalidStateError):
|
||||
store.get(("foo", "bar"), "baz")
|
||||
with pytest.raises(asyncio.InvalidStateError):
|
||||
store.delete(("foo", "bar"), "baz")
|
||||
with pytest.raises(asyncio.InvalidStateError):
|
||||
store.search(("foo", "bar"))
|
||||
with pytest.raises(asyncio.InvalidStateError):
|
||||
store.list_namespaces(prefix=("foo",))
|
||||
with pytest.raises(asyncio.InvalidStateError):
|
||||
store.batch([PutOp(namespace=("foo", "bar"), key="baz", value={"val": "baz"})])
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
future = executor.submit(store.put, ("foo", "bar"), "baz", {"val": "baz"})
|
||||
result = await asyncio.wrap_future(future)
|
||||
assert result is None
|
||||
future = executor.submit(store.get, ("foo", "bar"), "baz")
|
||||
result = await asyncio.wrap_future(future)
|
||||
assert result.value == {"val": "baz"}
|
||||
result = await asyncio.wrap_future(
|
||||
executor.submit(store.list_namespaces, prefix=("foo",))
|
||||
)
|
||||
|
||||
|
||||
async def test_large_batches(request: Any, store: AsyncPostgresStore) -> None:
|
||||
N = 100 # less important that we are performant here
|
||||
M = 10
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
futures = []
|
||||
for m in range(M):
|
||||
for i in range(N):
|
||||
futures += [
|
||||
executor.submit(
|
||||
store.put,
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
value={"foo": "bar" + str(i)},
|
||||
),
|
||||
executor.submit(
|
||||
store.get,
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
),
|
||||
executor.submit(
|
||||
store.list_namespaces,
|
||||
prefix=None,
|
||||
max_depth=m + 1,
|
||||
),
|
||||
executor.submit(
|
||||
store.search,
|
||||
("test",),
|
||||
),
|
||||
executor.submit(
|
||||
store.put,
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
value={"foo": "bar" + str(i)},
|
||||
),
|
||||
executor.submit(
|
||||
store.put,
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
None,
|
||||
),
|
||||
]
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(asyncio.wrap_future(future) for future in futures)
|
||||
)
|
||||
assert len(results) == M * N * 6
|
||||
|
||||
|
||||
async def test_large_batches_async(store: AsyncPostgresStore) -> None:
|
||||
N = 1000
|
||||
M = 10
|
||||
coros = []
|
||||
for m in range(M):
|
||||
for i in range(N):
|
||||
coros.append(
|
||||
store.aput(
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
value={"foo": "bar" + str(i)},
|
||||
)
|
||||
)
|
||||
coros.append(
|
||||
store.aget(
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
)
|
||||
)
|
||||
coros.append(
|
||||
store.alist_namespaces(
|
||||
prefix=None,
|
||||
max_depth=m + 1,
|
||||
)
|
||||
)
|
||||
coros.append(
|
||||
store.asearch(
|
||||
("test",),
|
||||
)
|
||||
)
|
||||
coros.append(
|
||||
store.aput(
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
value={"foo": "bar" + str(i)},
|
||||
)
|
||||
)
|
||||
coros.append(
|
||||
store.adelete(
|
||||
("test", "foo", "bar", "baz", str(m % 2)),
|
||||
f"key{i}",
|
||||
)
|
||||
)
|
||||
|
||||
results = await asyncio.gather(*coros)
|
||||
assert len(results) == M * N * 6
|
||||
|
||||
|
||||
async def test_abatch_order(store: AsyncPostgresStore) -> None:
|
||||
# Setup test data
|
||||
await store.aput(("test", "foo"), "key1", {"data": "value1"})
|
||||
await store.aput(("test", "bar"), "key2", {"data": "value2"})
|
||||
|
||||
ops = [
|
||||
GetOp(namespace=("test", "foo"), key="key1"),
|
||||
PutOp(namespace=("test", "bar"), key="key2", value={"data": "value2"}),
|
||||
SearchOp(
|
||||
namespace_prefix=("test",), filter={"data": "value1"}, limit=10, offset=0
|
||||
),
|
||||
ListNamespacesOp(match_conditions=None, max_depth=None, limit=10, offset=0),
|
||||
GetOp(namespace=("test",), key="key3"),
|
||||
]
|
||||
|
||||
results = await store.abatch(ops)
|
||||
assert len(results) == 5
|
||||
assert isinstance(results[0], Item)
|
||||
assert isinstance(results[0].value, dict)
|
||||
assert results[0].value == {"data": "value1"}
|
||||
assert results[0].key == "key1"
|
||||
assert results[1] is None
|
||||
assert isinstance(results[2], list)
|
||||
assert len(results[2]) == 1
|
||||
assert isinstance(results[3], list)
|
||||
assert ("test", "foo") in results[3] and ("test", "bar") in results[3]
|
||||
assert results[4] is None
|
||||
|
||||
ops_reordered = [
|
||||
SearchOp(namespace_prefix=("test",), filter=None, limit=5, offset=0),
|
||||
GetOp(namespace=("test", "bar"), key="key2"),
|
||||
ListNamespacesOp(match_conditions=None, max_depth=None, limit=5, offset=0),
|
||||
PutOp(namespace=("test",), key="key3", value={"data": "value3"}),
|
||||
GetOp(namespace=("test", "foo"), key="key1"),
|
||||
]
|
||||
|
||||
results_reordered = await store.abatch(ops_reordered)
|
||||
assert len(results_reordered) == 5
|
||||
assert isinstance(results_reordered[0], list)
|
||||
assert len(results_reordered[0]) == 2
|
||||
assert isinstance(results_reordered[1], Item)
|
||||
assert results_reordered[1].value == {"data": "value2"}
|
||||
assert results_reordered[1].key == "key2"
|
||||
assert isinstance(results_reordered[2], list)
|
||||
assert ("test", "foo") in results_reordered[2] and (
|
||||
"test",
|
||||
"bar",
|
||||
) in results_reordered[2]
|
||||
assert results_reordered[3] is None
|
||||
assert isinstance(results_reordered[4], Item)
|
||||
assert results_reordered[4].value == {"data": "value1"}
|
||||
assert results_reordered[4].key == "key1"
|
||||
|
||||
|
||||
async def test_batch_get_ops(store: AsyncPostgresStore) -> None:
|
||||
# Setup test data
|
||||
await store.aput(("test",), "key1", {"data": "value1"})
|
||||
await store.aput(("test",), "key2", {"data": "value2"})
|
||||
|
||||
ops = [
|
||||
GetOp(namespace=("test",), key="key1"),
|
||||
GetOp(namespace=("test",), key="key2"),
|
||||
GetOp(namespace=("test",), key="key3"),
|
||||
]
|
||||
|
||||
results = await store.abatch(ops)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0] is not None
|
||||
assert results[1] is not None
|
||||
assert results[2] is None
|
||||
assert results[0].key == "key1"
|
||||
assert results[1].key == "key2"
|
||||
|
||||
|
||||
async def test_batch_put_ops(store: AsyncPostgresStore) -> None:
|
||||
ops = [
|
||||
PutOp(namespace=("test",), key="key1", value={"data": "value1"}),
|
||||
PutOp(namespace=("test",), key="key2", value={"data": "value2"}),
|
||||
PutOp(namespace=("test",), key="key3", value=None),
|
||||
]
|
||||
|
||||
results = await store.abatch(ops)
|
||||
|
||||
assert len(results) == 3
|
||||
assert all(result is None for result in results)
|
||||
|
||||
# Verify the puts worked
|
||||
items = await store.asearch(["test"], limit=10)
|
||||
assert len(items) == 2 # key3 had None value so wasn't stored
|
||||
|
||||
|
||||
async def test_batch_search_ops(store: AsyncPostgresStore) -> None:
|
||||
# Setup test data
|
||||
await store.aput(("test", "foo"), "key1", {"data": "value1"})
|
||||
await store.aput(("test", "bar"), "key2", {"data": "value2"})
|
||||
|
||||
ops = [
|
||||
SearchOp(
|
||||
namespace_prefix=("test",), filter={"data": "value1"}, limit=10, offset=0
|
||||
),
|
||||
SearchOp(namespace_prefix=("test",), filter=None, limit=5, offset=0),
|
||||
]
|
||||
|
||||
results = await store.abatch(ops)
|
||||
|
||||
assert len(results) == 2
|
||||
assert len(results[0]) == 1 # Filtered results
|
||||
assert len(results[1]) == 2 # All results
|
||||
|
||||
|
||||
async def test_batch_list_namespaces_ops(store: AsyncPostgresStore) -> None:
|
||||
# Setup test data
|
||||
await store.aput(("test", "namespace1"), "key1", {"data": "value1"})
|
||||
await store.aput(("test", "namespace2"), "key2", {"data": "value2"})
|
||||
|
||||
ops = [ListNamespacesOp(match_conditions=None, max_depth=None, limit=10, offset=0)]
|
||||
|
||||
results = await store.abatch(ops)
|
||||
|
||||
assert len(results) == 1
|
||||
assert len(results[0]) == 2
|
||||
assert ("test", "namespace1") in results[0]
|
||||
assert ("test", "namespace2") in results[0]
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _create_pool_store() -> AsyncIterator[AsyncPostgresStore]:
|
||||
database = f"test_{uuid.uuid4().hex[:16]}"
|
||||
uri_parts = DEFAULT_URI.split("/")
|
||||
uri_base = "/".join(uri_parts[:-1])
|
||||
query_params = ""
|
||||
if "?" in uri_parts[-1]:
|
||||
_, query_params = uri_parts[-1].split("?", 1)
|
||||
query_params = "?" + query_params
|
||||
|
||||
conn_string = f"{uri_base}/{database}{query_params}"
|
||||
admin_conn_string = DEFAULT_URI
|
||||
async with await AsyncConnection.connect(
|
||||
admin_conn_string, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
async with AsyncPostgresStore.from_conn_string(
|
||||
conn_string, pool_config={"min_size": 1, "max_size": 1}
|
||||
) as store:
|
||||
await store.setup()
|
||||
yield store
|
||||
finally:
|
||||
async with await AsyncConnection.connect(
|
||||
admin_conn_string, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
async def test_abatch_uses_single_pool_checkout(monkeypatch) -> None:
|
||||
async with _create_pool_store() as store:
|
||||
await store.aput(("test",), "key1", {"data": "value1"})
|
||||
|
||||
original_get_connection = _ainternal.get_connection
|
||||
checkout_count = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def counting_get_connection(conn):
|
||||
nonlocal checkout_count
|
||||
checkout_count += 1
|
||||
async with original_get_connection(conn) as checked_out_conn:
|
||||
yield checked_out_conn
|
||||
|
||||
monkeypatch.setattr(_ainternal, "get_connection", counting_get_connection)
|
||||
|
||||
results = await store.abatch([GetOp(namespace=("test",), key="key1")])
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0] is not None
|
||||
assert results[0].value == {"data": "value1"}
|
||||
assert checkout_count == 1
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def _create_vector_store(
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
text_fields: list[str] | None = None,
|
||||
) -> AsyncIterator[AsyncPostgresStore]:
|
||||
"""Create a store with vector search enabled."""
|
||||
|
||||
database = f"test_{uuid.uuid4().hex[:16]}"
|
||||
uri_parts = DEFAULT_URI.split("/")
|
||||
uri_base = "/".join(uri_parts[:-1])
|
||||
query_params = ""
|
||||
if "?" in uri_parts[-1]:
|
||||
db_name, query_params = uri_parts[-1].split("?", 1)
|
||||
query_params = "?" + query_params
|
||||
|
||||
conn_string = f"{uri_base}/{database}{query_params}"
|
||||
admin_conn_string = DEFAULT_URI
|
||||
|
||||
index_config = {
|
||||
"dims": fake_embeddings.dims,
|
||||
"embed": fake_embeddings,
|
||||
"ann_index_config": {
|
||||
"vector_type": vector_type,
|
||||
},
|
||||
"distance_type": distance_type,
|
||||
"fields": text_fields,
|
||||
}
|
||||
|
||||
async with await AsyncConnection.connect(
|
||||
admin_conn_string, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
async with AsyncPostgresStore.from_conn_string(
|
||||
conn_string,
|
||||
index=index_config,
|
||||
) as store:
|
||||
await store.setup()
|
||||
yield store
|
||||
finally:
|
||||
async with await AsyncConnection.connect(
|
||||
admin_conn_string, autocommit=True
|
||||
) as conn:
|
||||
await conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=[
|
||||
(vector_type, distance_type)
|
||||
for vector_type in VECTOR_TYPES
|
||||
for distance_type in (
|
||||
["hamming"] if vector_type == "bit" else ["l2", "inner_product", "cosine"]
|
||||
)
|
||||
],
|
||||
ids=lambda p: f"{p[0]}_{p[1]}",
|
||||
)
|
||||
async def vector_store(
|
||||
request,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
) -> AsyncIterator[AsyncPostgresStore]:
|
||||
"""Create a store with vector search enabled."""
|
||||
vector_type, distance_type = request.param
|
||||
async with _create_vector_store(
|
||||
vector_type, distance_type, fake_embeddings
|
||||
) as store:
|
||||
yield store
|
||||
|
||||
|
||||
async def test_vector_store_initialization(
|
||||
vector_store: AsyncPostgresStore, fake_embeddings: CharacterEmbeddings
|
||||
) -> None:
|
||||
"""Test store initialization with embedding config."""
|
||||
assert vector_store.index_config is not None
|
||||
assert vector_store.index_config["dims"] == fake_embeddings.dims
|
||||
if isinstance(vector_store.index_config["embed"], Embeddings):
|
||||
assert vector_store.index_config["embed"] == fake_embeddings
|
||||
|
||||
|
||||
async def test_vector_insert_with_auto_embedding(
|
||||
vector_store: AsyncPostgresStore,
|
||||
) -> None:
|
||||
"""Test inserting items that get auto-embedded."""
|
||||
docs = [
|
||||
("doc1", {"text": "short text"}),
|
||||
("doc2", {"text": "longer text document"}),
|
||||
("doc3", {"text": "longest text document here"}),
|
||||
("doc4", {"description": "text in description field"}),
|
||||
("doc5", {"content": "text in content field"}),
|
||||
("doc6", {"body": "text in body field"}),
|
||||
]
|
||||
|
||||
for key, value in docs:
|
||||
await vector_store.aput(("test",), key, value)
|
||||
|
||||
results = await vector_store.asearch(("test",), query="long text")
|
||||
assert len(results) > 0
|
||||
|
||||
doc_order = [r.key for r in results]
|
||||
assert "doc2" in doc_order
|
||||
assert "doc3" in doc_order
|
||||
|
||||
|
||||
async def test_vector_update_with_embedding(vector_store: AsyncPostgresStore) -> None:
|
||||
"""Test that updating items properly updates their embeddings."""
|
||||
await vector_store.aput(("test",), "doc1", {"text": "zany zebra Xerxes"})
|
||||
await vector_store.aput(("test",), "doc2", {"text": "something about dogs"})
|
||||
await vector_store.aput(("test",), "doc3", {"text": "text about birds"})
|
||||
|
||||
results_initial = await vector_store.asearch(("test",), query="Zany Xerxes")
|
||||
assert len(results_initial) > 0
|
||||
assert results_initial[0].key == "doc1"
|
||||
initial_score = results_initial[0].score
|
||||
|
||||
await vector_store.aput(("test",), "doc1", {"text": "new text about dogs"})
|
||||
|
||||
results_after = await vector_store.asearch(("test",), query="Zany Xerxes")
|
||||
after_score = next((r.score for r in results_after if r.key == "doc1"), 0.0)
|
||||
assert after_score < initial_score
|
||||
|
||||
results_new = await vector_store.asearch(("test",), query="new text about dogs")
|
||||
for r in results_new:
|
||||
if r.key == "doc1":
|
||||
assert r.score > after_score
|
||||
|
||||
# Don't index this one
|
||||
await vector_store.aput(
|
||||
("test",), "doc4", {"text": "new text about dogs"}, index=False
|
||||
)
|
||||
results_new = await vector_store.asearch(
|
||||
("test",), query="new text about dogs", limit=3
|
||||
)
|
||||
assert not any(r.key == "doc4" for r in results_new)
|
||||
|
||||
|
||||
async def test_vector_search_with_filters(vector_store: AsyncPostgresStore) -> None:
|
||||
"""Test combining vector search with filters."""
|
||||
docs = [
|
||||
("doc1", {"text": "red apple", "color": "red", "score": 4.5}),
|
||||
("doc2", {"text": "red car", "color": "red", "score": 3.0}),
|
||||
("doc3", {"text": "green apple", "color": "green", "score": 4.0}),
|
||||
("doc4", {"text": "blue car", "color": "blue", "score": 3.5}),
|
||||
]
|
||||
|
||||
for key, value in docs:
|
||||
await vector_store.aput(("test",), key, value)
|
||||
|
||||
results = await vector_store.asearch(
|
||||
("test",), query="apple", filter={"color": "red"}
|
||||
)
|
||||
assert len(results) == 2
|
||||
assert results[0].key == "doc1"
|
||||
|
||||
results = await vector_store.asearch(
|
||||
("test",), query="car", filter={"color": "red"}
|
||||
)
|
||||
assert len(results) == 2
|
||||
assert results[0].key == "doc2"
|
||||
|
||||
results = await vector_store.asearch(
|
||||
("test",), query="bbbbluuu", filter={"score": {"$gt": 3.2}}
|
||||
)
|
||||
assert len(results) == 3
|
||||
assert results[0].key == "doc4"
|
||||
|
||||
results = await vector_store.asearch(
|
||||
("test",), query="apple", filter={"score": {"$gte": 4.0}, "color": "green"}
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert results[0].key == "doc3"
|
||||
|
||||
|
||||
async def test_vector_search_pagination(vector_store: AsyncPostgresStore) -> None:
|
||||
"""Test pagination with vector search."""
|
||||
for i in range(5):
|
||||
await vector_store.aput(
|
||||
("test",), f"doc{i}", {"text": f"test document number {i}"}
|
||||
)
|
||||
|
||||
results_page1 = await vector_store.asearch(("test",), query="test", limit=2)
|
||||
results_page2 = await vector_store.asearch(
|
||||
("test",), query="test", limit=2, offset=2
|
||||
)
|
||||
|
||||
assert len(results_page1) == 2
|
||||
assert len(results_page2) == 2
|
||||
assert results_page1[0].key != results_page2[0].key
|
||||
|
||||
all_results = await vector_store.asearch(("test",), query="test", limit=10)
|
||||
assert len(all_results) == 5
|
||||
|
||||
|
||||
async def test_vector_search_edge_cases(vector_store: AsyncPostgresStore) -> None:
|
||||
"""Test edge cases in vector search."""
|
||||
await vector_store.aput(("test",), "doc1", {"text": "test document"})
|
||||
|
||||
perfect_match = await vector_store.asearch(("test",), query="text test document")
|
||||
perfect_score = perfect_match[0].score
|
||||
|
||||
results = await vector_store.asearch(("test",), query="")
|
||||
assert len(results) == 1
|
||||
assert results[0].score is None
|
||||
|
||||
results = await vector_store.asearch(("test",), query=None)
|
||||
assert len(results) == 1
|
||||
assert results[0].score is None
|
||||
|
||||
long_query = "foo " * 100
|
||||
results = await vector_store.asearch(("test",), query=long_query)
|
||||
assert len(results) == 1
|
||||
assert results[0].score < perfect_score
|
||||
|
||||
special_query = "test!@#$%^&*()"
|
||||
results = await vector_store.asearch(("test",), query=special_query)
|
||||
assert len(results) == 1
|
||||
assert results[0].score < perfect_score
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
*itertools.product(["vector", "halfvec"], ["cosine", "inner_product", "l2"]),
|
||||
],
|
||||
)
|
||||
async def test_embed_with_path(
|
||||
request: Any,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test vector search with specific text fields in Postgres store."""
|
||||
async with _create_vector_store(
|
||||
vector_type,
|
||||
distance_type,
|
||||
fake_embeddings,
|
||||
text_fields=["key0", "key1", "key3"],
|
||||
) as store:
|
||||
# This will have 2 vectors representing it
|
||||
doc1 = {
|
||||
# Omit key0 - check it doesn't raise an error
|
||||
"key1": "xxx",
|
||||
"key2": "yyy",
|
||||
"key3": "zzz",
|
||||
}
|
||||
# This will have 3 vectors representing it
|
||||
doc2 = {
|
||||
"key0": "uuu",
|
||||
"key1": "vvv",
|
||||
"key2": "www",
|
||||
"key3": "xxx",
|
||||
}
|
||||
await store.aput(("test",), "doc1", doc1)
|
||||
await store.aput(("test",), "doc2", doc2)
|
||||
|
||||
# doc2.key3 and doc1.key1 both would have the highest score
|
||||
results = await store.asearch(("test",), query="xxx")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
ascore = results[0].score
|
||||
bscore = results[1].score
|
||||
assert ascore == pytest.approx(bscore, abs=1e-3)
|
||||
|
||||
results = await store.asearch(("test",), query="uuu")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
assert results[0].key == "doc2"
|
||||
assert results[0].score > results[1].score
|
||||
assert ascore == pytest.approx(results[0].score, abs=1e-3)
|
||||
|
||||
# Un-indexed - will have low results for both. Not zero (because we're projecting)
|
||||
# but less than the above.
|
||||
results = await store.asearch(("test",), query="www")
|
||||
assert len(results) == 2
|
||||
assert results[0].score < ascore
|
||||
assert results[1].score < ascore
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
*itertools.product(["vector", "halfvec"], ["cosine", "inner_product", "l2"]),
|
||||
],
|
||||
)
|
||||
async def test_search_sorting(
|
||||
request: Any,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test operation-level field configuration for vector search."""
|
||||
async with _create_vector_store(
|
||||
vector_type,
|
||||
distance_type,
|
||||
fake_embeddings,
|
||||
text_fields=["key1"], # Default fields that won't match our test data
|
||||
) as store:
|
||||
amatch = {
|
||||
"key1": "mmm",
|
||||
}
|
||||
|
||||
await store.aput(("test", "M"), "M", amatch)
|
||||
N = 100
|
||||
for i in range(N):
|
||||
await store.aput(("test", "A"), f"A{i}", {"key1": "no"})
|
||||
for i in range(N):
|
||||
await store.aput(("test", "Z"), f"Z{i}", {"key1": "no"})
|
||||
|
||||
results = await store.asearch(("test",), query="mmm", limit=10)
|
||||
assert len(results) == 10
|
||||
assert len(set(r.key for r in results)) == 10
|
||||
assert results[0].key == "M"
|
||||
assert results[0].score > results[1].score
|
||||
|
||||
|
||||
async def test_store_ttl(store):
|
||||
# Assumes a TTL of 1 minute = 60 seconds
|
||||
ns = ("foo",)
|
||||
await store.start_ttl_sweeper()
|
||||
await store.aput(
|
||||
ns,
|
||||
key="item1",
|
||||
value={"foo": "bar"},
|
||||
ttl=TTL_MINUTES, # type: ignore
|
||||
)
|
||||
await asyncio.sleep(TTL_SECONDS - 2)
|
||||
res = await store.aget(ns, key="item1", refresh_ttl=True)
|
||||
assert res is not None
|
||||
await asyncio.sleep(TTL_SECONDS - 2)
|
||||
results = await store.asearch(ns, query="foo", refresh_ttl=True)
|
||||
assert len(results) == 1
|
||||
await asyncio.sleep(TTL_SECONDS - 2)
|
||||
res = await store.aget(ns, key="item1", refresh_ttl=False)
|
||||
assert res is not None
|
||||
await asyncio.sleep(TTL_SECONDS - 1)
|
||||
# Now has been (TTL_SECONDS-2)*2 > TTL_SECONDS + TTL_SECONDS/2
|
||||
results = await store.asearch(ns, query="bar", refresh_ttl=False)
|
||||
assert len(results) == 0
|
||||
@@ -0,0 +1,901 @@
|
||||
# type: ignore
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from langchain_core.embeddings import Embeddings
|
||||
from langgraph.store.base import (
|
||||
GetOp,
|
||||
Item,
|
||||
ListNamespacesOp,
|
||||
MatchCondition,
|
||||
PutOp,
|
||||
SearchOp,
|
||||
)
|
||||
from psycopg import Connection
|
||||
|
||||
from langgraph.store.postgres import PostgresStore
|
||||
from tests.conftest import (
|
||||
DEFAULT_URI,
|
||||
VECTOR_TYPES,
|
||||
CharacterEmbeddings,
|
||||
)
|
||||
|
||||
TTL_SECONDS = 6
|
||||
TTL_MINUTES = TTL_SECONDS / 60
|
||||
|
||||
|
||||
@pytest.fixture(scope="function", params=["default", "pipe", "pool"])
|
||||
def store(request) -> PostgresStore:
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
uri_parts = DEFAULT_URI.split("/")
|
||||
uri_base = "/".join(uri_parts[:-1])
|
||||
query_params = ""
|
||||
if "?" in uri_parts[-1]:
|
||||
_, query_params = uri_parts[-1].split("?", 1)
|
||||
query_params = "?" + query_params
|
||||
|
||||
conn_string = f"{uri_base}/{database}{query_params}"
|
||||
admin_conn_string = DEFAULT_URI
|
||||
ttl_config = {
|
||||
"default_ttl": TTL_MINUTES,
|
||||
"refresh_on_read": True,
|
||||
"sweep_interval_minutes": TTL_MINUTES / 2,
|
||||
}
|
||||
with Connection.connect(admin_conn_string, autocommit=True) as conn:
|
||||
conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
with PostgresStore.from_conn_string(conn_string, ttl=ttl_config) as store:
|
||||
store.MIGRATIONS = [
|
||||
(
|
||||
mig.replace("ttl_minutes INT;", "ttl_minutes FLOAT;")
|
||||
if isinstance(mig, str)
|
||||
else mig
|
||||
)
|
||||
for mig in store.MIGRATIONS
|
||||
]
|
||||
store.setup()
|
||||
|
||||
if request.param == "pipe":
|
||||
with PostgresStore.from_conn_string(
|
||||
conn_string,
|
||||
pipeline=True,
|
||||
ttl=ttl_config,
|
||||
) as store:
|
||||
store.start_ttl_sweeper()
|
||||
yield store
|
||||
|
||||
store.stop_ttl_sweeper()
|
||||
elif request.param == "pool":
|
||||
with PostgresStore.from_conn_string(
|
||||
conn_string,
|
||||
pool_config={"min_size": 1, "max_size": 10},
|
||||
ttl=ttl_config,
|
||||
) as store:
|
||||
store.start_ttl_sweeper()
|
||||
yield store
|
||||
|
||||
store.stop_ttl_sweeper()
|
||||
else: # default
|
||||
with PostgresStore.from_conn_string(conn_string, ttl=ttl_config) as store:
|
||||
store.start_ttl_sweeper()
|
||||
yield store
|
||||
|
||||
store.stop_ttl_sweeper()
|
||||
finally:
|
||||
with Connection.connect(admin_conn_string, autocommit=True) as conn:
|
||||
conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
def test_batch_order(store: PostgresStore) -> None:
|
||||
# Setup test data
|
||||
store.put(("test", "foo"), "key1", {"data": "value1"})
|
||||
store.put(("test", "bar"), "key2", {"data": "value2"})
|
||||
|
||||
ops = [
|
||||
GetOp(namespace=("test", "foo"), key="key1"),
|
||||
PutOp(namespace=("test", "bar"), key="key2", value={"data": "value2"}),
|
||||
SearchOp(
|
||||
namespace_prefix=("test",), filter={"data": "value1"}, limit=10, offset=0
|
||||
),
|
||||
ListNamespacesOp(match_conditions=None, max_depth=None, limit=10, offset=0),
|
||||
GetOp(namespace=("test",), key="key3"),
|
||||
]
|
||||
|
||||
results = store.batch(ops)
|
||||
assert len(results) == 5
|
||||
assert isinstance(results[0], Item)
|
||||
assert isinstance(results[0].value, dict)
|
||||
assert results[0].value == {"data": "value1"}
|
||||
assert results[0].key == "key1"
|
||||
assert results[1] is None # Put operation returns None
|
||||
assert isinstance(results[2], list)
|
||||
assert len(results[2]) == 1
|
||||
assert isinstance(results[3], list)
|
||||
assert len(results[3]) > 0 # Should contain at least our test namespaces
|
||||
assert results[4] is None # Non-existent key returns None
|
||||
|
||||
# Test reordered operations
|
||||
ops_reordered = [
|
||||
SearchOp(namespace_prefix=("test",), filter=None, limit=5, offset=0),
|
||||
GetOp(namespace=("test", "bar"), key="key2"),
|
||||
ListNamespacesOp(match_conditions=None, max_depth=None, limit=5, offset=0),
|
||||
PutOp(namespace=("test",), key="key3", value={"data": "value3"}),
|
||||
GetOp(namespace=("test", "foo"), key="key1"),
|
||||
]
|
||||
|
||||
results_reordered = store.batch(ops_reordered)
|
||||
assert len(results_reordered) == 5
|
||||
assert isinstance(results_reordered[0], list)
|
||||
assert len(results_reordered[0]) >= 2 # Should find at least our two test items
|
||||
assert isinstance(results_reordered[1], Item)
|
||||
assert results_reordered[1].value == {"data": "value2"}
|
||||
assert results_reordered[1].key == "key2"
|
||||
assert isinstance(results_reordered[2], list)
|
||||
assert len(results_reordered[2]) > 0
|
||||
assert results_reordered[3] is None # Put operation returns None
|
||||
assert isinstance(results_reordered[4], Item)
|
||||
assert results_reordered[4].value == {"data": "value1"}
|
||||
assert results_reordered[4].key == "key1"
|
||||
|
||||
|
||||
def test_batch_get_ops(store: PostgresStore) -> None:
|
||||
# Setup test data
|
||||
store.put(("test",), "key1", {"data": "value1"})
|
||||
store.put(("test",), "key2", {"data": "value2"})
|
||||
|
||||
ops = [
|
||||
GetOp(namespace=("test",), key="key1"),
|
||||
GetOp(namespace=("test",), key="key2"),
|
||||
GetOp(namespace=("test",), key="key3"), # Non-existent key
|
||||
]
|
||||
|
||||
results = store.batch(ops)
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0] is not None
|
||||
assert results[1] is not None
|
||||
assert results[2] is None
|
||||
assert results[0].key == "key1"
|
||||
assert results[1].key == "key2"
|
||||
|
||||
|
||||
def test_batch_put_ops(store: PostgresStore) -> None:
|
||||
ops = [
|
||||
PutOp(namespace=("test",), key="key1", value={"data": "value1"}),
|
||||
PutOp(namespace=("test",), key="key2", value={"data": "value2"}),
|
||||
PutOp(namespace=("test",), key="key3", value=None), # Delete operation
|
||||
]
|
||||
|
||||
results = store.batch(ops)
|
||||
assert len(results) == 3
|
||||
assert all(result is None for result in results)
|
||||
|
||||
# Verify the puts worked
|
||||
item1 = store.get(("test",), "key1")
|
||||
item2 = store.get(("test",), "key2")
|
||||
item3 = store.get(("test",), "key3")
|
||||
|
||||
assert item1 and item1.value == {"data": "value1"}
|
||||
assert item2 and item2.value == {"data": "value2"}
|
||||
assert item3 is None
|
||||
|
||||
|
||||
def test_batch_search_ops(store: PostgresStore) -> None:
|
||||
# Setup test data
|
||||
test_data = [
|
||||
(("test", "foo"), "key1", {"data": "value1", "tag": "a"}),
|
||||
(("test", "bar"), "key2", {"data": "value2", "tag": "a"}),
|
||||
(("test", "baz"), "key3", {"data": "value3", "tag": "b"}),
|
||||
]
|
||||
for namespace, key, value in test_data:
|
||||
store.put(namespace, key, value)
|
||||
|
||||
ops = [
|
||||
SearchOp(namespace_prefix=("test",), filter={"tag": "a"}, limit=10, offset=0),
|
||||
SearchOp(namespace_prefix=("test",), filter=None, limit=2, offset=0),
|
||||
SearchOp(namespace_prefix=("test", "foo"), filter=None, limit=10, offset=0),
|
||||
]
|
||||
|
||||
results = store.batch(ops)
|
||||
assert len(results) == 3
|
||||
|
||||
# First search should find items with tag "a"
|
||||
assert len(results[0]) == 2
|
||||
assert all(item.value["tag"] == "a" for item in results[0])
|
||||
|
||||
# Second search should return first 2 items
|
||||
assert len(results[1]) == 2
|
||||
|
||||
# Third search should only find items in test/foo namespace
|
||||
assert len(results[2]) == 1
|
||||
assert results[2][0].namespace == ("test", "foo")
|
||||
|
||||
|
||||
def test_batch_list_namespaces_ops(store: PostgresStore) -> None:
|
||||
# Setup test data with various namespaces
|
||||
test_data = [
|
||||
(("test", "documents", "public"), "doc1", {"content": "public doc"}),
|
||||
(("test", "documents", "private"), "doc2", {"content": "private doc"}),
|
||||
(("test", "images", "public"), "img1", {"content": "public image"}),
|
||||
(("prod", "documents", "public"), "doc3", {"content": "prod doc"}),
|
||||
]
|
||||
for namespace, key, value in test_data:
|
||||
store.put(namespace, key, value)
|
||||
|
||||
ops = [
|
||||
ListNamespacesOp(match_conditions=None, max_depth=None, limit=10, offset=0),
|
||||
ListNamespacesOp(match_conditions=None, max_depth=2, limit=10, offset=0),
|
||||
ListNamespacesOp(
|
||||
match_conditions=[MatchCondition("suffix", "public")],
|
||||
max_depth=None,
|
||||
limit=10,
|
||||
offset=0,
|
||||
),
|
||||
]
|
||||
|
||||
results = store.batch(ops)
|
||||
assert len(results) == 3
|
||||
|
||||
# First operation should list all namespaces
|
||||
assert len(results[0]) == len(test_data)
|
||||
|
||||
# Second operation should only return namespaces up to depth 2
|
||||
assert all(len(ns) <= 2 for ns in results[1])
|
||||
|
||||
# Third operation should only return namespaces ending with "public"
|
||||
assert all(ns[-1] == "public" for ns in results[2])
|
||||
|
||||
|
||||
def test_basic_store_ops(store) -> None:
|
||||
namespace = ("test", "documents")
|
||||
item_id = "doc1"
|
||||
item_value = {"title": "Test Document", "content": "Hello, World!"}
|
||||
|
||||
store.put(namespace, item_id, item_value)
|
||||
item = store.get(namespace, item_id)
|
||||
|
||||
assert item
|
||||
assert item.namespace == namespace
|
||||
assert item.key == item_id
|
||||
assert item.value == item_value
|
||||
|
||||
# Test update
|
||||
updated_value = {"title": "Updated Document", "content": "Hello, Updated!"}
|
||||
store.put(namespace, item_id, updated_value)
|
||||
updated_item = store.get(namespace, item_id)
|
||||
|
||||
assert updated_item.value == updated_value
|
||||
assert updated_item.updated_at > item.updated_at
|
||||
|
||||
# Test get from non-existent namespace
|
||||
different_namespace = ("test", "other_documents")
|
||||
item_in_different_namespace = store.get(different_namespace, item_id)
|
||||
assert item_in_different_namespace is None
|
||||
|
||||
# Test delete
|
||||
store.delete(namespace, item_id)
|
||||
deleted_item = store.get(namespace, item_id)
|
||||
assert deleted_item is None
|
||||
|
||||
|
||||
def test_list_namespaces(store) -> None:
|
||||
# Create test data with various namespaces
|
||||
test_namespaces = [
|
||||
("test", "documents", "public"),
|
||||
("test", "documents", "private"),
|
||||
("test", "images", "public"),
|
||||
("test", "images", "private"),
|
||||
("prod", "documents", "public"),
|
||||
("prod", "documents", "private"),
|
||||
]
|
||||
|
||||
# Insert test data
|
||||
for namespace in test_namespaces:
|
||||
store.put(namespace, "dummy", {"content": "dummy"})
|
||||
|
||||
# Test listing with various filters
|
||||
all_namespaces = store.list_namespaces()
|
||||
assert len(all_namespaces) == len(test_namespaces)
|
||||
|
||||
# Test prefix filtering
|
||||
test_prefix_namespaces = store.list_namespaces(prefix=["test"])
|
||||
assert len(test_prefix_namespaces) == 4
|
||||
assert all(ns[0] == "test" for ns in test_prefix_namespaces)
|
||||
|
||||
# Test suffix filtering
|
||||
public_namespaces = store.list_namespaces(suffix=["public"])
|
||||
assert len(public_namespaces) == 3
|
||||
assert all(ns[-1] == "public" for ns in public_namespaces)
|
||||
|
||||
# Test max depth
|
||||
depth_2_namespaces = store.list_namespaces(max_depth=2)
|
||||
assert all(len(ns) <= 2 for ns in depth_2_namespaces)
|
||||
|
||||
# Test pagination
|
||||
paginated_namespaces = store.list_namespaces(limit=3)
|
||||
assert len(paginated_namespaces) == 3
|
||||
|
||||
# Cleanup
|
||||
for namespace in test_namespaces:
|
||||
store.delete(namespace, "dummy")
|
||||
|
||||
|
||||
def test_search(store) -> None:
|
||||
# Create test data
|
||||
test_data = [
|
||||
(
|
||||
("test", "docs"),
|
||||
"doc1",
|
||||
{"title": "First Doc", "author": "Alice", "tags": ["important"]},
|
||||
),
|
||||
(
|
||||
("test", "docs"),
|
||||
"doc2",
|
||||
{"title": "Second Doc", "author": "Bob", "tags": ["draft"]},
|
||||
),
|
||||
(
|
||||
("test", "images"),
|
||||
"img1",
|
||||
{"title": "Image 1", "author": "Alice", "tags": ["final"]},
|
||||
),
|
||||
]
|
||||
|
||||
for namespace, key, value in test_data:
|
||||
store.put(namespace, key, value)
|
||||
|
||||
# Test basic search
|
||||
all_items = store.search(["test"])
|
||||
assert len(all_items) == 3
|
||||
|
||||
# Test namespace filtering
|
||||
docs_items = store.search(["test", "docs"])
|
||||
assert len(docs_items) == 2
|
||||
assert all(item.namespace == ("test", "docs") for item in docs_items)
|
||||
|
||||
# Test value filtering
|
||||
alice_items = store.search(["test"], filter={"author": "Alice"})
|
||||
assert len(alice_items) == 2
|
||||
assert all(item.value["author"] == "Alice" for item in alice_items)
|
||||
|
||||
# Test pagination
|
||||
paginated_items = store.search(["test"], limit=2)
|
||||
assert len(paginated_items) == 2
|
||||
|
||||
offset_items = store.search(["test"], offset=2)
|
||||
assert len(offset_items) == 1
|
||||
|
||||
# Cleanup
|
||||
for namespace, key, _ in test_data:
|
||||
store.delete(namespace, key)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _create_vector_store(
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
fake_embeddings: Embeddings,
|
||||
text_fields: list[str] | None = None,
|
||||
enable_ttl: bool = True,
|
||||
) -> PostgresStore:
|
||||
"""Create a store with vector search enabled."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
uri_parts = DEFAULT_URI.split("/")
|
||||
uri_base = "/".join(uri_parts[:-1])
|
||||
query_params = ""
|
||||
if "?" in uri_parts[-1]:
|
||||
db_name, query_params = uri_parts[-1].split("?", 1)
|
||||
query_params = "?" + query_params
|
||||
|
||||
conn_string = f"{uri_base}/{database}{query_params}"
|
||||
admin_conn_string = DEFAULT_URI
|
||||
|
||||
index_config = {
|
||||
"dims": fake_embeddings.dims,
|
||||
"embed": fake_embeddings,
|
||||
"ann_index_config": {
|
||||
"vector_type": vector_type,
|
||||
},
|
||||
"distance_type": distance_type,
|
||||
"fields": text_fields,
|
||||
}
|
||||
|
||||
with Connection.connect(admin_conn_string, autocommit=True) as conn:
|
||||
conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
with PostgresStore.from_conn_string(
|
||||
conn_string,
|
||||
index=index_config,
|
||||
ttl={"default_ttl": 2, "refresh_on_read": True} if enable_ttl else None,
|
||||
) as store:
|
||||
store.setup()
|
||||
with store._cursor() as cur:
|
||||
# drop the migration index
|
||||
cur.execute("DROP TABLE IF EXISTS store_migrations")
|
||||
store.setup() # Will fail if migrations aren't idempotent
|
||||
yield store
|
||||
finally:
|
||||
with Connection.connect(admin_conn_string, autocommit=True) as conn:
|
||||
conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
_vector_params = [
|
||||
(vector_type, distance_type, True)
|
||||
for vector_type in VECTOR_TYPES
|
||||
for distance_type in (
|
||||
["hamming"] if vector_type == "bit" else ["l2", "inner_product", "cosine"]
|
||||
)
|
||||
]
|
||||
_vector_params += [(*_vector_params[-1][:2], False)]
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
scope="function",
|
||||
params=_vector_params,
|
||||
ids=lambda p: f"{p[0]}_{p[1]}",
|
||||
)
|
||||
def vector_store(
|
||||
request,
|
||||
fake_embeddings: Embeddings,
|
||||
) -> PostgresStore:
|
||||
"""Create a store with vector search enabled."""
|
||||
vector_type, distance_type, enable_ttl = request.param
|
||||
with _create_vector_store(
|
||||
vector_type, distance_type, fake_embeddings, enable_ttl=enable_ttl
|
||||
) as store:
|
||||
yield store
|
||||
|
||||
|
||||
def test_vector_store_initialization(
|
||||
vector_store: PostgresStore, fake_embeddings: CharacterEmbeddings
|
||||
) -> None:
|
||||
"""Test store initialization with embedding config."""
|
||||
# Store should be initialized with embedding config
|
||||
assert vector_store.index_config is not None
|
||||
assert vector_store.index_config["dims"] == fake_embeddings.dims
|
||||
assert vector_store.index_config["embed"] == fake_embeddings
|
||||
|
||||
|
||||
def test_vector_insert_with_auto_embedding(vector_store: PostgresStore) -> None:
|
||||
"""Test inserting items that get auto-embedded."""
|
||||
docs = [
|
||||
("doc1", {"text": "short text"}),
|
||||
("doc2", {"text": "longer text document"}),
|
||||
("doc3", {"text": "longest text document here"}),
|
||||
("doc4", {"description": "text in description field"}),
|
||||
("doc5", {"content": "text in content field"}),
|
||||
("doc6", {"body": "text in body field"}),
|
||||
]
|
||||
|
||||
for key, value in docs:
|
||||
vector_store.put(("test",), key, value)
|
||||
|
||||
results = vector_store.search(("test",), query="long text")
|
||||
assert len(results) > 0
|
||||
|
||||
doc_order = [r.key for r in results]
|
||||
assert "doc2" in doc_order
|
||||
assert "doc3" in doc_order
|
||||
|
||||
|
||||
def test_vector_update_with_embedding(vector_store: PostgresStore) -> None:
|
||||
"""Test that updating items properly updates their embeddings."""
|
||||
vector_store.put(("test",), "doc1", {"text": "zany zebra Xerxes"})
|
||||
vector_store.put(("test",), "doc2", {"text": "something about dogs"})
|
||||
vector_store.put(("test",), "doc3", {"text": "text about birds"})
|
||||
|
||||
results_initial = vector_store.search(("test",), query="Zany Xerxes")
|
||||
assert len(results_initial) > 0
|
||||
assert results_initial[0].key == "doc1"
|
||||
initial_score = results_initial[0].score
|
||||
|
||||
vector_store.put(("test",), "doc1", {"text": "new text about dogs"})
|
||||
|
||||
results_after = vector_store.search(("test",), query="Zany Xerxes")
|
||||
after_score = next((r.score for r in results_after if r.key == "doc1"), 0.0)
|
||||
assert after_score < initial_score
|
||||
|
||||
results_new = vector_store.search(("test",), query="new text about dogs")
|
||||
for r in results_new:
|
||||
if r.key == "doc1":
|
||||
assert r.score > after_score
|
||||
|
||||
# Don't index this one
|
||||
vector_store.put(("test",), "doc4", {"text": "new text about dogs"}, index=False)
|
||||
results_new = vector_store.search(("test",), query="new text about dogs", limit=3)
|
||||
assert not any(r.key == "doc4" for r in results_new)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("refresh_ttl", [True, False])
|
||||
def test_vector_search_with_filters(
|
||||
vector_store: PostgresStore, refresh_ttl: bool
|
||||
) -> None:
|
||||
"""Test combining vector search with filters."""
|
||||
# Insert test documents
|
||||
docs = [
|
||||
("doc1", {"text": "red apple", "color": "red", "score": 4.5}),
|
||||
("doc2", {"text": "red car", "color": "red", "score": 3.0}),
|
||||
("doc3", {"text": "green apple", "color": "green", "score": 4.0}),
|
||||
("doc4", {"text": "blue car", "color": "blue", "score": 3.5}),
|
||||
]
|
||||
|
||||
for key, value in docs:
|
||||
vector_store.put(("test",), key, value)
|
||||
|
||||
results = vector_store.search(
|
||||
("test",), query="apple", filter={"color": "red"}, refresh_ttl=refresh_ttl
|
||||
)
|
||||
assert len(results) == 2
|
||||
assert results[0].key == "doc1"
|
||||
|
||||
results = vector_store.search(
|
||||
("test",), query="car", filter={"color": "red"}, refresh_ttl=refresh_ttl
|
||||
)
|
||||
assert len(results) == 2
|
||||
assert results[0].key == "doc2"
|
||||
|
||||
results = vector_store.search(
|
||||
("test",),
|
||||
query="bbbbluuu",
|
||||
filter={"score": {"$gt": 3.2}},
|
||||
refresh_ttl=refresh_ttl,
|
||||
)
|
||||
assert len(results) == 3
|
||||
assert results[0].key == "doc4"
|
||||
|
||||
# Multiple filters
|
||||
results = vector_store.search(
|
||||
("test",), query="apple", filter={"score": {"$gte": 4.0}, "color": "green"}
|
||||
)
|
||||
assert len(results) == 1
|
||||
assert results[0].key == "doc3"
|
||||
|
||||
|
||||
def test_vector_search_pagination(vector_store: PostgresStore) -> None:
|
||||
"""Test pagination with vector search."""
|
||||
# Insert multiple similar documents
|
||||
for i in range(5):
|
||||
vector_store.put(("test",), f"doc{i}", {"text": f"test document number {i}"})
|
||||
|
||||
# Test with different page sizes
|
||||
results_page1 = vector_store.search(("test",), query="test", limit=2)
|
||||
results_page2 = vector_store.search(("test",), query="test", limit=2, offset=2)
|
||||
|
||||
assert len(results_page1) == 2
|
||||
assert len(results_page2) == 2
|
||||
assert results_page1[0].key != results_page2[0].key
|
||||
|
||||
# Get all results
|
||||
all_results = vector_store.search(("test",), query="test", limit=10)
|
||||
assert len(all_results) == 5
|
||||
|
||||
|
||||
def test_vector_search_edge_cases(vector_store: PostgresStore) -> None:
|
||||
"""Test edge cases in vector search."""
|
||||
vector_store.put(("test",), "doc1", {"text": "test document"})
|
||||
|
||||
results = vector_store.search(("test",), query="")
|
||||
assert len(results) == 1
|
||||
|
||||
results = vector_store.search(("test",), query=None)
|
||||
assert len(results) == 1
|
||||
|
||||
long_query = "test " * 100
|
||||
results = vector_store.search(("test",), query=long_query)
|
||||
assert len(results) == 1
|
||||
|
||||
special_query = "test!@#$%^&*()"
|
||||
results = vector_store.search(("test",), query=special_query)
|
||||
assert len(results) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
("vector", "cosine"),
|
||||
("vector", "inner_product"),
|
||||
("halfvec", "cosine"),
|
||||
("halfvec", "inner_product"),
|
||||
],
|
||||
)
|
||||
def test_embed_with_path_sync(
|
||||
request: Any,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test vector search with specific text fields in Postgres store."""
|
||||
with _create_vector_store(
|
||||
vector_type,
|
||||
distance_type,
|
||||
fake_embeddings,
|
||||
text_fields=["key0", "key1", "key3"],
|
||||
) as store:
|
||||
# This will have 2 vectors representing it
|
||||
doc1 = {
|
||||
# Omit key0 - check it doesn't raise an error
|
||||
"key1": "xxx",
|
||||
"key2": "yyy",
|
||||
"key3": "zzz",
|
||||
}
|
||||
# This will have 3 vectors representing it
|
||||
doc2 = {
|
||||
"key0": "uuu",
|
||||
"key1": "vvv",
|
||||
"key2": "www",
|
||||
"key3": "xxx",
|
||||
}
|
||||
store.put(("test",), "doc1", doc1)
|
||||
store.put(("test",), "doc2", doc2)
|
||||
|
||||
# doc2.key3 and doc1.key1 both would have the highest score
|
||||
results = store.search(("test",), query="xxx")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
ascore = results[0].score
|
||||
bscore = results[1].score
|
||||
assert ascore == pytest.approx(bscore, abs=1e-3)
|
||||
|
||||
# ~Only match doc2
|
||||
results = store.search(("test",), query="uuu")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
assert results[0].key == "doc2"
|
||||
assert results[0].score > results[1].score
|
||||
assert ascore == pytest.approx(results[0].score, abs=1e-3)
|
||||
|
||||
# ~Only match doc1
|
||||
results = store.search(("test",), query="zzz")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
assert results[0].key == "doc1"
|
||||
assert results[0].score > results[1].score
|
||||
assert ascore == pytest.approx(results[0].score, abs=1e-3)
|
||||
|
||||
# Un-indexed - will have low results for both. Not zero (because we're projecting)
|
||||
# but less than the above.
|
||||
results = store.search(("test",), query="www")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
assert results[0].score < ascore
|
||||
assert results[1].score < ascore
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
("vector", "cosine"),
|
||||
("vector", "inner_product"),
|
||||
("halfvec", "cosine"),
|
||||
("halfvec", "inner_product"),
|
||||
],
|
||||
)
|
||||
def test_embed_with_path_operation_config(
|
||||
request: Any,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test operation-level field configuration for vector search."""
|
||||
|
||||
with _create_vector_store(
|
||||
vector_type,
|
||||
distance_type,
|
||||
fake_embeddings,
|
||||
text_fields=["key17"], # Default fields that won't match our test data
|
||||
) as store:
|
||||
doc3 = {
|
||||
"key0": "aaa",
|
||||
"key1": "bbb",
|
||||
"key2": "ccc",
|
||||
"key3": "ddd",
|
||||
}
|
||||
doc4 = {
|
||||
"key0": "eee",
|
||||
"key1": "bbb", # Same as doc3.key1
|
||||
"key2": "fff",
|
||||
"key3": "ggg",
|
||||
}
|
||||
|
||||
store.put(("test",), "doc3", doc3, index=["key0", "key1"])
|
||||
store.put(("test",), "doc4", doc4, index=["key1", "key3"])
|
||||
|
||||
results = store.search(("test",), query="aaa")
|
||||
assert len(results) == 2
|
||||
assert results[0].key == "doc3"
|
||||
assert len(set(r.key for r in results)) == 2
|
||||
assert results[0].score > results[1].score
|
||||
|
||||
results = store.search(("test",), query="ggg")
|
||||
assert len(results) == 2
|
||||
assert results[0].key == "doc4"
|
||||
assert results[0].score > results[1].score
|
||||
|
||||
results = store.search(("test",), query="bbb")
|
||||
assert len(results) == 2
|
||||
assert results[0].key != results[1].key
|
||||
assert results[0].score == pytest.approx(results[1].score, abs=1e-3)
|
||||
|
||||
results = store.search(("test",), query="ccc")
|
||||
assert len(results) == 2
|
||||
assert all(
|
||||
r.score < 0.9 for r in results
|
||||
) # Unindexed field should have low scores
|
||||
|
||||
# Test index=False behavior
|
||||
doc5 = {
|
||||
"key0": "hhh",
|
||||
"key1": "iii",
|
||||
}
|
||||
store.put(("test",), "doc5", doc5, index=False)
|
||||
results = store.search(("test",))
|
||||
assert len(results) == 3
|
||||
assert all(r.score is None for r in results), f"{results}"
|
||||
assert any(r.key == "doc5" for r in results)
|
||||
|
||||
results = store.search(("test",), query="hhh")
|
||||
# TODO: We don't currently fill in additional results if there are not enough
|
||||
# returned during vector search.
|
||||
# assert len(results) == 3
|
||||
# doc5_result = next(r for r in results if r.key == "doc5")
|
||||
# assert doc5_result.score is None
|
||||
|
||||
|
||||
def _cosine_similarity(X: list[float], Y: list[list[float]]) -> list[float]:
|
||||
"""
|
||||
Compute cosine similarity between a vector X and a matrix Y.
|
||||
Lazy import numpy for efficiency.
|
||||
"""
|
||||
|
||||
similarities = []
|
||||
for y in Y:
|
||||
dot_product = sum(a * b for a, b in zip(X, y, strict=False))
|
||||
norm1 = sum(a * a for a in X) ** 0.5
|
||||
norm2 = sum(a * a for a in y) ** 0.5
|
||||
similarity = dot_product / (norm1 * norm2) if norm1 > 0 and norm2 > 0 else 0.0
|
||||
similarities.append(similarity)
|
||||
|
||||
return similarities
|
||||
|
||||
|
||||
def _inner_product(X: list[float], Y: list[list[float]]) -> list[float]:
|
||||
"""
|
||||
Compute inner product between a vector X and a matrix Y.
|
||||
Lazy import numpy for efficiency.
|
||||
"""
|
||||
|
||||
similarities = []
|
||||
for y in Y:
|
||||
similarity = sum(a * b for a, b in zip(X, y, strict=False))
|
||||
similarities.append(similarity)
|
||||
|
||||
return similarities
|
||||
|
||||
|
||||
def _neg_l2_distance(X: list[float], Y: list[list[float]]) -> list[float]:
|
||||
"""
|
||||
Compute l2 distance between a vector X and a matrix Y.
|
||||
Lazy import numpy for efficiency.
|
||||
"""
|
||||
|
||||
similarities = []
|
||||
for y in Y:
|
||||
similarity = sum((a - b) ** 2 for a, b in zip(X, y, strict=False)) ** 0.5
|
||||
similarities.append(-similarity)
|
||||
|
||||
return similarities
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
("vector", "cosine"),
|
||||
("vector", "inner_product"),
|
||||
("halfvec", "l2"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("query", ["aaa", "bbb", "ccc", "abcd", "poisson"])
|
||||
def test_scores(
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
query: str,
|
||||
) -> None:
|
||||
"""Test operation-level field configuration for vector search."""
|
||||
with _create_vector_store(
|
||||
vector_type,
|
||||
distance_type,
|
||||
fake_embeddings,
|
||||
text_fields=["key0"],
|
||||
) as store:
|
||||
doc = {
|
||||
"key0": "aaa",
|
||||
}
|
||||
store.put(("test",), "doc", doc, index=["key0", "key1"])
|
||||
|
||||
results = store.search((), query=query)
|
||||
vec0 = fake_embeddings.embed_query(doc["key0"])
|
||||
vec1 = fake_embeddings.embed_query(query)
|
||||
if distance_type == "cosine":
|
||||
similarities = _cosine_similarity(vec1, [vec0])
|
||||
elif distance_type == "inner_product":
|
||||
similarities = _inner_product(vec1, [vec0])
|
||||
elif distance_type == "l2":
|
||||
similarities = _neg_l2_distance(vec1, [vec0])
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].score == pytest.approx(similarities[0], abs=1e-3)
|
||||
|
||||
|
||||
def test_nonnull_migrations() -> None:
|
||||
_leading_comment_remover = re.compile(r"^/\*.*?\*/")
|
||||
for migration in PostgresStore.MIGRATIONS:
|
||||
statement = _leading_comment_remover.sub("", migration).split()[0]
|
||||
assert statement.strip()
|
||||
|
||||
|
||||
def test_store_ttl(store):
|
||||
# Assumes a TTL of 1 minute = 60 seconds
|
||||
ns = ("foo",)
|
||||
store.put(
|
||||
ns,
|
||||
key="item1",
|
||||
value={"foo": "bar"},
|
||||
ttl=TTL_MINUTES, # type: ignore
|
||||
)
|
||||
time.sleep(TTL_SECONDS - 2)
|
||||
res = store.get(ns, key="item1", refresh_ttl=True)
|
||||
assert res is not None
|
||||
time.sleep(TTL_SECONDS - 2)
|
||||
results = store.search(ns, query="foo", refresh_ttl=True)
|
||||
assert len(results) == 1
|
||||
time.sleep(TTL_SECONDS - 2)
|
||||
res = store.get(ns, key="item1", refresh_ttl=False)
|
||||
assert res is not None
|
||||
time.sleep(TTL_SECONDS - 1)
|
||||
# Now has been (TTL_SECONDS-2)*2 > TTL_SECONDS + TTL_SECONDS/2
|
||||
res = store.search(ns, query="bar", refresh_ttl=False)
|
||||
assert len(res) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
("vector", "cosine"),
|
||||
("vector", "inner_product"),
|
||||
("halfvec", "cosine"),
|
||||
("halfvec", "inner_product"),
|
||||
],
|
||||
)
|
||||
def test_non_ascii(
|
||||
request: Any,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test support for non-ascii characters"""
|
||||
with _create_vector_store(vector_type, distance_type, fake_embeddings) as store:
|
||||
store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese
|
||||
store.put(
|
||||
("user_123", "memories"), "2", {"text": "これは日本語です"}
|
||||
) # Japanese
|
||||
store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean
|
||||
store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian
|
||||
store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi
|
||||
|
||||
result1 = store.search(("user_123", "memories"), query="这是中文")
|
||||
result2 = store.search(("user_123", "memories"), query="これは日本語です")
|
||||
result3 = store.search(("user_123", "memories"), query="이건 한국어야")
|
||||
result4 = store.search(("user_123", "memories"), query="Это русский")
|
||||
result5 = store.search(("user_123", "memories"), query="यह रूसी है")
|
||||
|
||||
assert result1[0].key == "1"
|
||||
assert result2[0].key == "2"
|
||||
assert result3[0].key == "3"
|
||||
assert result4[0].key == "4"
|
||||
assert result5[0].key == "5"
|
||||
@@ -0,0 +1,360 @@
|
||||
# type: ignore
|
||||
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
from langgraph.checkpoint.base import (
|
||||
EXCLUDED_METADATA_KEYS,
|
||||
Checkpoint,
|
||||
CheckpointMetadata,
|
||||
create_checkpoint,
|
||||
empty_checkpoint,
|
||||
)
|
||||
from langgraph.checkpoint.serde.types import TASKS
|
||||
from psycopg import Connection
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
from langgraph.checkpoint.postgres import PostgresSaver, ShallowPostgresSaver
|
||||
from tests.conftest import DEFAULT_POSTGRES_URI
|
||||
|
||||
|
||||
def _exclude_keys(config: dict[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in config.items() if k not in EXCLUDED_METADATA_KEYS}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _pool_saver():
|
||||
"""Fixture for pool mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
# yield checkpointer
|
||||
with ConnectionPool(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
max_size=10,
|
||||
kwargs={"autocommit": True, "row_factory": dict_row},
|
||||
) as pool:
|
||||
checkpointer = PostgresSaver(pool)
|
||||
checkpointer.setup()
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _pipe_saver():
|
||||
"""Fixture for pipeline mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
with Connection.connect(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
autocommit=True,
|
||||
prepare_threshold=0,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
checkpointer = PostgresSaver(conn)
|
||||
checkpointer.setup()
|
||||
with conn.pipeline() as pipe:
|
||||
checkpointer = PostgresSaver(conn, pipe=pipe)
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _base_saver():
|
||||
"""Fixture for regular connection mode testing."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
with Connection.connect(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
autocommit=True,
|
||||
prepare_threshold=0,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
checkpointer = PostgresSaver(conn)
|
||||
checkpointer.setup()
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _shallow_saver():
|
||||
"""Fixture for regular connection mode testing with a shallow checkpointer."""
|
||||
database = f"test_{uuid4().hex[:16]}"
|
||||
# create unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"CREATE DATABASE {database}")
|
||||
try:
|
||||
with Connection.connect(
|
||||
DEFAULT_POSTGRES_URI + database,
|
||||
autocommit=True,
|
||||
prepare_threshold=0,
|
||||
row_factory=dict_row,
|
||||
) as conn:
|
||||
checkpointer = ShallowPostgresSaver(conn)
|
||||
checkpointer.setup()
|
||||
yield checkpointer
|
||||
finally:
|
||||
# drop unique db
|
||||
with Connection.connect(DEFAULT_POSTGRES_URI, autocommit=True) as conn:
|
||||
conn.execute(f"DROP DATABASE {database}")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _saver(name: str):
|
||||
if name == "base":
|
||||
with _base_saver() as saver:
|
||||
yield saver
|
||||
elif name == "shallow":
|
||||
with _shallow_saver() as saver:
|
||||
yield saver
|
||||
elif name == "pool":
|
||||
with _pool_saver() as saver:
|
||||
yield saver
|
||||
elif name == "pipe":
|
||||
with _pipe_saver() as saver:
|
||||
yield saver
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_data():
|
||||
"""Fixture providing test data for checkpoint tests."""
|
||||
config_1: RunnableConfig = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-1",
|
||||
"checkpoint_id": "1",
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
config_2: RunnableConfig = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_id": "2",
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
config_3: RunnableConfig = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_id": "2-inner",
|
||||
"checkpoint_ns": "inner",
|
||||
}
|
||||
}
|
||||
|
||||
chkpnt_1: Checkpoint = empty_checkpoint()
|
||||
chkpnt_2: Checkpoint = create_checkpoint(chkpnt_1, {}, 1)
|
||||
chkpnt_3: Checkpoint = empty_checkpoint()
|
||||
|
||||
metadata_1: CheckpointMetadata = {
|
||||
"source": "input",
|
||||
"step": 2,
|
||||
"score": 1,
|
||||
}
|
||||
metadata_2: CheckpointMetadata = {
|
||||
"source": "loop",
|
||||
"step": 1,
|
||||
"score": None,
|
||||
}
|
||||
metadata_3: CheckpointMetadata = {}
|
||||
|
||||
return {
|
||||
"configs": [config_1, config_2, config_3],
|
||||
"checkpoints": [chkpnt_1, chkpnt_2, chkpnt_3],
|
||||
"metadata": [metadata_1, metadata_2, metadata_3],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe", "shallow"])
|
||||
def test_combined_metadata(saver_name: str, test_data) -> None:
|
||||
with _saver(saver_name) as saver:
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_ns": "",
|
||||
"__super_private_key": "super_private_value",
|
||||
},
|
||||
"metadata": {"run_id": "my_run_id"},
|
||||
}
|
||||
chkpnt: Checkpoint = create_checkpoint(empty_checkpoint(), {}, 1)
|
||||
metadata: CheckpointMetadata = {
|
||||
"source": "loop",
|
||||
"step": 1,
|
||||
"score": None,
|
||||
}
|
||||
saver.put(config, chkpnt, metadata, {})
|
||||
checkpoint = saver.get_tuple(config)
|
||||
assert checkpoint.metadata == {
|
||||
**metadata,
|
||||
"run_id": "my_run_id",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe", "shallow"])
|
||||
def test_search(saver_name: str, test_data) -> None:
|
||||
with _saver(saver_name) as saver:
|
||||
configs = test_data["configs"]
|
||||
checkpoints = test_data["checkpoints"]
|
||||
metadata = test_data["metadata"]
|
||||
|
||||
saver.put(configs[0], checkpoints[0], metadata[0], {})
|
||||
saver.put(configs[1], checkpoints[1], metadata[1], {})
|
||||
saver.put(configs[2], checkpoints[2], metadata[2], {})
|
||||
|
||||
# call method / assertions
|
||||
query_1 = {"source": "input"} # search by 1 key
|
||||
query_2 = {
|
||||
"step": 1,
|
||||
} # search by multiple keys
|
||||
query_3: dict[str, Any] = {} # search by no keys, return all checkpoints
|
||||
query_4 = {"source": "update", "step": 1} # no match
|
||||
|
||||
search_results_1 = list(saver.list(None, filter=query_1))
|
||||
assert len(search_results_1) == 1
|
||||
assert search_results_1[0].metadata == {
|
||||
**_exclude_keys(configs[0]["configurable"]),
|
||||
**metadata[0],
|
||||
}
|
||||
|
||||
search_results_2 = list(saver.list(None, filter=query_2))
|
||||
assert len(search_results_2) == 1
|
||||
assert search_results_2[0].metadata == {
|
||||
**_exclude_keys(configs[1]["configurable"]),
|
||||
**metadata[1],
|
||||
}
|
||||
|
||||
search_results_3 = list(saver.list(None, filter=query_3))
|
||||
assert len(search_results_3) == 3
|
||||
|
||||
search_results_4 = list(saver.list(None, filter=query_4))
|
||||
assert len(search_results_4) == 0
|
||||
|
||||
# search by config (defaults to checkpoints across all namespaces)
|
||||
search_results_5 = list(saver.list({"configurable": {"thread_id": "thread-2"}}))
|
||||
assert len(search_results_5) == 2
|
||||
assert {
|
||||
search_results_5[0].config["configurable"]["checkpoint_ns"],
|
||||
search_results_5[1].config["configurable"]["checkpoint_ns"],
|
||||
} == {"", "inner"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe", "shallow"])
|
||||
def test_null_chars(saver_name: str, test_data) -> None:
|
||||
with _saver(saver_name) as saver:
|
||||
config = saver.put(
|
||||
test_data["configs"][0],
|
||||
test_data["checkpoints"][0],
|
||||
{"my_key": "\x00abc"},
|
||||
{},
|
||||
)
|
||||
assert saver.get_tuple(config).metadata["my_key"] == "abc" # type: ignore
|
||||
assert (
|
||||
list(saver.list(None, filter={"my_key": "abc"}))[0].metadata["my_key"]
|
||||
== "abc"
|
||||
)
|
||||
|
||||
|
||||
def test_nonnull_migrations() -> None:
|
||||
_leading_comment_remover = re.compile(r"^/\*.*?\*/")
|
||||
for migration in PostgresSaver.MIGRATIONS:
|
||||
statement = _leading_comment_remover.sub("", migration).split()[0]
|
||||
assert statement.strip()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe"])
|
||||
def test_pending_sends_migration(saver_name: str) -> None:
|
||||
with _saver(saver_name) as saver:
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-1",
|
||||
"checkpoint_ns": "",
|
||||
}
|
||||
}
|
||||
|
||||
# create the first checkpoint
|
||||
# and put some pending sends
|
||||
checkpoint_0 = empty_checkpoint()
|
||||
config = saver.put(config, checkpoint_0, {}, {})
|
||||
saver.put_writes(
|
||||
config, [(TASKS, "send-1"), (TASKS, "send-2")], task_id="task-1"
|
||||
)
|
||||
saver.put_writes(config, [(TASKS, "send-3")], task_id="task-2")
|
||||
|
||||
# check that fetching checkpoint_0 doesn't attach pending sends
|
||||
# (they should be attached to the next checkpoint)
|
||||
tuple_0 = saver.get_tuple(config)
|
||||
assert tuple_0.checkpoint["channel_values"] == {}
|
||||
assert tuple_0.checkpoint["channel_versions"] == {}
|
||||
|
||||
# create the second checkpoint
|
||||
checkpoint_1 = create_checkpoint(checkpoint_0, {}, 1)
|
||||
config = saver.put(config, checkpoint_1, {}, {})
|
||||
|
||||
# check that pending sends are attached to checkpoint_1
|
||||
checkpoint_1 = saver.get_tuple(config)
|
||||
assert checkpoint_1.checkpoint["channel_values"] == {
|
||||
TASKS: ["send-1", "send-2", "send-3"]
|
||||
}
|
||||
assert TASKS in checkpoint_1.checkpoint["channel_versions"]
|
||||
|
||||
# check that list also applies the migration
|
||||
search_results = [
|
||||
c for c in saver.list({"configurable": {"thread_id": "thread-1"}})
|
||||
]
|
||||
assert len(search_results) == 2
|
||||
assert search_results[-1].checkpoint["channel_values"] == {}
|
||||
assert search_results[-1].checkpoint["channel_versions"] == {}
|
||||
assert search_results[0].checkpoint["channel_values"] == {
|
||||
TASKS: ["send-1", "send-2", "send-3"]
|
||||
}
|
||||
assert TASKS in search_results[0].checkpoint["channel_versions"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("saver_name", ["base", "pool", "pipe"])
|
||||
def test_get_checkpoint_no_channel_values(
|
||||
monkeypatch, saver_name: str, test_data
|
||||
) -> None:
|
||||
"""Backwards compatibility test that verifies a checkpoint with no channel_values key can be retrieved without throwing an error."""
|
||||
with _saver(saver_name) as saver:
|
||||
config = {
|
||||
"configurable": {
|
||||
"thread_id": "thread-2",
|
||||
"checkpoint_ns": "",
|
||||
"__super_private_key": "super_private_value",
|
||||
},
|
||||
}
|
||||
chkpnt: Checkpoint = create_checkpoint(empty_checkpoint(), {}, 1)
|
||||
saver.put(config, chkpnt, {}, {})
|
||||
|
||||
load_checkpoint_tuple = saver._load_checkpoint_tuple
|
||||
|
||||
def patched_load_checkpoint_tuple(value):
|
||||
value["checkpoint"].pop("channel_values", None)
|
||||
return load_checkpoint_tuple(value)
|
||||
|
||||
monkeypatch.setattr(
|
||||
saver, "_load_checkpoint_tuple", patched_load_checkpoint_tuple
|
||||
)
|
||||
|
||||
checkpoint = saver.get_tuple(config)
|
||||
assert checkpoint.checkpoint["channel_values"] == {}
|
||||
Reference in New Issue
Block a user