wehub-resource-sync
97e91a83f3
Ruff / Ruff (push) Has been cancelled
Test / Core Tests (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.10) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.11) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.12) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.13) (push) Has been cancelled
Test / Offline Coverage Tests (Python 3.9) (push) Has been cancelled
Test / Full Coverage (Python 3.11) (push) Has been cancelled
Test / Core Provider Tests (OpenAI) (push) Has been cancelled
Test / Core Provider Tests (Anthropic) (push) Has been cancelled
Test / Core Provider Tests (Google) (push) Has been cancelled
Test / Core Provider Tests (Other) (push) Has been cancelled
Test / Anthropic Tests (push) Has been cancelled
Test / Gemini Tests (push) Has been cancelled
Test / Google GenAI Tests (push) Has been cancelled
Test / Vertex AI Tests (push) Has been cancelled
Test / OpenAI Tests (push) Has been cancelled
Test / Writer Tests (push) Has been cancelled
Test / Auto Client Tests (push) Has been cancelled
ty / type-check (push) Has been cancelled
1195 lines
36 KiB
Python
1195 lines
36 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from pydantic import BaseModel, ValidationError
|
|
from tenacity import (
|
|
AsyncRetrying,
|
|
Retrying,
|
|
retry_if_exception_type,
|
|
stop_after_attempt,
|
|
)
|
|
|
|
from instructor import Mode, Provider
|
|
from instructor.v2.core.errors import InstructorRetryException, ResponseParsingError
|
|
from instructor.v2.core.hooks import Hooks
|
|
from instructor.v2.core.retry import (
|
|
_finalize_parsed_response,
|
|
_initialize_usage,
|
|
retry_async_v2,
|
|
retry_sync_v2,
|
|
)
|
|
from instructor.v2.dsl.response_list import ListResponse
|
|
|
|
|
|
class Answer(BaseModel):
|
|
value: int
|
|
|
|
|
|
def _validation_error() -> ValidationError:
|
|
try:
|
|
Answer.model_validate({"value": "bad"})
|
|
except ValidationError as exc:
|
|
return exc
|
|
raise AssertionError("Expected a validation error")
|
|
|
|
|
|
def test_finalize_parsed_response_wraps_plain_list_and_sets_raw_response() -> None:
|
|
response = object()
|
|
parsed = [Answer(value=1), Answer(value=2)]
|
|
|
|
finalized = _finalize_parsed_response(parsed, response)
|
|
|
|
assert isinstance(finalized, ListResponse)
|
|
assert list(finalized) == parsed
|
|
assert finalized._raw_response is response
|
|
|
|
|
|
def test_initialize_usage_returns_openai_usage_shape() -> None:
|
|
usage = _initialize_usage(Provider.OPENAI)
|
|
|
|
assert usage.completion_tokens == 0
|
|
assert usage.prompt_tokens == 0
|
|
assert usage.total_tokens == 0
|
|
|
|
|
|
def test_retry_sync_v2_returns_raw_result_when_no_response_model() -> None:
|
|
def fake_func(*args: Any, **kwargs: Any) -> str:
|
|
return f"{args[0]}:{kwargs['suffix']}"
|
|
|
|
result = retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=None,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=1,
|
|
args=("hello",),
|
|
kwargs={"suffix": "world"},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert result == "hello:world"
|
|
|
|
|
|
def test_retry_sync_v2_reasks_after_validation_error(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls: list[dict[str, Any]] = []
|
|
parser_calls: list[str] = []
|
|
emitted: dict[str, list[Any]] = {
|
|
"args": [],
|
|
"responses": [],
|
|
"errors": [],
|
|
"completion_errors": [],
|
|
"last_attempts": [],
|
|
}
|
|
|
|
def fake_func(*_args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
calls.append(dict(kwargs))
|
|
return {"payload": kwargs["messages"][-1]["content"]}
|
|
|
|
def fake_parser(**kwargs: Any) -> Answer:
|
|
parser_calls.append(kwargs["response"]["payload"])
|
|
if len(parser_calls) == 1:
|
|
raise _validation_error()
|
|
return Answer(value=7)
|
|
|
|
def fake_reask_handler(
|
|
kwargs: dict[str, Any], response: Any, exception: ValidationError
|
|
) -> dict[str, Any]:
|
|
assert response == {"payload": "first"}
|
|
assert isinstance(exception, ValidationError)
|
|
return {
|
|
**kwargs,
|
|
"messages": [*kwargs["messages"], {"role": "user", "content": "second"}],
|
|
}
|
|
|
|
hooks = Hooks()
|
|
hooks.on("completion:kwargs", lambda **kwargs: emitted["args"].append(kwargs))
|
|
hooks.on(
|
|
"completion:response",
|
|
lambda response: emitted["responses"].append(response),
|
|
)
|
|
hooks.on(
|
|
"parse:error",
|
|
lambda error, **kwargs: emitted["errors"].append((error, kwargs)),
|
|
)
|
|
hooks.on(
|
|
"completion:error",
|
|
lambda error, **kwargs: emitted["completion_errors"].append((error, kwargs)),
|
|
)
|
|
hooks.on(
|
|
"completion:last_attempt",
|
|
lambda error, **kwargs: emitted["last_attempts"].append((error, kwargs)),
|
|
)
|
|
|
|
def no_validate(_provider: Provider, _mode: Mode) -> None:
|
|
return None
|
|
|
|
def get_handlers(_provider: Provider, _mode: Mode) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
response_parser=fake_parser,
|
|
reask_handler=fake_reask_handler,
|
|
)
|
|
|
|
def update_usage(response: Any, total_usage: Any) -> Any:
|
|
assert total_usage == {"tokens": 0}
|
|
return response
|
|
|
|
def initialize_usage(_provider: Provider) -> dict[str, int]:
|
|
return {"tokens": 0}
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
no_validate,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
get_handlers,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.update_total_usage",
|
|
update_usage,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry._initialize_usage",
|
|
initialize_usage,
|
|
)
|
|
|
|
result = retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context={"tenant": "acme"},
|
|
max_retries=Retrying(
|
|
stop=stop_after_attempt(2),
|
|
retry=retry_if_exception_type(ValidationError),
|
|
reraise=True,
|
|
),
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=hooks,
|
|
)
|
|
|
|
assert result.value == 7
|
|
assert len(calls) == 2
|
|
assert calls[1]["messages"][-1]["content"] == "second"
|
|
assert parser_calls == ["first", "second"]
|
|
assert len(emitted["args"]) == 2
|
|
assert len(emitted["responses"]) == 2
|
|
assert len(emitted["errors"]) == 1
|
|
assert isinstance(emitted["errors"][0][0], ValidationError)
|
|
assert emitted["errors"][0][1]["attempt_number"] == 1
|
|
assert emitted["completion_errors"] == []
|
|
assert emitted["last_attempts"] == []
|
|
|
|
|
|
def test_retry_sync_v2_emits_last_attempt_metadata_on_exhaustion(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
emitted: dict[str, list[Any]] = {"completion_errors": [], "last_attempts": []}
|
|
|
|
def fake_func(*_args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
return {"payload": kwargs["messages"][-1]["content"]}
|
|
|
|
def always_fail_parser(**_kwargs: Any) -> Answer:
|
|
raise _validation_error()
|
|
|
|
def reask_kwargs(
|
|
kwargs: dict[str, Any], response: Any, exception: ValidationError
|
|
) -> dict[str, Any]:
|
|
assert response["payload"] in {"first", "retry"}
|
|
assert isinstance(exception, ValidationError)
|
|
return {
|
|
**kwargs,
|
|
"messages": [*kwargs["messages"], {"role": "user", "content": "retry"}],
|
|
}
|
|
|
|
hooks = Hooks()
|
|
hooks.on(
|
|
"completion:error",
|
|
lambda error, **kwargs: emitted["completion_errors"].append((error, kwargs)),
|
|
)
|
|
hooks.on(
|
|
"completion:last_attempt",
|
|
lambda error, **kwargs: emitted["last_attempts"].append((error, kwargs)),
|
|
)
|
|
|
|
def no_validate(_provider: Provider, _mode: Mode) -> None:
|
|
return None
|
|
|
|
def get_handlers(_provider: Provider, _mode: Mode) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
response_parser=always_fail_parser,
|
|
reask_handler=reask_kwargs,
|
|
)
|
|
|
|
def update_usage(response: Any, total_usage: Any) -> Any:
|
|
assert total_usage == {"tokens": 0}
|
|
return response
|
|
|
|
def initialize_usage(_provider: Provider) -> dict[str, int]:
|
|
return {"tokens": 0}
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
no_validate,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
get_handlers,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.update_total_usage",
|
|
update_usage,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry._initialize_usage",
|
|
initialize_usage,
|
|
)
|
|
|
|
with pytest.raises(InstructorRetryException):
|
|
retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=hooks,
|
|
)
|
|
|
|
assert emitted["completion_errors"] == []
|
|
assert len(emitted["last_attempts"]) == 1
|
|
assert emitted["last_attempts"][0][1] == {
|
|
"attempt_number": 2,
|
|
"max_attempts": 2,
|
|
"is_last_attempt": True,
|
|
}
|
|
|
|
|
|
def test_retry_sync_v2_marks_api_error_as_last_attempt(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
emitted: dict[str, list[Any]] = {"completion_errors": [], "last_attempts": []}
|
|
|
|
def fake_func(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
nonlocal calls
|
|
calls += 1
|
|
raise RuntimeError("boom")
|
|
|
|
hooks = Hooks()
|
|
hooks.on(
|
|
"completion:error",
|
|
lambda error, **kwargs: emitted["completion_errors"].append((error, kwargs)),
|
|
)
|
|
hooks.on(
|
|
"completion:last_attempt",
|
|
lambda error, **kwargs: emitted["last_attempts"].append((error, kwargs)),
|
|
)
|
|
|
|
def no_validate(_provider: Provider, _mode: Mode) -> None:
|
|
return None
|
|
|
|
def get_handlers(_provider: Provider, _mode: Mode) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
response_parser=lambda **_kwargs: Answer(value=1),
|
|
reask_handler=lambda **kwargs: kwargs,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
no_validate,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
get_handlers,
|
|
)
|
|
|
|
with pytest.raises(InstructorRetryException) as exc_info:
|
|
retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=3,
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=hooks,
|
|
)
|
|
|
|
assert calls == 1
|
|
assert exc_info.value.n_attempts == 1
|
|
assert len(emitted["completion_errors"]) == 1
|
|
assert emitted["completion_errors"][0][1] == {
|
|
"attempt_number": 1,
|
|
"max_attempts": 4,
|
|
"is_last_attempt": True,
|
|
}
|
|
assert len(emitted["last_attempts"]) == 1
|
|
assert emitted["last_attempts"][0][1] == {
|
|
"attempt_number": 1,
|
|
"max_attempts": 4,
|
|
"is_last_attempt": True,
|
|
}
|
|
|
|
|
|
def test_retry_sync_v2_integer_max_retries_counts_retries(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
parser_calls = 0
|
|
|
|
def fake_func(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
return {"payload": "ok"}
|
|
|
|
def fake_parser(**_kwargs: Any) -> Answer:
|
|
nonlocal parser_calls
|
|
parser_calls += 1
|
|
if parser_calls == 1:
|
|
raise _validation_error()
|
|
return Answer(value=42)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=fake_parser,
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
|
|
result = retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert result == Answer(value=42)
|
|
assert parser_calls == 2
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"parse_error",
|
|
[
|
|
json.JSONDecodeError("bad json", "{", 1),
|
|
pytest.param(
|
|
ResponseParsingError("missing tool call"),
|
|
id="response-parsing-error",
|
|
),
|
|
],
|
|
)
|
|
def test_retry_sync_v2_retries_parse_errors(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
parse_error: Exception,
|
|
) -> None:
|
|
parser_calls = 0
|
|
|
|
def fake_parser(**_kwargs: Any) -> Answer:
|
|
nonlocal parser_calls
|
|
parser_calls += 1
|
|
if parser_calls == 1:
|
|
raise parse_error
|
|
return Answer(value=42)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=fake_parser,
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
|
|
result = retry_sync_v2(
|
|
func=lambda *_args, **_kwargs: {"payload": "ok"},
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.JSON,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert result == Answer(value=42)
|
|
assert parser_calls == 2
|
|
|
|
|
|
def test_retry_sync_v2_reports_terminal_api_error_after_validation_failure(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
|
|
def fake_func(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
nonlocal calls
|
|
calls += 1
|
|
if calls == 2:
|
|
raise RuntimeError("terminal API failure")
|
|
return {"payload": "invalid"}
|
|
|
|
def always_fail_parser(**_kwargs: Any) -> Answer:
|
|
raise _validation_error()
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=always_fail_parser,
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
|
|
with pytest.raises(
|
|
InstructorRetryException, match="terminal API failure"
|
|
) as exc_info:
|
|
retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert exc_info.value.n_attempts == 2
|
|
|
|
|
|
def test_retry_sync_v2_raises_instructor_retry_exception_after_exhaustion(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
def fake_func(*_args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
return {"payload": kwargs["messages"][-1]["content"]}
|
|
|
|
def always_fail_parser(**_kwargs: Any) -> Answer:
|
|
raise _validation_error()
|
|
|
|
def reask_kwargs(
|
|
kwargs: dict[str, Any], response: Any, exception: ValidationError
|
|
) -> dict[str, Any]:
|
|
assert response["payload"] in {"first", "retry"}
|
|
assert isinstance(exception, ValidationError)
|
|
return {
|
|
**kwargs,
|
|
"messages": [*kwargs["messages"], {"role": "user", "content": "retry"}],
|
|
}
|
|
|
|
def no_validate(_provider: Provider, _mode: Mode) -> None:
|
|
return None
|
|
|
|
def get_handlers(_provider: Provider, _mode: Mode) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
response_parser=always_fail_parser,
|
|
reask_handler=reask_kwargs,
|
|
)
|
|
|
|
def update_usage(response: Any, total_usage: Any) -> Any:
|
|
assert total_usage == {"tokens": 0}
|
|
return response
|
|
|
|
def initialize_usage(_provider: Provider) -> dict[str, int]:
|
|
return {"tokens": 0}
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
no_validate,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
get_handlers,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.update_total_usage",
|
|
update_usage,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry._initialize_usage",
|
|
initialize_usage,
|
|
)
|
|
|
|
with pytest.raises(InstructorRetryException) as exc_info:
|
|
retry_sync_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=Retrying(
|
|
stop=stop_after_attempt(2),
|
|
retry=retry_if_exception_type(ValidationError),
|
|
reraise=True,
|
|
),
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
error = exc_info.value
|
|
assert error.n_attempts == 2
|
|
assert error.last_completion == {"payload": "retry"}
|
|
assert error.create_kwargs is not None
|
|
assert error.create_kwargs["messages"][-1]["content"] == "retry"
|
|
assert len(error.failed_attempts or []) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_reasks_after_validation_error(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
parser_calls: list[str] = []
|
|
|
|
async def fake_func(*_args: Any, **kwargs: Any) -> dict[str, Any]:
|
|
return {"payload": kwargs["messages"][-1]["content"]}
|
|
|
|
def fake_parser(**kwargs: Any) -> Answer:
|
|
parser_calls.append(kwargs["response"]["payload"])
|
|
if len(parser_calls) == 1:
|
|
raise _validation_error()
|
|
return Answer(value=9)
|
|
|
|
def reask_kwargs(
|
|
kwargs: dict[str, Any], response: Any, exception: ValidationError
|
|
) -> dict[str, Any]:
|
|
assert response["payload"] in {"first", "retry"}
|
|
assert isinstance(exception, ValidationError)
|
|
return {
|
|
**kwargs,
|
|
"messages": [*kwargs["messages"], {"role": "user", "content": "retry"}],
|
|
}
|
|
|
|
def no_validate(_provider: Provider, _mode: Mode) -> None:
|
|
return None
|
|
|
|
def get_handlers(_provider: Provider, _mode: Mode) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
response_parser=fake_parser,
|
|
reask_handler=reask_kwargs,
|
|
)
|
|
|
|
def update_usage(response: Any, total_usage: Any) -> Any:
|
|
assert total_usage == {"tokens": 0}
|
|
return response
|
|
|
|
def initialize_usage(_provider: Provider) -> dict[str, int]:
|
|
return {"tokens": 0}
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
no_validate,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
get_handlers,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.update_total_usage",
|
|
update_usage,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry._initialize_usage",
|
|
initialize_usage,
|
|
)
|
|
|
|
result = await retry_async_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=AsyncRetrying(
|
|
stop=stop_after_attempt(2),
|
|
retry=retry_if_exception_type(ValidationError),
|
|
reraise=True,
|
|
),
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert result == Answer(value=9)
|
|
assert parser_calls == ["first", "retry"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_marks_api_error_as_last_attempt(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
calls = 0
|
|
emitted: dict[str, list[Any]] = {"completion_errors": [], "last_attempts": []}
|
|
|
|
async def fake_func(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
nonlocal calls
|
|
calls += 1
|
|
raise RuntimeError("boom")
|
|
|
|
hooks = Hooks()
|
|
hooks.on(
|
|
"completion:error",
|
|
lambda error, **kwargs: emitted["completion_errors"].append((error, kwargs)),
|
|
)
|
|
hooks.on(
|
|
"completion:last_attempt",
|
|
lambda error, **kwargs: emitted["last_attempts"].append((error, kwargs)),
|
|
)
|
|
|
|
def no_validate(_provider: Provider, _mode: Mode) -> None:
|
|
return None
|
|
|
|
def get_handlers(_provider: Provider, _mode: Mode) -> SimpleNamespace:
|
|
return SimpleNamespace(
|
|
response_parser=lambda **_kwargs: Answer(value=1),
|
|
reask_handler=lambda **kwargs: kwargs,
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
no_validate,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
get_handlers,
|
|
)
|
|
|
|
with pytest.raises(InstructorRetryException) as exc_info:
|
|
await retry_async_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=3,
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=hooks,
|
|
)
|
|
|
|
assert calls == 1
|
|
assert exc_info.value.n_attempts == 1
|
|
assert len(emitted["completion_errors"]) == 1
|
|
assert emitted["completion_errors"][0][1] == {
|
|
"attempt_number": 1,
|
|
"max_attempts": 4,
|
|
"is_last_attempt": True,
|
|
}
|
|
assert len(emitted["last_attempts"]) == 1
|
|
assert emitted["last_attempts"][0][1] == {
|
|
"attempt_number": 1,
|
|
"max_attempts": 4,
|
|
"is_last_attempt": True,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_integer_max_retries_counts_retries(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
parser_calls = 0
|
|
|
|
async def fake_func(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
return {"payload": "ok"}
|
|
|
|
def fake_parser(**_kwargs: Any) -> Answer:
|
|
nonlocal parser_calls
|
|
parser_calls += 1
|
|
if parser_calls == 1:
|
|
raise _validation_error()
|
|
return Answer(value=42)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=fake_parser,
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
|
|
result = await retry_async_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert result == Answer(value=42)
|
|
assert parser_calls == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_retries_json_decode_errors(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
) -> None:
|
|
parser_calls = 0
|
|
|
|
async def fake_func(*_args: Any, **_kwargs: Any) -> dict[str, Any]:
|
|
return {"payload": "ok"}
|
|
|
|
def fake_parser(**_kwargs: Any) -> Answer:
|
|
nonlocal parser_calls
|
|
parser_calls += 1
|
|
if parser_calls == 1:
|
|
raise json.JSONDecodeError("bad json", "{", 1)
|
|
return Answer(value=42)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=fake_parser,
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
|
|
result = await retry_async_v2(
|
|
func=fake_func,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.JSON,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert result == Answer(value=42)
|
|
assert parser_calls == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("stream_kwargs", "expected_stream"),
|
|
[({}, False), ({"stream": True}, True)],
|
|
ids=["default-stream", "explicit-stream"],
|
|
)
|
|
async def test_retry_async_v2_preserves_parser_response_and_usage_contract(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
stream_kwargs: dict[str, bool],
|
|
expected_stream: bool,
|
|
) -> None:
|
|
response = SimpleNamespace(payload="answer")
|
|
context = {"tenant": "async"}
|
|
usage = {"tokens": 0}
|
|
seen_responses: list[Any] = []
|
|
seen_updates: list[tuple[Any, Any]] = []
|
|
|
|
def parser(**kwargs: Any) -> Answer:
|
|
assert kwargs == {
|
|
"response": response,
|
|
"response_model": Answer,
|
|
"validation_context": context,
|
|
"strict": False,
|
|
"stream": expected_stream,
|
|
"is_async": True,
|
|
}
|
|
return Answer(value=11)
|
|
|
|
def initialize_usage(provider: Provider) -> dict[str, int]:
|
|
assert provider is Provider.OPENAI
|
|
return usage
|
|
|
|
def update_usage(response: Any, total_usage: Any) -> Any:
|
|
seen_updates.append((response, total_usage))
|
|
return response
|
|
|
|
hooks = Hooks()
|
|
hooks.on("completion:response", lambda value: seen_responses.append(value))
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=parser,
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
monkeypatch.setattr("instructor.v2.core.retry._initialize_usage", initialize_usage)
|
|
monkeypatch.setattr("instructor.v2.core.retry.update_total_usage", update_usage)
|
|
caplog.set_level(logging.DEBUG, logger="instructor.v2.retry")
|
|
|
|
async def create(**_kwargs: Any) -> SimpleNamespace:
|
|
return response
|
|
|
|
result = await retry_async_v2(
|
|
func=create,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.JSON,
|
|
context=context,
|
|
max_retries=0,
|
|
args=(),
|
|
kwargs={"messages": [], **stream_kwargs},
|
|
strict=False,
|
|
hooks=hooks,
|
|
)
|
|
|
|
assert result == Answer(value=11)
|
|
assert getattr(result, "_raw_response", None) is response
|
|
assert seen_responses == [response]
|
|
assert seen_updates == [(response, usage)]
|
|
assert "Successfully parsed response on attempt 1" in caplog.text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("max_retries", "timeout"),
|
|
[(0, None), (3, 0)],
|
|
ids=["zero-retries", "expired-timeout"],
|
|
)
|
|
async def test_retry_async_v2_stops_after_one_attempt_when_retry_is_disabled(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
max_retries: int,
|
|
timeout: int | None,
|
|
) -> None:
|
|
calls = 0
|
|
response = {"payload": "invalid"}
|
|
|
|
async def create(**_kwargs: Any) -> dict[str, str]:
|
|
nonlocal calls
|
|
calls += 1
|
|
return response
|
|
|
|
def retrying(*, stop: Any, retry: Any, reraise: bool) -> AsyncRetrying:
|
|
return AsyncRetrying(stop=stop, retry=retry, reraise=reraise)
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=lambda **_kwargs: Answer.model_validate({"value": "bad"}),
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
monkeypatch.setattr("instructor.v2.core.retry.AsyncRetrying", retrying)
|
|
caplog.set_level(logging.DEBUG, logger="instructor.v2.retry")
|
|
|
|
with pytest.raises(InstructorRetryException) as exc_info:
|
|
await retry_async_v2(
|
|
func=create,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.JSON,
|
|
context=None,
|
|
max_retries=max_retries,
|
|
args=(),
|
|
kwargs={"messages": [], "timeout": timeout},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
assert calls == 1
|
|
assert exc_info.value.n_attempts == 1
|
|
assert exc_info.value.last_completion is response
|
|
assert "Validation error on attempt 1" in caplog.text
|
|
assert "Max retries exceeded. Total attempts: 1" in caplog.text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_reports_each_validation_api_error_and_last_attempt(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
errors = [_validation_error(), _validation_error()]
|
|
completion_errors: list[tuple[Exception, dict[str, Any]]] = []
|
|
last_attempts: list[tuple[Exception, dict[str, Any]]] = []
|
|
calls = 0
|
|
|
|
async def create(**_kwargs: Any) -> dict[str, str]:
|
|
nonlocal calls
|
|
error = errors[calls]
|
|
calls += 1
|
|
raise error
|
|
|
|
hooks = Hooks()
|
|
hooks.on(
|
|
"completion:error",
|
|
lambda error, **metadata: completion_errors.append((error, metadata)),
|
|
)
|
|
hooks.on(
|
|
"completion:last_attempt",
|
|
lambda error, **metadata: last_attempts.append((error, metadata)),
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=lambda **_kwargs: Answer(value=1),
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
caplog.set_level(logging.ERROR, logger="instructor.v2.retry")
|
|
|
|
with pytest.raises(InstructorRetryException) as exc_info:
|
|
await retry_async_v2(
|
|
func=create,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.TOOLS,
|
|
context=None,
|
|
max_retries=1,
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "answer"}]},
|
|
strict=True,
|
|
hooks=hooks,
|
|
)
|
|
|
|
assert calls == 2
|
|
assert [error for error, _ in completion_errors] == errors
|
|
assert [metadata for _, metadata in completion_errors] == [
|
|
{"attempt_number": 1, "max_attempts": 2, "is_last_attempt": False},
|
|
{"attempt_number": 2, "max_attempts": 2, "is_last_attempt": True},
|
|
]
|
|
assert last_attempts == [
|
|
(
|
|
errors[-1],
|
|
{"attempt_number": 2, "max_attempts": 2, "is_last_attempt": True},
|
|
)
|
|
]
|
|
assert exc_info.value.n_attempts == 2
|
|
assert exc_info.value.failed_attempts == []
|
|
assert "API call failed on attempt 1" in caplog.text
|
|
assert "API call failed on attempt 2" in caplog.text
|
|
assert "Max retries exceeded. Total attempts: 2" in caplog.text
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_keeps_every_failed_completion_on_exhaustion(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
responses = [
|
|
{"payload": "first"},
|
|
{"payload": "second"},
|
|
{"payload": "third"},
|
|
]
|
|
errors = [_validation_error(), _validation_error(), _validation_error()]
|
|
parse_errors: list[tuple[Exception, dict[str, Any]]] = []
|
|
completion_responses: list[Any] = []
|
|
updates: list[tuple[Any, Any]] = []
|
|
usage = {"tokens": 7}
|
|
calls = 0
|
|
parser_calls = 0
|
|
|
|
async def create(**_kwargs: Any) -> dict[str, str]:
|
|
nonlocal calls
|
|
response = responses[calls]
|
|
calls += 1
|
|
return response
|
|
|
|
def parser(**_kwargs: Any) -> Answer:
|
|
nonlocal parser_calls
|
|
error = errors[parser_calls]
|
|
parser_calls += 1
|
|
raise error
|
|
|
|
def reask(
|
|
kwargs: dict[str, Any], response: Any, exception: ValidationError
|
|
) -> dict[str, Any]:
|
|
assert response is responses[parser_calls - 1]
|
|
assert exception is errors[parser_calls - 1]
|
|
return {
|
|
**kwargs,
|
|
"messages": [
|
|
*kwargs["messages"],
|
|
{"role": "user", "content": f"retry-{parser_calls}"},
|
|
],
|
|
}
|
|
|
|
hooks = Hooks()
|
|
hooks.on(
|
|
"parse:error",
|
|
lambda error, **metadata: parse_errors.append((error, metadata)),
|
|
)
|
|
hooks.on(
|
|
"completion:response",
|
|
lambda response: completion_responses.append(response),
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=parser,
|
|
reask_handler=reask,
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry._initialize_usage",
|
|
lambda provider: usage if provider is Provider.OPENAI else None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.update_total_usage",
|
|
lambda response, total_usage: updates.append((response, total_usage)),
|
|
)
|
|
caplog.set_level(logging.DEBUG, logger="instructor.v2.retry")
|
|
|
|
with pytest.raises(InstructorRetryException) as exc_info:
|
|
await retry_async_v2(
|
|
func=create,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.JSON,
|
|
context=None,
|
|
max_retries=2,
|
|
args=(),
|
|
kwargs={"messages": [{"role": "user", "content": "first"}]},
|
|
strict=True,
|
|
hooks=hooks,
|
|
)
|
|
|
|
error = exc_info.value
|
|
assert calls == parser_calls == 3
|
|
assert completion_responses == responses
|
|
assert updates == [(response, usage) for response in responses]
|
|
assert [caught for caught, _ in parse_errors] == errors
|
|
assert [metadata for _, metadata in parse_errors] == [
|
|
{"attempt_number": 1, "max_attempts": 3, "is_last_attempt": False},
|
|
{"attempt_number": 2, "max_attempts": 3, "is_last_attempt": False},
|
|
{"attempt_number": 3, "max_attempts": 3, "is_last_attempt": True},
|
|
]
|
|
assert error.n_attempts == 3
|
|
assert error.last_completion is responses[-1]
|
|
assert error.total_usage is usage
|
|
assert error.messages == [
|
|
{"role": "user", "content": "first"},
|
|
{"role": "user", "content": "retry-1"},
|
|
{"role": "user", "content": "retry-2"},
|
|
{"role": "user", "content": "retry-3"},
|
|
]
|
|
assert error.create_kwargs is not None
|
|
assert error.create_kwargs["messages"] == error.messages
|
|
assert error.failed_attempts is not None
|
|
assert len(error.failed_attempts) == 3
|
|
assert [attempt.attempt_number for attempt in error.failed_attempts] == [1, 2, 3]
|
|
assert [attempt.exception for attempt in error.failed_attempts] == errors
|
|
assert [attempt.completion for attempt in error.failed_attempts] == responses
|
|
assert "Validation error on attempt 1" in caplog.text
|
|
assert "Validation error on attempt 2" in caplog.text
|
|
assert "Validation error on attempt 3" in caplog.text
|
|
assert "Max retries exceeded. Total attempts: 3" in caplog.text
|
|
|
|
|
|
class NoAttemptsAsyncRetrying(AsyncRetrying):
|
|
def __aiter__(self) -> NoAttemptsAsyncRetrying:
|
|
return self
|
|
|
|
async def __anext__(self) -> Any:
|
|
raise StopAsyncIteration
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_retry_async_v2_reports_an_empty_retry_policy_without_losing_context(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
usage = {"tokens": 0}
|
|
messages = [{"role": "user", "content": "unused"}]
|
|
|
|
async def create(**_kwargs: Any) -> dict[str, str]:
|
|
raise AssertionError("an empty retry policy must not call the provider")
|
|
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.RegistryValidationMixin.validate_mode_registration",
|
|
lambda _provider, _mode: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry.mode_registry.get_handlers",
|
|
lambda _provider, _mode: SimpleNamespace(
|
|
response_parser=lambda **_kwargs: Answer(value=1),
|
|
reask_handler=lambda **call: call["kwargs"],
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
"instructor.v2.core.retry._initialize_usage",
|
|
lambda provider: usage if provider is Provider.OPENAI else None,
|
|
)
|
|
caplog.set_level(logging.ERROR, logger="instructor.v2.retry")
|
|
|
|
with pytest.raises(InstructorRetryException, match=r"^Unknown error$") as exc_info:
|
|
await retry_async_v2(
|
|
func=create,
|
|
response_model=Answer,
|
|
provider=Provider.OPENAI,
|
|
mode=Mode.JSON,
|
|
context=None,
|
|
max_retries=NoAttemptsAsyncRetrying(),
|
|
args=(),
|
|
kwargs={"messages": messages},
|
|
strict=True,
|
|
hooks=None,
|
|
)
|
|
|
|
error = exc_info.value
|
|
assert error.n_attempts == 0
|
|
assert error.last_completion is None
|
|
assert error.total_usage is usage
|
|
assert error.messages == messages
|
|
assert error.create_kwargs == {"messages": messages}
|
|
assert error.failed_attempts == []
|
|
assert "Unexpected code path in retry_async_v2" in [
|
|
record.getMessage() for record in caplog.records
|
|
]
|