wehub-resource-sync
475 lines
18 KiB
Python
475 lines
18 KiB
Python
"""Tests for tool_result_meta normalization logic."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import pytest
|
|
from langchain_core.messages import ToolMessage
|
|
from langgraph.types import Command
|
|
|
|
from deerflow.agents.middlewares.tool_result_meta import (
|
|
TOOL_META_KEY,
|
|
ToolResultMeta,
|
|
normalize_tool_message,
|
|
normalize_tool_result,
|
|
stamp_exception_meta,
|
|
)
|
|
|
|
|
|
def _make_msg(content: str, *, status: str = "success", kwargs: dict[str, object] | None = None) -> ToolMessage:
|
|
return ToolMessage(
|
|
content=content,
|
|
tool_call_id="tc-1",
|
|
name="test_tool",
|
|
status=status,
|
|
additional_kwargs=kwargs or {},
|
|
)
|
|
|
|
|
|
def _meta(msg: ToolMessage) -> dict[str, object]:
|
|
return msg.additional_kwargs[TOOL_META_KEY]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Already-stamped messages are not overwritten
|
|
|
|
|
|
def test_existing_meta_is_preserved():
|
|
existing = {"status": "success", "source": "custom"}
|
|
msg = _make_msg("hello", kwargs={TOOL_META_KEY: existing})
|
|
result = normalize_tool_message(msg)
|
|
assert result.additional_kwargs[TOOL_META_KEY] is existing
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Error prefix (tool_return path)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"snippet,expected_type",
|
|
[
|
|
("Error: 401 unauthorized", "auth"),
|
|
("Error: permission denied for path", "permission"),
|
|
("Error: 429 rate limit exceeded", "rate_limited"),
|
|
("Error: connection timeout", "transient"),
|
|
("Error: tool not configured", "config"),
|
|
("Error: no results found for query", "no_results"),
|
|
("Error: file not found", "not_found"),
|
|
("Error: internal error 500", "internal"),
|
|
("Error: something completely unexpected happened", "unknown"),
|
|
],
|
|
)
|
|
def test_error_prefix_classification(snippet: str, expected_type: str):
|
|
msg = _make_msg(snippet, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == expected_type
|
|
assert m["source"] == "tool_return"
|
|
|
|
|
|
def test_auth_error_is_unrecoverable_and_stop():
|
|
msg = _make_msg("Error: invalid api key", status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["recoverable_by_model"] is False
|
|
assert m["recommended_next_action"] == "stop"
|
|
|
|
|
|
def test_no_api_key_is_config_not_auth():
|
|
# Distinguish "missing key" (config) from "wrong key" (auth):
|
|
# - auth rule keyword: "invalid api key" (key provided but rejected)
|
|
# - config rule keyword: "no api key" (key not set at all)
|
|
# The two phrases do not overlap, so rule order does not affect this particular
|
|
# case. This test documents the semantic distinction — a missing API key is a
|
|
# configuration issue, not an authentication failure.
|
|
msg = _make_msg("Error: no api key configured", status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["error_type"] == "config", "missing API key is a config issue, not auth"
|
|
assert m["recommended_next_action"] == "stop"
|
|
assert m["recoverable_by_model"] is False
|
|
|
|
|
|
def test_rate_limited_error_suggests_summarize():
|
|
msg = _make_msg("Error: rate limited", status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["recommended_next_action"] == "summarize"
|
|
assert m["recoverable_by_model"] is False
|
|
|
|
|
|
def test_no_results_suggests_rewrite_query():
|
|
msg = _make_msg("Error: no results found", status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["recoverable_by_model"] is True
|
|
assert m["recommended_next_action"] == "rewrite_query"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Non-standard error path (status="error", no "Error:" prefix)
|
|
|
|
|
|
def test_nonstd_error_status_classifies_from_content():
|
|
# Tools that return status="error" without the "Error:" prefix are tool_return, not exception.
|
|
# Actual exceptions are pre-stamped by stamp_exception_meta and exit normalize_tool_message early.
|
|
msg = _make_msg("ConnectionError: connection refused", status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["source"] == "tool_return"
|
|
assert m["error_type"] == "transient"
|
|
|
|
|
|
def test_nonstd_error_status_timeout_content():
|
|
msg = _make_msg("timeout occurred", status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["source"] == "tool_return"
|
|
assert m["error_type"] == "transient"
|
|
|
|
|
|
def test_nonstd_error_status_json_classifies_from_error_field():
|
|
# When status="error" and content is JSON, classification must use only the "error"
|
|
# field value — not keywords that appear in other fields like "query".
|
|
content = '{"error": "api limit exceeded", "query": "connection test timeout"}'
|
|
msg = _make_msg(content, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["source"] == "tool_return"
|
|
# "connection" and "timeout" in query must not trigger transient; the error field
|
|
# "api limit exceeded" doesn't match any rule → unknown.
|
|
assert m["error_type"] == "unknown"
|
|
|
|
|
|
def test_nonstd_error_status_json_no_error_key_is_unknown():
|
|
# JSON with no 'error' key must NOT be classified from other field values.
|
|
# Previously, {"message": "connection refused"} would be passed to _classify_error_text
|
|
# and match the transient rule via "connection"; now the full JSON is treated as unknown.
|
|
content = '{"message": "connection refused"}'
|
|
msg = _make_msg(content, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "unknown", "JSON dict with no 'error' key must resolve to unknown"
|
|
|
|
|
|
def test_nonstd_error_status_json_no_error_key_with_dangerous_field_is_unknown():
|
|
# {"user_id": 401} previously triggered auth stop; must now be unknown.
|
|
content = '{"user_id": 401, "action": "login"}'
|
|
msg = _make_msg(content, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "unknown"
|
|
assert m["recommended_next_action"] != "stop", "spurious 401 in non-error field must not trigger stop"
|
|
|
|
|
|
def test_nonstd_error_status_non_json_content_still_classified():
|
|
# Plain text (not JSON) with status="error" must still be classified from content.
|
|
content = "connection refused: remote host unreachable"
|
|
msg = _make_msg(content, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "transient"
|
|
|
|
|
|
def test_json_error_field_dict_is_serialized_not_repr():
|
|
# FastAPI-style: {"error": [{"loc": ["body"], "msg": "missing required field"}]}
|
|
# str() would produce Python repr containing 'missing required' → config → stop.
|
|
# json.dumps produces a clean JSON string that should not spuriously match.
|
|
import json as _json
|
|
|
|
error_val = [{"loc": ["body"], "msg": "missing required field"}]
|
|
content = _json.dumps({"error": error_val})
|
|
msg = _make_msg(content, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
# The JSON-serialized error list contains "missing required field" which IS in the config rule.
|
|
# This is correct classification (the validation error IS a config-class problem).
|
|
# The key requirement is that we're classifying from the error field value, not repr noise.
|
|
assert m["error_type"] == "config"
|
|
|
|
|
|
def test_no_results_success_response_is_partial_success():
|
|
# Tools that return status="success" with "no results found" content must be treated as
|
|
# partial_success so ToolProgressMiddleware can detect stagnation.
|
|
for phrase in ("no results found", "No Content Found here", "no images found for query"):
|
|
msg = _make_msg(phrase, status="success")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "partial_success", f"expected partial_success for: {phrase!r}"
|
|
assert m["recommended_next_action"] == "rewrite_query"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Partial success detection
|
|
|
|
|
|
def test_partial_markers_detected():
|
|
for marker in ("partial results available", "limited results returned", "truncated output", "results may be incomplete"):
|
|
msg = _make_msg(f"Here are some {marker} from the search.", status="success")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "partial_success", f"expected partial_success for: {marker}"
|
|
assert m["recommended_next_action"] == "rewrite_query"
|
|
|
|
|
|
def test_short_terse_success_is_not_partial():
|
|
# "Ok." is a valid, complete success response from mutation tools like write_file/str_replace.
|
|
# partial_success is now gated only on _PARTIAL_MARKERS, not content length.
|
|
msg = _make_msg("Ok.", status="success")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "success"
|
|
assert m["source"] == "content_analysis"
|
|
|
|
|
|
def test_empty_content_is_not_partial():
|
|
# Empty content has no partial markers, so it falls through to success.
|
|
msg = _make_msg("", status="success")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
# Empty content falls through to success (no partial markers)
|
|
assert m["status"] == "success"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Success path
|
|
|
|
|
|
def test_substantial_content_is_success():
|
|
content = "A" * 200
|
|
msg = _make_msg(content, status="success")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "success"
|
|
assert m["source"] == "content_analysis"
|
|
assert m["recommended_next_action"] == "continue"
|
|
assert m["error_type"] is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ToolResultMeta dataclass round-trip
|
|
|
|
|
|
def test_tool_result_meta_from_dict():
|
|
msg = _make_msg("A" * 200)
|
|
result = normalize_tool_message(msg)
|
|
meta_dict = _meta(result)
|
|
meta = ToolResultMeta(**meta_dict)
|
|
assert meta.status == "success"
|
|
assert meta.error_type is None
|
|
assert meta.recommended_next_action == "continue"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# stamp_exception_meta
|
|
|
|
|
|
def test_stamp_exception_meta_classifies_from_exc_info_not_content():
|
|
# Content says "no results" but exc_info says "connection refused" —
|
|
# stamp_exception_meta must use exc_info, producing transient, not no_results.
|
|
msg = _make_msg("Error: no results found", status="error")
|
|
result = stamp_exception_meta(msg, "ConnectionError: connection refused")
|
|
m = _meta(result)
|
|
assert m["source"] == "exception"
|
|
assert m["error_type"] == "transient"
|
|
|
|
|
|
def test_stamp_exception_meta_overwrites_existing_meta():
|
|
pre_existing = {TOOL_META_KEY: {"source": "tool_return", "error_type": "unknown"}}
|
|
msg = _make_msg("Error: no results found", status="error", kwargs=pre_existing)
|
|
result = stamp_exception_meta(msg, "PermissionError: access denied")
|
|
m = _meta(result)
|
|
assert m["source"] == "exception"
|
|
assert m["error_type"] == "permission"
|
|
|
|
|
|
def test_stamp_exception_meta_preserves_other_additional_kwargs():
|
|
msg = _make_msg("irrelevant", status="error", kwargs={"subagent_status": "running"})
|
|
result = stamp_exception_meta(msg, "TimeoutError: timed out")
|
|
assert result.additional_kwargs["subagent_status"] == "running"
|
|
assert TOOL_META_KEY in result.additional_kwargs
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# normalize_tool_result handles Command wrappers
|
|
|
|
|
|
def test_normalize_tool_result_passthrough_command():
|
|
cmd = Command(goto="next_node")
|
|
result = normalize_tool_result(cmd)
|
|
assert result is cmd
|
|
|
|
|
|
def test_normalize_tool_result_stamps_tool_message():
|
|
msg = _make_msg("A" * 200)
|
|
result = normalize_tool_result(msg)
|
|
assert isinstance(result, ToolMessage)
|
|
assert TOOL_META_KEY in result.additional_kwargs
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# JSON-wrapped error detection
|
|
|
|
|
|
def test_normalize_json_error_config_classified_as_error():
|
|
content = '{"error": "BRAVE_SEARCH_API_KEY is not configured", "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "config"
|
|
assert m["source"] == "tool_return"
|
|
|
|
|
|
def test_normalize_json_error_no_results_classified_correctly():
|
|
content = '{"error": "No results found", "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "no_results"
|
|
assert m["recoverable_by_model"] is True
|
|
|
|
|
|
def test_normalize_json_null_error_not_treated_as_error():
|
|
content = '{"error": null, "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] != "error"
|
|
|
|
|
|
def test_normalize_json_no_error_key_not_treated_as_error():
|
|
content = '{"results": [{"title": "page one", "url": "https://example.com/one", "content": "summary one"}], "total": 1}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "success"
|
|
|
|
|
|
def test_normalize_malformed_json_not_treated_as_error():
|
|
content = '{"error": "broken json'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] != "error"
|
|
|
|
|
|
def test_normalize_json_error_with_leading_whitespace():
|
|
content = ' {"error": "No results found", "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "no_results"
|
|
|
|
|
|
def test_normalize_json_numeric_error_classified_correctly():
|
|
content = '{"error": 404, "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "not_found"
|
|
|
|
|
|
def test_normalize_json_zero_error_not_treated_as_error():
|
|
content = '{"error": 0, "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] != "error"
|
|
|
|
|
|
def test_normalize_json_false_error_not_treated_as_error():
|
|
content = '{"error": false, "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] != "error"
|
|
|
|
|
|
def test_normalize_json_boolean_true_error_classified_as_unknown():
|
|
"""Boolean True in the error field means 'an error occurred' and must be classified.
|
|
|
|
str(True) = "True" which matches no keyword rule, so the result is error/unknown.
|
|
This is intentional: a boolean True error is a real error with no further detail.
|
|
"""
|
|
content = '{"error": true, "query": "test"}'
|
|
msg = _make_msg(content)
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == "unknown" # str(True)="True" matches no keyword rule
|
|
assert m["recoverable_by_model"] is True
|
|
assert m["recommended_next_action"] == "try_alternative"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# M2 regression: semantic-zero error strings must NOT be treated as errors
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"error_value",
|
|
["none", "None", "NONE", "null", "Null", "false", "False", "no", "ok", "success", "n/a", ""],
|
|
)
|
|
def test_normalize_json_semantic_zero_error_string_not_treated_as_error(error_value: str):
|
|
"""M2 regression: error field containing a conventional 'no-error' string must not trigger misclassification.
|
|
|
|
Tools sometimes return {"error": "none", "results": [...]} on success.
|
|
The string "none" is truthy in Python, so without this guard the message
|
|
would have been classified as error (unknown), inflating stagnation counters.
|
|
|
|
Note: the empty-string case ("") is handled by the falsy guard (`if not error: return None`)
|
|
in _extract_json_error_text rather than by _SEMANTIC_ZERO_ERROR_STRINGS. Both paths produce
|
|
the same outcome (no misclassification), but the mechanism differs from the other cases here.
|
|
"""
|
|
content = json.dumps({"error": error_value, "results": ["item1", "item2", "item3"]})
|
|
msg = _make_msg(content, status="success")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] != "error", f'error="{error_value}" should not be treated as an error; got status={m["status"]!r}'
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Numeric keyword word-boundary matching (_match_keyword)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"content, expected_error_type",
|
|
[
|
|
# Positive: numeric code at a word boundary → correct classification
|
|
("Error: HTTP 500 Internal Server Error", "internal"),
|
|
("Error: returned status 500", "internal"),
|
|
("Error: 401 Unauthorized", "auth"),
|
|
("Error: 404 Not Found", "not_found"),
|
|
# Negative: numeric code embedded inside a longer token → must resolve to "unknown".
|
|
# Use exact "unknown" assertions so any future rule additions that accidentally
|
|
# absorb these strings are caught (a broad exclusion list would miss new matches).
|
|
("Error: took 500ms to respond", "unknown"),
|
|
("Error: query returned 4010 rows", "unknown"),
|
|
("Error: batch 401A failed", "unknown"),
|
|
("Error: response contained 5000 items", "unknown"),
|
|
],
|
|
)
|
|
def test_numeric_keyword_word_boundary(content: str, expected_error_type: str):
|
|
"""Numeric HTTP codes must match only at word boundaries to avoid false positives.
|
|
|
|
'500ms', '4010', '401A', '5000' must not trigger internal/auth/not_found rules.
|
|
Negative cases assert exactly 'unknown' so future rule additions that accidentally
|
|
absorb these strings are caught — a broad exclusion-list assertion would not be.
|
|
"""
|
|
msg = _make_msg(content, status="error")
|
|
result = normalize_tool_message(msg)
|
|
m = _meta(result)
|
|
assert m["status"] == "error"
|
|
assert m["error_type"] == expected_error_type, f"{content!r} → expected {expected_error_type!r}, got {m['error_type']!r}"
|