chore: import upstream snapshot with attribution

tests/sandbox/test_token_truncation.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from agents.sandbox.util.token_truncation import (
+    TruncationPolicy,
+    approx_bytes_for_tokens,
+    approx_token_count,
+    approx_tokens_from_byte_count,
+    format_truncation_marker,
+    formatted_truncate_text,
+    formatted_truncate_text_with_token_count,
+    removed_units_for_source,
+    split_budget,
+    split_string,
+    truncate_text,
+    truncate_with_byte_estimate,
+    truncate_with_token_budget,
+)
+
+
+def test_truncation_policy_clamps_negative_limits_and_converts_budgets() -> None:
+    byte_policy = TruncationPolicy.bytes(-10)
+    token_policy = TruncationPolicy.tokens(-2)
+
+    assert byte_policy.limit == 0
+    assert byte_policy.token_budget() == 0
+    assert byte_policy.byte_budget() == 0
+    assert token_policy.limit == 0
+    assert token_policy.token_budget() == 0
+    assert token_policy.byte_budget() == 0
+
+
+def test_formatted_truncate_text_returns_short_content_unchanged() -> None:
+    assert formatted_truncate_text("short", TruncationPolicy.bytes(20)) == "short"
+
+
+def test_formatted_truncate_text_adds_line_count_when_truncated() -> None:
+    result = formatted_truncate_text("alpha\nbeta\ngamma", TruncationPolicy.bytes(8))
+
+    assert result.startswith("Total output lines: 3\n\n")
+    assert "chars truncated" in result
+
+
+def test_formatted_truncate_text_with_token_count_handles_none_and_short_content() -> None:
+    assert formatted_truncate_text_with_token_count("short", None) == ("short", None)
+    assert formatted_truncate_text_with_token_count("short", 10) == ("short", None)
+
+
+def test_formatted_truncate_text_with_token_count_reports_original_count() -> None:
+    result, original_token_count = formatted_truncate_text_with_token_count("abcdefghi", 1)
+
+    assert result.startswith("Total output lines: 1\n\n")
+    assert "tokens truncated" in result
+    assert original_token_count == approx_token_count("abcdefghi")
+
+
+def test_truncate_text_dispatches_byte_and_token_modes() -> None:
+    assert truncate_text("abcdef", TruncationPolicy.bytes(4)).startswith("a")
+    assert "tokens truncated" in truncate_text("abcdefghi", TruncationPolicy.tokens(1))
+
+
+def test_truncate_with_token_budget_handles_empty_and_short_content() -> None:
+    assert truncate_with_token_budget("", TruncationPolicy.tokens(1)) == ("", None)
+    assert truncate_with_token_budget("abc", TruncationPolicy.tokens(1)) == ("abc", None)
+
+
+def test_truncate_with_byte_estimate_handles_empty_zero_and_short_content() -> None:
+    assert truncate_with_byte_estimate("", TruncationPolicy.bytes(0)) == ""
+    assert "chars truncated" in truncate_with_byte_estimate("abc", TruncationPolicy.bytes(0))
+    assert truncate_with_byte_estimate("abc", TruncationPolicy.bytes(10)) == "abc"
+
+
+def test_split_string_preserves_utf8_boundaries() -> None:
+    removed_chars, prefix, suffix = split_string("aあbいc", 2, 4)
+
+    assert prefix == "a"
+    assert suffix == "いc"
+    assert removed_chars == 2
+
+
+def test_split_string_handles_empty_content() -> None:
+    assert split_string("", 10, 10) == (0, "", "")
+
+
+def test_formatting_and_estimate_helpers() -> None:
+    byte_policy = TruncationPolicy.bytes(8)
+    token_policy = TruncationPolicy.tokens(2)
+
+    assert "chars truncated" in format_truncation_marker(byte_policy, 3)
+    assert "tokens truncated" in format_truncation_marker(token_policy, 2)
+    assert split_budget(5) == (2, 3)
+    assert removed_units_for_source(byte_policy, removed_bytes=10, removed_chars=4) == 4
+    assert removed_units_for_source(token_policy, removed_bytes=9, removed_chars=4) == 3
+    assert approx_token_count("abcde") == 2
+    assert approx_bytes_for_tokens(-1) == 0
+    assert approx_tokens_from_byte_count(0) == 0
+    assert approx_tokens_from_byte_count(5) == 2