sgl-project--sglang/python/sglang/test/tool_call_test_runner.py

import json
from dataclasses import dataclass
from typing import List, Optional

import openai

from sglang.srt.utils import kill_process_tree
from sglang.test.test_utils import (
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
    DEFAULT_URL_FOR_TEST,
    ModelLaunchSettings,
    popen_launch_server,
)


@dataclass
class ToolCallTestParams:
    test_basic: bool = True
    test_auto: bool = True
    test_streaming: bool = True
    test_required: bool = True
    test_none: bool = True
    test_specific: bool = True
    test_strict: bool = True
    test_multiturn: bool = True
    test_thinking: bool = False  # model-specific, e.g. DeepSeek
    test_reasoning_usage: bool = False  # verify usage.reasoning_tokens > 0
    test_parallel: bool = True
    test_streaming_parallel: bool = True


@dataclass
class ToolCallTestResult:
    model: str
    passed: bool
    num_passed: int
    num_total: int
    failures: List[str]
    variant: Optional[str] = None


# ---- tool definitions ----

ADD_TOOL = {
    "type": "function",
    "function": {
        "name": "add",
        "description": "Compute the sum of two integers",
        "parameters": {
            "type": "object",
            "properties": {
                "a": {"type": "integer", "description": "First integer"},
                "b": {"type": "integer", "description": "Second integer"},
            },
            "required": ["a", "b"],
        },
    },
}

WEATHER_TOOL = {
    "type": "function",
    "function": {
        "name": "get_weather",
        "description": "Get the current weather for a city",
        "parameters": {
            "type": "object",
            "properties": {
                "city": {"type": "string", "description": "City name"},
            },
            "required": ["city"],
        },
    },
}

ADD_TOOL_STRICT = {
    "type": "function",
    "function": {**ADD_TOOL["function"], "strict": True},
}

WEATHER_TOOL_STRICT = {
    "type": "function",
    "function": {**WEATHER_TOOL["function"], "strict": True},
}


def _call(
    client,
    model,
    content,
    tools=None,
    tool_choice="required",
    temperature=0.1,
    **kwargs,
):
    """Single-turn tool call request. Defaults to ADD_TOOL_STRICT + required."""
    return client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": content}],
        tools=tools or [ADD_TOOL_STRICT],
        tool_choice=tool_choice,
        temperature=temperature,
        **kwargs,
    )


# ---- test cases ----


def _test_basic_format(client, model):
    """Format + field placement: tool_calls present, content empty, valid JSON args."""
    response = _call(client, model, "Compute 3 + 5")
    msg = response.choices[0].message
    assert msg.tool_calls and len(msg.tool_calls) > 0
    assert not msg.content, f"content should be empty, got: {msg.content}"
    tc = msg.tool_calls[0]
    assert tc.function.name == "add", f"expected 'add', got '{tc.function.name}'"
    assert isinstance(json.loads(tc.function.arguments), dict)
    assert response.choices[0].finish_reason == "tool_calls"


def _test_auto(client, model):
    """tool_choice=auto should populate tool_calls, not content (#17942)."""
    response = _call(client, model, "Compute 3 + 5", tool_choice="auto")
    msg = response.choices[0].message
    assert msg.tool_calls and len(msg.tool_calls) > 0
    assert not msg.content, f"content should be empty, got: {msg.content}"
    assert response.choices[0].finish_reason == "tool_calls"


def _test_streaming(client, model):
    """Streaming chunks should concatenate to valid JSON."""
    response = _call(client, model, "Compute 5 + 7", stream=True)
    chunks = list(response)
    assert len(chunks) > 0
    arg_fragments = []
    name = None
    for chunk in chunks:
        if chunk.choices[0].delta.tool_calls:
            tc = chunk.choices[0].delta.tool_calls[0]
            name = tc.function.name or name
            if tc.function.arguments:
                arg_fragments.append(tc.function.arguments)
    assert name == "add", f"expected 'add', got '{name}'"
    args = json.loads("".join(arg_fragments))
    assert "a" in args and "b" in args
    assert chunks[-1].choices[0].finish_reason == "tool_calls"


def _test_required(client, model):
    """tool_choice='required' must return a tool call even for unrelated queries."""
    response = _call(
        client,
        model,
        "What is the capital of France?",
        tools=[ADD_TOOL, WEATHER_TOOL],
    )
    assert response.choices[0].message.tool_calls


def _test_none(client, model):
    """tool_choice='none' must not return any tool call."""
    response = _call(client, model, "What is 1+1?", tool_choice="none")
    assert response.choices[0].message.tool_calls is None
    assert response.choices[0].finish_reason == "stop"


def _test_specific(client, model):
    """Specifying a function name should return that function."""
    response = _call(
        client,
        model,
        "What is the capital of France?",
        tools=[ADD_TOOL, WEATHER_TOOL],
        tool_choice={"type": "function", "function": {"name": "get_weather"}},
    )
    tc = response.choices[0].message.tool_calls
    assert tc and tc[0].function.name == "get_weather"


def _test_strict(client, model):
    """strict: true should enforce schema on arguments."""
    response = _call(client, model, "Compute 5 - 7")
    args = json.loads(response.choices[0].message.tool_calls[0].function.arguments)
    assert "a" in args and "b" in args


def _test_multiturn(client, model):
    """Pass tool result back, model should reply based on it."""
    # turn 1: get tool call
    messages = [{"role": "user", "content": "What is 3 + 5?"}]
    r1 = client.chat.completions.create(
        model=model,
        messages=messages,
        tools=[ADD_TOOL_STRICT],
        tool_choice="required",
        temperature=0.1,
    )
    tc = r1.choices[0].message.tool_calls[0]
    # turn 2: pass result back
    messages.append(r1.choices[0].message)
    messages.append(
        {
            "role": "tool",
            "tool_call_id": tc.id,
            "content": "8",
            "name": tc.function.name,
        }
    )
    r2 = client.chat.completions.create(
        model=model,
        messages=messages,
        tools=[ADD_TOOL],
        temperature=0.1,
    )
    assert "8" in (r2.choices[0].message.content or "")


def _test_thinking(client, model):
    """After tool result with thinking enabled, output should be in content not reasoning_content."""
    thinking_body = {"thinking": {"type": "enabled", "budget_tokens": 1024}}
    # turn 1
    messages = [{"role": "user", "content": "What is 3 + 5?"}]
    r1 = client.chat.completions.create(
        model=model,
        messages=messages,
        tools=[ADD_TOOL_STRICT],
        tool_choice="required",
        temperature=0.1,
        extra_body=thinking_body,
    )
    tc = r1.choices[0].message.tool_calls[0]
    # turn 2
    messages.append(r1.choices[0].message)
    messages.append(
        {
            "role": "tool",
            "tool_call_id": tc.id,
            "content": "8",
            "name": tc.function.name,
        }
    )
    r2 = client.chat.completions.create(
        model=model,
        messages=messages,
        tools=[ADD_TOOL],
        temperature=0.1,
        extra_body=thinking_body,
    )
    content = r2.choices[0].message.content or ""
    assert "8" in content, f"expected '8' in content, got: {content}"


def _test_reasoning_usage(client, model):
    """With thinking enabled, usage.reasoning_tokens should be reported as > 0."""
    thinking_body = {"thinking": {"type": "enabled", "budget_tokens": 1024}}
    response = client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": "What is 3 + 5?"}],
        tools=[ADD_TOOL_STRICT],
        tool_choice="required",
        temperature=0.1,
        extra_body=thinking_body,
    )
    usage = response.usage
    assert usage is not None, "usage should not be None"
    assert (
        usage.reasoning_tokens and usage.reasoning_tokens > 0
    ), f"expected reasoning_tokens > 0, got {usage.reasoning_tokens}"
    if usage.completion_tokens_details:
        detail_reasoning = usage.completion_tokens_details.get("reasoning_tokens", 0)
        assert (
            detail_reasoning > 0
        ), f"expected completion_tokens_details.reasoning_tokens > 0, got {detail_reasoning}"


def _test_parallel(client, model):
    """Single request should return multiple tool calls."""
    response = _call(
        client,
        model,
        "Please call both functions: use add to compute 3+5, and use get_weather to check the weather in Tokyo.",
        tools=[ADD_TOOL_STRICT, WEATHER_TOOL_STRICT],
        tool_choice="auto",
        temperature=0,
    )
    tc = response.choices[0].message.tool_calls
    assert tc and len(tc) >= 2, f"expected >= 2 tool calls, got {len(tc) if tc else 0}"


def _test_streaming_parallel(client, model):
    """Streaming with tool_choice=auto should return multiple tool calls."""
    response = _call(
        client,
        model,
        "What is 3+5 and what is the weather in Tokyo?",
        tools=[ADD_TOOL, WEATHER_TOOL],
        tool_choice="auto",
        stream=True,
    )
    # collect tool calls from streaming chunks
    tool_calls = {}
    for chunk in response:
        if not chunk.choices[0].delta.tool_calls:
            continue
        for tc in chunk.choices[0].delta.tool_calls:
            idx = tc.index
            if idx not in tool_calls:
                tool_calls[idx] = {"name": "", "arguments": ""}
            if tc.function.name:
                tool_calls[idx]["name"] = tc.function.name
            if tc.function.arguments:
                tool_calls[idx]["arguments"] += tc.function.arguments
    assert len(tool_calls) >= 2, f"expected >= 2 tool calls, got {len(tool_calls)}"
    for idx, tc in tool_calls.items():
        assert tc["name"], f"tool call {idx} missing function name"
        args = json.loads(tc["arguments"])
        assert isinstance(args, dict), f"tool call {idx} arguments not a dict"


_TESTS = [
    ("basic_format", _test_basic_format, "test_basic"),
    # ("auto", _test_auto, "test_auto"),
    ("streaming", _test_streaming, "test_streaming"),
    ("required", _test_required, "test_required"),
    ("none", _test_none, "test_none"),
    ("specific", _test_specific, "test_specific"),
    ("strict", _test_strict, "test_strict"),
    ("multiturn", _test_multiturn, "test_multiturn"),
    ("thinking", _test_thinking, "test_thinking"),
    # ("reasoning_usage", _test_reasoning_usage, "test_reasoning_usage"),
    ("parallel", _test_parallel, "test_parallel"),
    # ("streaming_parallel", _test_streaming_parallel, "test_streaming_parallel"),
]


# ---- runner ----


def run_tool_call_test(
    model: ModelLaunchSettings,
    params: ToolCallTestParams,
    base_url: Optional[str] = None,
) -> ToolCallTestResult:
    """Launch server, run enabled test cases, return results."""
    base_url = base_url or DEFAULT_URL_FOR_TEST

    print(f"\n{'=' * 60}")
    print(f"Running TOOL CALL test for {model.model_path}")
    if model.variant:
        print(f"  Variant: {model.variant}")
    print(f"{'=' * 60}\n")

    process = None
    try:
        process = popen_launch_server(
            model.model_path,
            base_url,
            other_args=model.extra_args,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            env=model.env,
        )
        client = openai.Client(api_key="sk-test", base_url=base_url + "/v1")

        passed_list = []
        failed_list = []

        for name, fn, flag in _TESTS:
            if not getattr(params, flag):
                continue
            try:
                fn(client, model.model_path)
                passed_list.append(name)
                print(f"  PASS: {name}")
            except Exception as e:
                failed_list.append(f"{name}: {e}")
                print(f"  FAIL: {name}: {e}")

        total = len(passed_list) + len(failed_list)
        print(f"\n  Result: {len(passed_list)}/{total} passed")

        return ToolCallTestResult(
            model=model.model_path,
            passed=len(failed_list) == 0,
            num_passed=len(passed_list),
            num_total=total,
            failures=failed_list,
            variant=model.variant,
        )

    except Exception as e:
        print(f"  Server launch failed: {e}")
        return ToolCallTestResult(
            model=model.model_path,
            passed=False,
            num_passed=0,
            num_total=0,
            failures=[f"Server launch failed: {e}"],
            variant=model.variant,
        )

    finally:
        if process:
            kill_process_tree(process.pid)