wehub-resource-sync
612 lines
21 KiB
Python
612 lines
21 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Cross-API render parity tests.
|
|
|
|
Verifies that the chat completion input path (parse_chat_input_to_harmony_message)
|
|
and the responses API input path (response_input_to_harmony) produce identical
|
|
Harmony messages and identical rendered token sequences when given equivalent
|
|
conversation representations.
|
|
|
|
The chat completion API encodes reasoning and tool calls as fields on a single
|
|
assistant message dict; the responses API encodes them as separate typed items
|
|
in request.input. Both paths must converge on the same Harmony message list and
|
|
therefore the same rendered prompt.
|
|
|
|
Each test:
|
|
1. Builds Harmony messages from each path for a single message or sequence.
|
|
2. Asserts message-level properties (role, channel, recipient, content)
|
|
using verify_harmony_messages.
|
|
3. Asserts that render_for_completion produces identical token sequences.
|
|
"""
|
|
|
|
from openai.types.responses import ResponseFunctionToolCall
|
|
|
|
from tests.entrypoints.openai.utils import verify_harmony_messages
|
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
|
get_encoding,
|
|
get_system_message,
|
|
parse_chat_input_to_harmony_message,
|
|
render_for_completion,
|
|
)
|
|
from vllm.entrypoints.openai.responses.harmony import (
|
|
response_input_to_harmony,
|
|
response_previous_input_to_harmony,
|
|
)
|
|
|
|
# Use a fixed date so the system message is deterministic across both paths.
|
|
_DATE = "2025-01-01"
|
|
|
|
|
|
def _system():
|
|
return get_system_message(start_date=_DATE)
|
|
|
|
|
|
class TestResponseInputToHarmonyRenderParity:
|
|
"""Each test drives the same conversation through both APIs and asserts
|
|
identical Harmony messages and rendered token sequences."""
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Single-message cases
|
|
# -----------------------------------------------------------------------
|
|
|
|
def test_developer_message(self):
|
|
"""Both APIs must render developer messages identically using
|
|
DeveloperContent (with the '# Instructions' header)."""
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{"role": "developer", "content": "Be concise."}
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "message",
|
|
"role": "developer",
|
|
"content": "Be concise.",
|
|
},
|
|
prev_responses=[],
|
|
)
|
|
]
|
|
|
|
expected = [{"role": "developer", "instructions": "Be concise."}]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_user_message(self):
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{"role": "user", "content": "What's the weather in Paris?"}
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "message",
|
|
"role": "user",
|
|
"content": "What's the weather in Paris?",
|
|
},
|
|
prev_responses=[],
|
|
)
|
|
]
|
|
|
|
expected = [{"role": "user", "content": "What's the weather in Paris?"}]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_assistant_final_message(self):
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{"role": "assistant", "content": "It is 18°C in Paris."}
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": "It is 18°C in Paris.",
|
|
},
|
|
prev_responses=[],
|
|
)
|
|
]
|
|
|
|
expected = [
|
|
{"role": "assistant", "channel": "final", "content": "It is 18°C in Paris."}
|
|
]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_reasoning_item(self):
|
|
# Chat path: assistant message with only a reasoning field and no content.
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": "I should call get_weather.",
|
|
"content": "",
|
|
}
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "reasoning",
|
|
"content": [
|
|
{"type": "reasoning_text", "text": "I should call get_weather."}
|
|
],
|
|
},
|
|
prev_responses=[],
|
|
)
|
|
]
|
|
|
|
expected = [
|
|
{
|
|
"role": "assistant",
|
|
"channel": "analysis",
|
|
"content": "I should call get_weather.",
|
|
}
|
|
]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_function_call(self):
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "function_call",
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
prev_responses=[],
|
|
)
|
|
]
|
|
|
|
expected = [
|
|
{
|
|
"role": "assistant",
|
|
"channel": "commentary",
|
|
"recipient": "functions.get_weather",
|
|
"content": '{"location": "Paris"}',
|
|
"content_type": "json",
|
|
}
|
|
]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_tool_output(self):
|
|
prev_call = ResponseFunctionToolCall(
|
|
id="fc_1",
|
|
call_id="call_1",
|
|
name="get_weather",
|
|
arguments='{"location": "Paris"}',
|
|
type="function_call",
|
|
)
|
|
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."},
|
|
tool_id_names={"call_1": "get_weather"},
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_1",
|
|
"output": "18°C, clear skies.",
|
|
},
|
|
prev_responses=[prev_call],
|
|
)
|
|
]
|
|
|
|
expected = [
|
|
{
|
|
"role": "tool",
|
|
"author_name": "functions.get_weather",
|
|
"channel": "commentary",
|
|
"recipient": "assistant",
|
|
"content": "18°C, clear skies.",
|
|
}
|
|
]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
# -----------------------------------------------------------------------
|
|
# Combined and multi-turn cases
|
|
# -----------------------------------------------------------------------
|
|
|
|
def test_reasoning_combined_with_function_call(self):
|
|
"""Chat API packs reasoning + tool_calls into one dict; responses API
|
|
represents them as two separate items. Both must produce the same two
|
|
Harmony messages in the same order: analysis then commentary."""
|
|
chat_msgs = parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": "I should get the weather for Paris.",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
resp_msgs = [
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "reasoning",
|
|
"content": [
|
|
{
|
|
"type": "reasoning_text",
|
|
"text": "I should get the weather for Paris.",
|
|
}
|
|
],
|
|
},
|
|
prev_responses=[],
|
|
),
|
|
response_input_to_harmony(
|
|
{
|
|
"type": "function_call",
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
prev_responses=[],
|
|
),
|
|
]
|
|
|
|
expected = [
|
|
{
|
|
"role": "assistant",
|
|
"channel": "analysis",
|
|
"content": "I should get the weather for Paris.",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"channel": "commentary",
|
|
"recipient": "functions.get_weather",
|
|
"content": '{"location": "Paris"}',
|
|
"content_type": "json",
|
|
},
|
|
]
|
|
verify_harmony_messages(chat_msgs, expected)
|
|
verify_harmony_messages(resp_msgs, expected)
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_full_multi_turn_tool_call_conversation(self):
|
|
"""Full conversation: user -> reasoning + tool_call -> tool_output -> final.
|
|
|
|
Both APIs must render the complete conversation to identical token sequences.
|
|
This exercises the entire input pipeline including all message types and
|
|
the Rust harmony encoder.
|
|
"""
|
|
prev_call = ResponseFunctionToolCall(
|
|
id="fc_1",
|
|
call_id="call_1",
|
|
name="get_weather",
|
|
arguments='{"location": "Paris"}',
|
|
type="function_call",
|
|
)
|
|
|
|
# --- Chat completion API path ---
|
|
tool_id_names = {"call_1": "get_weather"}
|
|
chat_msgs = []
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{"role": "user", "content": "What's the weather in Paris?"}
|
|
)
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": "I should call get_weather for Paris.",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."},
|
|
tool_id_names=tool_id_names,
|
|
)
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"content": "It is currently 18°C in Paris with clear skies.",
|
|
}
|
|
)
|
|
|
|
# --- Responses API path ---
|
|
resp_input = [
|
|
{
|
|
"type": "message",
|
|
"role": "user",
|
|
"content": "What's the weather in Paris?",
|
|
},
|
|
{
|
|
"type": "reasoning",
|
|
"content": [
|
|
{
|
|
"type": "reasoning_text",
|
|
"text": "I should call get_weather for Paris.",
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"type": "function_call",
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_1",
|
|
"output": "18°C, clear skies.",
|
|
},
|
|
{
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"content": "It is currently 18°C in Paris with clear skies.",
|
|
},
|
|
]
|
|
resp_msgs = [
|
|
response_input_to_harmony(item, prev_responses=[prev_call])
|
|
for item in resp_input
|
|
]
|
|
|
|
assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
|
|
[_system()] + resp_msgs
|
|
)
|
|
|
|
def test_multi_turn_two_tool_calls_with_reasoning_between(self):
|
|
"""Validates parity for a chain of two tool calls, each with its own
|
|
reasoning trace. Reasoning traces in between commentary-channel tool
|
|
calls must survive as analysis-channel messages in both paths.
|
|
"""
|
|
first_reasoning = "I need current weather first."
|
|
second_reasoning = "Now I need the weekly forecast."
|
|
|
|
prev_call_1 = ResponseFunctionToolCall(
|
|
id="fc_1",
|
|
call_id="call_1",
|
|
name="get_weather",
|
|
arguments='{"location": "Paris"}',
|
|
type="function_call",
|
|
)
|
|
prev_call_2 = ResponseFunctionToolCall(
|
|
id="fc_2",
|
|
call_id="call_2",
|
|
name="get_forecast",
|
|
arguments='{"location": "Paris", "days": 7}',
|
|
type="function_call",
|
|
)
|
|
|
|
# --- Chat completion API path ---
|
|
tool_id_names = {"call_1": "get_weather", "call_2": "get_forecast"}
|
|
chat_msgs = []
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{"role": "user", "content": "What's the weather and forecast for Paris?"}
|
|
)
|
|
# First reasoning + tool call
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": first_reasoning,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"function": {
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{"role": "tool", "tool_call_id": "call_1", "content": "18°C, clear skies."},
|
|
tool_id_names=tool_id_names,
|
|
)
|
|
# Second reasoning + tool call
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": second_reasoning,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_2",
|
|
"function": {
|
|
"name": "get_forecast",
|
|
"arguments": '{"location": "Paris", "days": 7}',
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
chat_msgs += parse_chat_input_to_harmony_message(
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": "call_2",
|
|
"content": "Mon 17°C, Tue 19°C, Wed 16°C",
|
|
},
|
|
tool_id_names=tool_id_names,
|
|
)
|
|
|
|
# --- Responses API path ---
|
|
prev_responses = [prev_call_1, prev_call_2]
|
|
resp_input = [
|
|
{
|
|
"type": "message",
|
|
"role": "user",
|
|
"content": "What's the weather and forecast for Paris?",
|
|
},
|
|
# First reasoning + tool call
|
|
{
|
|
"type": "reasoning",
|
|
"content": [{"type": "reasoning_text", "text": first_reasoning}],
|
|
},
|
|
{
|
|
"type": "function_call",
|
|
"name": "get_weather",
|
|
"arguments": '{"location": "Paris"}',
|
|
},
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_1",
|
|
"output": "18°C, clear skies.",
|
|
},
|
|
# Second reasoning + tool call
|
|
{
|
|
"type": "reasoning",
|
|
"content": [
|
|
{
|
|
"type": "reasoning_text",
|
|
"text": second_reasoning,
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"type": "function_call",
|
|
"name": "get_forecast",
|
|
"arguments": '{"location": "Paris", "days": 7}',
|
|
},
|
|
{
|
|
"type": "function_call_output",
|
|
"call_id": "call_2",
|
|
"output": "Mon 17°C, Tue 19°C, Wed 16°C",
|
|
},
|
|
]
|
|
resp_msgs = [
|
|
response_input_to_harmony(item, prev_responses=prev_responses)
|
|
for item in resp_input
|
|
]
|
|
|
|
chat_completion_tokens = render_for_completion([_system()] + chat_msgs)
|
|
responses_tokens = render_for_completion([_system()] + resp_msgs)
|
|
|
|
assert chat_completion_tokens == responses_tokens
|
|
|
|
rendered_prompt = get_encoding().decode(chat_completion_tokens)
|
|
assert first_reasoning in rendered_prompt
|
|
assert second_reasoning in rendered_prompt
|
|
|
|
def test_completed_turns_drop_reasoning(self):
|
|
"""Validates that reasoning from completed turns is dropped, while
|
|
reasoning from the current in-progress tool-call turn is preserved
|
|
in both chat completions and responses previous_input_messages."""
|
|
first_turn_reasoning = "FIRST_TURN_REASONING"
|
|
second_turn_reasoning = "SECOND_TURN_REASONING"
|
|
|
|
chat_completion_msgs = []
|
|
for chat_message in [
|
|
{"role": "user", "content": "What is 2+2?"},
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": first_turn_reasoning,
|
|
"content": "The answer is 4.",
|
|
},
|
|
{"role": "user", "content": "Now what is 3+3?"},
|
|
{
|
|
"role": "assistant",
|
|
"reasoning": second_turn_reasoning,
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_1",
|
|
"function": {
|
|
"name": "calc",
|
|
"arguments": '{"a":3,"b":3}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
]:
|
|
chat_completion_msgs.extend(
|
|
parse_chat_input_to_harmony_message(chat_message)
|
|
)
|
|
|
|
responses_prev_input_msgs = []
|
|
for responses_message in [
|
|
{
|
|
"author": {"role": "user"},
|
|
"content": [{"type": "text", "text": "What is 2+2?"}],
|
|
},
|
|
{
|
|
"author": {"role": "assistant"},
|
|
"channel": "analysis",
|
|
"content": [{"type": "text", "text": first_turn_reasoning}],
|
|
},
|
|
{
|
|
"author": {"role": "assistant"},
|
|
"channel": "final",
|
|
"content": [{"type": "text", "text": "The answer is 4."}],
|
|
},
|
|
{
|
|
"author": {"role": "user"},
|
|
"content": [{"type": "text", "text": "Now what is 3+3?"}],
|
|
},
|
|
{
|
|
"author": {"role": "assistant"},
|
|
"channel": "analysis",
|
|
"content": [{"type": "text", "text": second_turn_reasoning}],
|
|
},
|
|
{
|
|
"author": {"role": "assistant"},
|
|
"channel": "commentary",
|
|
"recipient": "functions.calc",
|
|
"content_type": "json",
|
|
"content": [{"type": "text", "text": '{"a":3,"b":3}'}],
|
|
},
|
|
]:
|
|
responses_prev_input_msgs.extend(
|
|
response_previous_input_to_harmony(responses_message)
|
|
)
|
|
|
|
chat_completion_tokens = render_for_completion(
|
|
[_system()] + chat_completion_msgs
|
|
)
|
|
responses_tokens = render_for_completion(
|
|
[_system()] + responses_prev_input_msgs
|
|
)
|
|
|
|
assert chat_completion_tokens == responses_tokens
|
|
|
|
rendered_prompt = get_encoding().decode(responses_tokens)
|
|
assert first_turn_reasoning not in rendered_prompt
|
|
assert second_turn_reasoning in rendered_prompt
|