lightseekorg--tokenspeed/test/runtime/test_inline_detokenizer_e2e.py

"""End-to-end correctness: AsyncLLM vs HuggingFace reference.

Independent ground-truth parity test:

* :class:`HFRunner` loads ``Qwen/Qwen3-0.6B-Base`` via
  ``transformers.AutoModelForCausalLM`` in a dedicated subprocess
  and runs HuggingFace's own ``model.generate`` — the ground truth.
* A local ``_run_rt_generate`` helper instantiates the tokenspeed
  ``Engine`` (which constructs ``AsyncLLM`` wired to the scheduler
  subprocess and the inline ``IncrementalDetokenizer``), runs greedy
  generation with ``return_logprob=False``, and collects the output
  strings.
* :func:`check_close_model_outputs` with ``check_logprobs=False``
  asserts ROUGE-L on the output strings.

Why the helper instead of ``RTRunner``: ``RTRunner.forward`` hardcodes
``engine.generate(return_logprob=True)``, which reaches an empty-list
logprob path in the scheduler output processor
(``generation_output_processor.stream_output`` hardcodes every
logprob field to ``[]``). That trips an ``IndexError`` inside
``convert_logprob_style`` when the engine tries to index a per-rid
logprob slot — a pre-existing latent bug masked in other CI tests
only because those models set ``speculative_algorithm`` and
``Engine.generate`` force-overrides ``return_logprob=False`` when
speculation is on. Our greedy HF-vs-RT comparison is the first to
actually drive the non-speculative ``return_logprob=True`` path, so
we bypass ``RTRunner`` and drop the logprob comparison; the
ground-truth ROUGE-L check still catches the real correctness
regressions this test is for.

Two runners share nothing beyond the checkpoint on disk. Any
AsyncLLM correctness drift (wrong token ids, broken detokenization,
missed finish_reason handling) surfaces as a ROUGE-L failure.

Registered on ``runtime-1gpu``. ``est_time=600`` covers two
cold-start model loads (HF subprocess + tokenspeed scheduler) plus
the two-prompt generation sweep.
"""

import multiprocessing as mp
import os
import sys
import unittest
from typing import List

import torch

# Repository root goes on sys.path so ``test.runners`` resolves.
sys.path.insert(
    0,
    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
)
from test.runners import (  # noqa: E402
    HFRunner,
    ModelOutput,
    check_close_model_outputs,
    get_dtype_str,
)

# CI registration (AST-parsed, runtime no-op).
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from ci_system.ci_register import register_cuda_ci  # noqa: E402

register_cuda_ci(
    est_time=600,
    suite="runtime-1gpu",
    # TODO(amd_ci): re-enable on AMD/ROCm runners. Hits a GPU memory access
    # fault inside reset_valid_cache_length on linux-mi35x runners after
    # cuda-graph capture; root cause still under investigation. NVIDIA
    # runners are unaffected and continue to run this test.
    disabled_on_runners=["linux-mi35*"],
    disabled_on_runners_reason=(
        "GPU memory access fault inside reset_valid_cache_length on " "AMD MI355X"
    ),
)

from tokenspeed.runtime.entrypoints.engine import Engine  # noqa: E402

_MODEL = "Qwen/Qwen3-0.6B-Base"

# Two short ASCII prompts keep the generate budget small and keep
# per-CI-run wall time dominated by model load, not token generation.
# ROUGE-L is asserted per prompt, so two prompts already guard
# against a degenerate "tokenspeed always returns empty string" pass.
_PROMPTS: List[str] = [
    "The capital of Switzerland is",
    "Photosynthesis is the process by which plants",
]

# ``max_new_tokens`` is tuned to the "deterministic window" for
# non-speculative greedy decoding of Qwen/Qwen3-0.6B-Base under
# bfloat16.
_MAX_NEW_TOKENS = 16
_TORCH_DTYPE = torch.bfloat16
# ROUGE-L ≥ 0.9 enforces near-identical output strings; with
# max_new_tokens=16 we measured ROUGE-L = 1.0 on both sample
# prompts in the first CI run, so 0.9 is a generous-but-real bar.
_ROUGE_L_TOLERANCE = 0.9

# Some hardware (e.g. H100) produces a different but equally valid
# list ordering for the first prompt ("Zurich" and "Geneva" swapped),
# scoring ROUGE-L ≈ 0.73 against the primary HF reference.  Both
# orderings are correct completions, so we register the alternative
# here rather than lowering the global tolerance.
_EXTRA_REFERENCES: List[List[str]] = [
    [
        " ____.\nA. Bern\nB. Zurich\nC. Geneva\nD.",
        ", algae, and some bacteria convert light energy into chemical energy. It is a",
    ],
]


def _run_rt_generate(
    prompts: List[str],
    max_new_tokens: int,
    torch_dtype: torch.dtype,
) -> ModelOutput:
    """Drive tokenspeed ``Engine`` end-to-end and collect the per-prompt
    output strings. Bypasses ``RTRunner`` because ``RTRunner.forward``
    hardcodes ``return_logprob=True`` and hits the pre-existing
    empty-logprob-list bug in the scheduler's output processor (see
    the module docstring for details).
    """
    engine = Engine(
        model=_MODEL,
        dtype=get_dtype_str(torch_dtype),
        seed=42,
    )
    try:
        output_strs: List[str] = []
        output_ids: List[List[int]] = []
        for prompt in prompts:
            response = engine.generate(
                prompt=prompt,
                sampling_params={
                    "max_new_tokens": max_new_tokens,
                    "temperature": 0,
                },
                stream=False,
            )
            text = response["text"]
            if not text.strip():
                raise ValueError(
                    f"tokenspeed Engine returned empty text for "
                    f"prompt {prompt!r}; cannot validate AsyncLLM correctness."
                )
            output_strs.append(text)
            output_ids.append(response["output_ids"])
        return ModelOutput(output_strs=output_strs, output_ids=output_ids)
    finally:
        engine.shutdown()


class TestAsyncLLMMatchesHuggingFaceReference(unittest.TestCase):
    """tokenspeed AsyncLLM output must match HuggingFace's reference
    generation on the same checkpoint. Fails loudly if the
    scheduler → tokenizer-manager → inline-detokenizer → collector
    pipeline drifts from what plain ``AutoModelForCausalLM.generate``
    produces.

    Runs HFRunner and the tokenspeed Engine sequentially on the
    shared GPU: HFRunner's context manager spawns and tears down
    its subprocess before the Engine starts, so only one model is
    resident at any time.
    """

    @classmethod
    def setUpClass(cls) -> None:
        # HFRunner spawns its model in a child process; force
        # ``spawn`` so CUDA state does not leak from the test runner.
        mp.set_start_method("spawn", force=True)

    def test_generation_matches_hf_reference(self) -> None:
        with HFRunner(
            _MODEL,
            torch_dtype=_TORCH_DTYPE,
            model_type="generation",
        ) as hf_runner:
            hf_outputs = hf_runner.forward(_PROMPTS, max_new_tokens=_MAX_NEW_TOKENS)

        rt_outputs = _run_rt_generate(
            _PROMPTS,
            max_new_tokens=_MAX_NEW_TOKENS,
            torch_dtype=_TORCH_DTYPE,
        )

        # ``check_logprobs=False`` skips the top-logprob diff — the
        # tokenspeed scheduler's output processor does not currently
        # populate per-rid logprob slots in ``BatchTokenIDOut``
        # (pre-existing bug, see module docstring). The ROUGE-L
        # assertion on output strings is what validates AsyncLLM's
        # end-to-end correctness here.
        check_close_model_outputs(
            hf_outputs=hf_outputs,
            rt_outputs=rt_outputs,
            prefill_tolerance=0.0,
            decode_tolerance=0.0,
            rouge_l_tolerance=_ROUGE_L_TOLERANCE,
            debug_text=f"model={_MODEL}",
            check_logprobs=False,
            extra_references=_EXTRA_REFERENCES,
        )


if __name__ == "__main__":
    unittest.main(verbosity=2)