lightseekorg--tokenspeed/test/runtime/distributed/test_dp_sampling_swap.py

"""Tests for ``swap_batch_vocab``."""

import socket
import traceback

import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp

from tokenspeed.runtime.distributed.dp_sampling_swap import swap_batch_vocab
from tokenspeed.runtime.distributed.process_group_manager import (
    process_group_manager as pg_manager,
)


def _get_open_port() -> int:
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
        s.bind(("", 0))
        return s.getsockname()[1]


def _worker_main(rank, world_size, port, test_fn, error_dict, args):
    try:
        device = torch.device(f"cuda:{rank}")
        torch.cuda.set_device(device)

        dist.init_process_group(
            backend="nccl",
            init_method=f"tcp://localhost:{port}",
            rank=rank,
            world_size=world_size,
        )

        group = tuple(range(world_size))
        pg_manager.init_process_group(group)

        test_fn(rank=rank, world_size=world_size, device=device, group=group, **args)

        dist.destroy_process_group()
    except Exception:
        error_dict[rank] = traceback.format_exc()


def _run(world_size, test_fn, **args):
    if world_size > torch.cuda.device_count():
        pytest.skip(f"Need {world_size} GPUs, have {torch.cuda.device_count()}")
    port = _get_open_port()
    error_dict = mp.Manager().dict()
    mp.spawn(
        _worker_main,
        args=(world_size, port, test_fn, error_dict, args),
        nprocs=world_size,
        join=True,
    )
    if error_dict:
        raise RuntimeError("\n".join(f"Rank {r}: {e}" for r, e in error_dict.items()))


def _ground_truth_full(pad_bs: int, n: int, vocab: int, *, dtype, device):
    return torch.arange(pad_bs * n * vocab, dtype=dtype, device=device).view(
        pad_bs * n, vocab
    )


def _test_swap_matches_reference(
    rank, world_size, device, group, *, pad_bs, n, vocab, dtype
):
    tp = world_size
    v_local = vocab // tp
    reqs_per_rank = pad_bs // tp

    full = _ground_truth_full(pad_bs, n, vocab, dtype=dtype, device=device)

    local_logits = full[:, rank * v_local : (rank + 1) * v_local].contiguous()

    out = swap_batch_vocab(
        local_logits,
        tp_size=tp,
        pad_bs=pad_bs,
        num_tokens_per_req=n,
        vocab_size=vocab,
        group=group,
    )

    expected = full[
        rank * reqs_per_rank * n : (rank + 1) * reqs_per_rank * n
    ].contiguous()
    assert tuple(out.shape) == tuple(
        expected.shape
    ), f"shape mismatch: got {tuple(out.shape)} expected {tuple(expected.shape)}"
    torch.testing.assert_close(out, expected)


def _test_swap_chain_safety(
    rank, world_size, device, group, *, pad_bs, n, vocab, dtype
):
    tp = world_size
    v_local = vocab // tp
    reqs_per_rank = pad_bs // tp

    full = torch.empty(pad_bs * n, vocab, dtype=dtype, device=device)
    for req in range(pad_bs):
        for d in range(n):
            base = req * 10_000 + d * 100
            full[req * n + d] = torch.arange(vocab, dtype=dtype, device=device) + base
    local_logits = full[:, rank * v_local : (rank + 1) * v_local].contiguous()

    out = swap_batch_vocab(
        local_logits,
        tp_size=tp,
        pad_bs=pad_bs,
        num_tokens_per_req=n,
        vocab_size=vocab,
        group=group,
    )

    for local_req in range(reqs_per_rank):
        global_req = rank * reqs_per_rank + local_req
        for d in range(n):
            row = out[local_req * n + d]
            expected_first = global_req * 10_000 + d * 100
            assert int(row[0].item()) == expected_first, (
                f"rank={rank} local_req={local_req} d={d} got row[0]={int(row[0].item())}"
                f" expected {expected_first}"
            )
            assert int(row[-1].item()) == expected_first + (vocab - 1)


WORLD_SIZES = [
    pytest.param(2, id="tp2"),
]

SHAPES = [
    pytest.param(8, 1, 64, id="sample_pad_bs8"),
    pytest.param(8, 4, 64, id="spec_pad_bs8_n4"),
]

DTYPES = [
    pytest.param(torch.float32, id="fp32"),
    pytest.param(torch.bfloat16, id="bf16"),
]


class TestDPSamplingSwap:

    @pytest.mark.parametrize("world_size", WORLD_SIZES)
    @pytest.mark.parametrize("pad_bs,n,vocab", SHAPES)
    @pytest.mark.parametrize("dtype", DTYPES)
    def test_swap_matches_reference(self, world_size, pad_bs, n, vocab, dtype):
        if pad_bs % world_size != 0:
            pytest.skip(f"pad_bs={pad_bs} not divisible by tp={world_size}")
        if vocab % world_size != 0:
            pytest.skip(f"vocab={vocab} not divisible by tp={world_size}")
        _run(
            world_size,
            _test_swap_matches_reference,
            pad_bs=pad_bs,
            n=n,
            vocab=vocab,
            dtype=dtype,
        )

    @pytest.mark.parametrize("world_size", WORLD_SIZES)
    @pytest.mark.parametrize("pad_bs,n,vocab", SHAPES)
    def test_swap_chain_safety(self, world_size, pad_bs, n, vocab):
        if pad_bs % world_size != 0:
            pytest.skip(f"pad_bs={pad_bs} not divisible by tp={world_size}")
        if vocab % world_size != 0:
            pytest.skip(f"vocab={vocab} not divisible by tp={world_size}")
        _run(
            world_size,
            _test_swap_chain_safety,
            pad_bs=pad_bs,
            n=n,
            vocab=vocab,
            dtype=torch.float32,
        )