apache--tvm/python/tvm/tirx/exec_context.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""ExecContext: per-program-point active-thread state.

The active thread set is represented as a ``TileLayout``: active axes live in
``layout.shard`` and per-axis lower bounds live in ``layout.offset``. Filters
narrow that layout; scope switches derive the current ``inter``/``intra`` view.
"""

from __future__ import annotations

from dataclasses import dataclass

from tvm.tirx.layout import Axis, Iter, TileLayout

WG_SIZE = 4

KERNEL = "kernel"
CLUSTER = "cluster"
CTA = "cta"
WARPGROUP = "warpgroup"
WARP = "warp"
THREAD = "thread"

SCOPE_KINDS = (KERNEL, CLUSTER, CTA, WARPGROUP, WARP, THREAD)

LANE_FLAT = "flat"
LANE_WG_OUTER = "wg_outer"
LANE_W_INNER = "w_inner"
LANE_CTA_THREAD = "cta_thread"
LANE_WG_THREAD = "wg_thread"


class ExecContextError(Exception):
    """Raised on structural violations of the ExecContext model."""


def _ceildiv(lhs: int, rhs: int) -> int:
    return -((-lhs) // rhs)


def _gcd(lhs: int, rhs: int) -> int:
    while rhs:
        lhs, rhs = rhs, lhs % rhs
    return abs(lhs)


def _extended_gcd(lhs: int, rhs: int) -> tuple[int, int, int]:
    if rhs == 0:
        return lhs, 1, 0
    gcd, x1, y1 = _extended_gcd(rhs, lhs % rhs)
    return gcd, y1, x1 - (lhs // rhs) * y1


def _mod_inverse(value: int, modulus: int) -> int:
    if modulus == 1:
        return 0
    gcd, inv, _ = _extended_gcd(value % modulus, modulus)
    if gcd != 1:
        raise ExecContextError(f"{value} has no inverse modulo {modulus}")
    return inv % modulus


@dataclass(frozen=True)
class AxisRange:
    """An active slice offset + stride * [0, extent) on one TileLayout axis."""

    extent: int
    offset: int = 0
    stride: int = 1

    def intersect(self, lo: int, hi: int) -> AxisRange:
        i_lo = max(0, _ceildiv(lo - self.offset, self.stride))
        i_hi = min(self.extent, (hi - 1 - self.offset) // self.stride + 1)
        if i_hi <= i_lo:
            raise ExecContextError(
                f"filter produces empty range: current=[{self.offset},"
                f" {self.offset + self.extent}) ∩ [{lo}, {hi})"
            )
        return AxisRange(
            extent=i_hi - i_lo, offset=self.offset + self.stride * i_lo, stride=self.stride
        )

    def modulo(self, modulus: int, residue: int) -> AxisRange:
        residue %= modulus
        rhs = (residue - self.offset) % modulus
        g = _gcd(self.stride, modulus)
        if rhs % g != 0:
            raise ExecContextError(
                f"modulo filter produces empty range: {self.offset} + {self.stride} * i"
                f" == {residue} mod {modulus}"
            )
        reduced_stride = self.stride // g
        reduced_rhs = rhs // g
        reduced_modulus = modulus // g
        period = reduced_modulus
        i0 = (reduced_rhs * _mod_inverse(reduced_stride, reduced_modulus)) % reduced_modulus
        if i0 >= self.extent:
            raise ExecContextError(
                f"modulo filter produces empty range: {self.offset} + {self.stride} * i"
                f" == {residue} mod {modulus}"
            )
        return AxisRange(
            extent=(self.extent - 1 - i0) // period + 1,
            offset=self.offset + self.stride * i0,
            stride=self.stride * period,
        )


@dataclass(frozen=True)
class ActiveSet:
    """Active thread set represented by a TileLayout."""

    layout: TileLayout

    @staticmethod
    def from_axes(axes: list[tuple[str, AxisRange]]) -> ActiveSet:
        shard = [Iter(axis_range.extent, axis_range.stride, name) for name, axis_range in axes]
        offset = {
            Axis.get(name): axis_range.offset for name, axis_range in axes if axis_range.offset != 0
        }
        return ActiveSet(TileLayout.from_iters(shard, [], offset))

    @property
    def size(self) -> int:
        result = 1
        for it in self.layout.shard:
            result *= int(it.extent)
        return result

    @property
    def axis_names(self) -> list[str]:
        return [str(it.axis.name) for it in self.layout.shard]

    def axis(self, name: str) -> AxisRange:
        for it in self.layout.shard:
            if str(it.axis.name) != name:
                continue
            offset = 0
            for axis, value in self.layout.offset.items():
                if str(axis.name) == name:
                    offset = int(value)
                    break
            return AxisRange(int(it.extent), offset, int(it.stride))
        raise ValueError(f"unknown active-set axis: {name!r}")

    def replace_axis(self, axis: str, axis_range: AxisRange) -> ActiveSet:
        axes: list[tuple[str, AxisRange]] = []
        found = False
        for name in self.axis_names:
            if name == axis:
                axes.append((name, axis_range))
                found = True
            else:
                axes.append((name, self.axis(name)))
        if not found:
            raise ValueError(f"unknown active-set axis: {axis!r}")
        return ActiveSet.from_axes(axes)

    @property
    def laneid(self) -> AxisRange:
        return self.axis("laneid")

    @property
    def warpid(self) -> AxisRange:
        return self.axis("warpid")

    @property
    def cta_id(self) -> AxisRange:
        return self.axis("cta_id")


@dataclass(frozen=True)
class LaneBinding:
    """Resolution of a user-declared ScopeIdDef Var to one active-set axis."""

    axis: str
    kind: str
    declared_extent: int


def initial_A(*, lane_ext: int = 32, warp_ext: int, cta_ext: int = 1) -> ActiveSet:
    """Build A at PrimFunc device entry: all threads active, offsets all zero."""
    return ActiveSet.from_axes(
        [
            ("laneid", AxisRange(lane_ext, 0)),
            ("warpid", AxisRange(warp_ext, 0)),
            ("cta_id", AxisRange(cta_ext, 0)),
        ]
    )


def filter_narrow(A: ActiveSet, binding: LaneBinding, lo: int, hi: int) -> ActiveSet:
    """Intersect A's binding axis with [lo, hi)."""
    if lo >= hi:
        raise ExecContextError(f"filter range [{lo}, {hi}) is empty or inverted")

    if binding.kind == LANE_CTA_THREAD:
        new_warpid, new_laneid = _flat_product_range(A.warpid, A.laneid, lo, hi)
        return A.replace_axis("laneid", new_laneid).replace_axis("warpid", new_warpid)

    if binding.kind == LANE_WG_THREAD:
        factored = _factor_warpid(A.warpid)
        if factored is None:
            raise ExecContextError(
                "filter on flat warpgroup-thread range requires factorable warpid axis"
            )
        wid_in_wg, wgid = factored
        new_wid_in_wg, new_laneid = _flat_product_range(wid_in_wg, A.laneid, lo, hi)
        if wgid.extent != 1:
            if new_wid_in_wg == wid_in_wg and new_laneid == A.laneid:
                return A
            raise ExecContextError(
                "flat warpgroup-thread range across multiple warpgroups is not representable"
            )
        new_warpid = AxisRange(
            extent=new_wid_in_wg.extent, offset=wgid.offset * WG_SIZE + new_wid_in_wg.offset
        )
        return A.replace_axis("laneid", new_laneid).replace_axis("warpid", new_warpid)

    if binding.kind == LANE_FLAT:
        new_axis = A.axis(binding.axis).intersect(lo, hi)
        return A.replace_axis(binding.axis, new_axis)

    if binding.axis != "warpid":
        raise ExecContextError(
            f"kind={binding.kind!r} only valid for axis='warpid'; got {binding.axis!r}"
        )

    wp = A.warpid
    if wp.stride != 1:
        raise ExecContextError(
            f"kind={binding.kind!r} requires unit-stride warpid axis; got stride={wp.stride}"
        )
    if binding.kind == LANE_WG_OUTER:
        if wp.offset % WG_SIZE != 0 or wp.extent % WG_SIZE != 0:
            raise ExecContextError(
                f"filter on wg_outer requires warpid axis aligned to WG_SIZE={WG_SIZE};"
                f" got extent={wp.extent}, offset={wp.offset}"
            )
        cur_outer = AxisRange(extent=wp.extent // WG_SIZE, offset=wp.offset // WG_SIZE)
        new_outer = cur_outer.intersect(lo, hi)
        return A.replace_axis(
            "warpid",
            AxisRange(extent=new_outer.extent * WG_SIZE, offset=new_outer.offset * WG_SIZE),
        )

    if binding.kind == LANE_W_INNER:
        cur_inner_off = wp.offset % WG_SIZE
        if wp.extent > WG_SIZE - cur_inner_off:
            raise ExecContextError(
                "filter on w_inner would break A's TileLayout box: warpid spans multiple"
                f" warpgroups (extent={wp.extent}, offset={wp.offset})"
            )
        cur_inner = AxisRange(extent=wp.extent, offset=cur_inner_off)
        new_inner = cur_inner.intersect(lo, hi)
        outer_base = (wp.offset // WG_SIZE) * WG_SIZE
        return A.replace_axis(
            "warpid", AxisRange(extent=new_inner.extent, offset=outer_base + new_inner.offset)
        )

    raise ValueError(f"unknown axis kind: {binding.kind!r}")


def filter_modulo(A: ActiveSet, axis: str, modulus: int, residue: int) -> ActiveSet:
    """Intersect an active-set axis with ``axis % modulus == residue``."""
    if modulus <= 0:
        raise ExecContextError(f"modulus must be positive, got {modulus}")
    new_axis = A.axis(axis).modulo(modulus, residue)
    return A.replace_axis(axis, new_axis)


@dataclass(frozen=True)
class Split:
    """A scope_switch split of A."""

    inter: dict[str, AxisRange]
    intra: dict[str, AxisRange]


def _factor_warpid(warp: AxisRange) -> tuple[AxisRange, AxisRange] | None:
    if warp.stride != 1:
        return None
    off = warp.offset
    ext = warp.extent
    wid_off = off % WG_SIZE
    wgid_off = off // WG_SIZE

    if wid_off == 0 and ext % WG_SIZE == 0:
        return (
            AxisRange(extent=WG_SIZE, offset=0),
            AxisRange(extent=ext // WG_SIZE, offset=wgid_off),
        )
    if ext <= WG_SIZE - wid_off:
        return (AxisRange(extent=ext, offset=wid_off), AxisRange(extent=1, offset=wgid_off))
    return None


def _flat_product_range(
    major: AxisRange, lane: AxisRange, lo: int, hi: int
) -> tuple[AxisRange, AxisRange]:
    active_min = major.offset * 32 + lane.offset
    active_max = (
        (major.offset + major.stride * (major.extent - 1)) * 32
        + lane.offset
        + lane.stride * (lane.extent - 1)
        + 1
    )
    if lo <= active_min and active_max <= hi:
        return major, lane

    if major.stride != 1 or lane.stride != 1:
        raise ExecContextError("flat thread range narrowing requires unit-stride axes")

    lane_hi = lane.offset + lane.extent
    major_hi = major.offset + major.extent
    hit_lo = max(major.offset, (lo - lane_hi) // 32 + 1)
    hit_hi = min(major_hi, _ceildiv(hi - lane.offset, 32))
    if hit_hi <= hit_lo:
        raise ExecContextError("flat thread range produces empty active set")

    if hit_hi == hit_lo + 1:
        new_lane_lo = max(lane.offset, lo - hit_lo * 32)
        new_lane_hi = min(lane_hi, hi - hit_lo * 32)
        if new_lane_hi <= new_lane_lo:
            raise ExecContextError("flat thread range produces empty lane range")
        return AxisRange(1, hit_lo), AxisRange(new_lane_hi - new_lane_lo, new_lane_lo)

    if lo <= hit_lo * 32 + lane.offset and (hit_hi - 1) * 32 + lane_hi <= hi:
        return AxisRange(hit_hi - hit_lo, hit_lo), lane

    raise ExecContextError("flat thread range would require a non-rectangular lane/warp active set")


def scope_switch(A: ActiveSet, scope_kind: str) -> Split:
    """Split A into (inter, intra) for the target scope kind."""
    if scope_kind == THREAD:
        return Split(inter={"laneid": A.laneid, "warpid": A.warpid, "cta_id": A.cta_id}, intra={})
    if scope_kind == WARP:
        return Split(inter={"warpid": A.warpid, "cta_id": A.cta_id}, intra={"laneid": A.laneid})
    if scope_kind == CTA:
        return Split(inter={"cta_id": A.cta_id}, intra={"laneid": A.laneid, "warpid": A.warpid})
    if scope_kind == CLUSTER:
        return Split(inter={}, intra={"laneid": A.laneid, "warpid": A.warpid, "cta_id": A.cta_id})
    if scope_kind == WARPGROUP:
        factored = _factor_warpid(A.warpid)
        if factored is None:
            raise ExecContextError(
                "scope_switch(warpgroup) failed: warpid axis"
                f" (extent={A.warpid.extent}, offset={A.warpid.offset})"
                " crosses warpgroup boundary and is not aligned"
            )
        wid_in_wg, wgid = factored
        return Split(
            inter={"wgid": wgid, "cta_id": A.cta_id},
            intra={"laneid": A.laneid, "wid_in_wg": wid_in_wg},
        )
    if scope_kind == KERNEL:
        return Split(inter={"laneid": A.laneid, "warpid": A.warpid, "cta_id": A.cta_id}, intra={})
    raise ValueError(f"unknown scope kind: {scope_kind!r}")


@dataclass(frozen=True)
class ExecContext:
    """Per-program-point compiler state: active set + scope kind + split."""

    A: ActiveSet
    scope_kind: str
    inter: dict[str, AxisRange]
    intra: dict[str, AxisRange]

    @staticmethod
    def at_kernel_entry(*, lane_ext: int = 32, warp_ext: int, cta_ext: int = 1) -> ExecContext:
        A = initial_A(lane_ext=lane_ext, warp_ext=warp_ext, cta_ext=cta_ext)
        split = scope_switch(A, KERNEL)
        return ExecContext(A=A, scope_kind=KERNEL, inter=split.inter, intra=split.intra)

    def with_filter(self, binding: LaneBinding, lo: int, hi: int) -> ExecContext:
        new_A = filter_narrow(self.A, binding, lo, hi)
        split = scope_switch(new_A, self.scope_kind)
        return ExecContext(
            A=new_A, scope_kind=self.scope_kind, inter=split.inter, intra=split.intra
        )

    def with_cta_axis_modulo(self, axis: str, modulus: int, residue: int) -> ExecContext:
        new_A = filter_modulo(self.A, axis, modulus, residue)
        split = scope_switch(new_A, self.scope_kind)
        return ExecContext(
            A=new_A, scope_kind=self.scope_kind, inter=split.inter, intra=split.intra
        )

    def with_scope_switch(self, scope_kind: str) -> ExecContext:
        split = scope_switch(self.A, scope_kind)
        return ExecContext(A=self.A, scope_kind=scope_kind, inter=split.inter, intra=split.intra)