apache--tvm/python/tvm/tirx/transform/common.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


from tvm.ir import Call, Op, is_prim_expr
from tvm.tirx import (
    AllocBuffer,
    BufferLoad,
    BufferRegion,
    BufferStore,
    DeclBuffer,
    Evaluate,
    Expr,
    Stmt,
    TilePrimitiveCall,
    Var,
    decl_buffer,
)
from tvm.tirx.buffer import Buffer
from tvm.tirx.layout import Iter, TileLayout
from tvm.tirx.stmt_functor import StmtExprMutator, StmtMutator


class BufferReplacer(StmtExprMutator):
    """
    Replace buffer with another buffer.
    Also replace the data of the buffer with another var.
    """

    def __init__(
        self, buffer_map: dict[Buffer, Buffer] | None = None, var_map: dict[Var, Var] | None = None
    ):
        super().__init__()
        self.buffer_map = buffer_map if buffer_map is not None else {}
        self.var_map = var_map if var_map is not None else {}
        self.buffer_attr_var_mutated = False
        for old_buffer, new_buffer in self.buffer_map.items():
            self.var_map[old_buffer.data] = new_buffer.data

    def mutate_buffer(self, buffer: Buffer):
        if buffer in self.buffer_map:
            return self.buffer_map[buffer]

        # Track mutations for this specific buffer only.  Without this reset,
        # unrelated buffers can be spuriously cloned and introduce alias buffers.
        prev_mutated = self.buffer_attr_var_mutated
        self.buffer_attr_var_mutated = False
        new_data = self.visit_expr(buffer.data)
        new_shape = [self.visit_expr(expr) for expr in buffer.shape]
        new_strides = [self.visit_expr(expr) for expr in buffer.strides]
        new_elem_offset = (
            self.visit_expr(buffer.elem_offset) if buffer.elem_offset is not None else None
        )
        if isinstance(buffer.layout, TileLayout):
            new_shard = []
            new_replicate = []
            for iter in buffer.layout.shard:
                new_iter = Iter(
                    self.visit_expr(iter.extent), self.visit_expr(iter.stride), iter.axis
                )
                new_shard.append(new_iter)
            for iter in buffer.layout.replica:
                new_iter = Iter(
                    self.visit_expr(iter.extent), self.visit_expr(iter.stride), iter.axis
                )
                new_replicate.append(new_iter)
            new_layout = TileLayout.from_iters(
                new_shard, new_replicate, offset=buffer.layout.offset
            )
        else:
            new_layout = buffer.layout
        buffer_attr_mutated = self.buffer_attr_var_mutated
        self.buffer_attr_var_mutated = prev_mutated or buffer_attr_mutated
        if not buffer_attr_mutated:
            return None
        new_buffer = decl_buffer(
            new_shape,
            buffer.dtype,
            buffer.name,
            new_data,
            new_strides,
            new_elem_offset,
            buffer.scope(),
            buffer.data_alignment,
            buffer.offset_factor,
            layout=new_layout,
        )
        self.buffer_map[buffer] = new_buffer
        return new_buffer

    def visit_var_(self, op: Var):
        op = super().visit_var_(op)
        if op in self.var_map:
            self.buffer_attr_var_mutated = True
            return self.var_map[op]
        return op

    def visit_buffer_load_(self, op: BufferLoad):
        new_buffer = self.mutate_buffer(op.buffer)
        op = super().visit_buffer_load_(op)
        if new_buffer is not None:
            return BufferLoad(new_buffer, op.indices)
        return op

    def visit_buffer_store_(self, op: BufferStore):
        new_buffer = self.mutate_buffer(op.buffer)
        op = super().visit_buffer_store_(op)
        if new_buffer is not None:
            return BufferStore(new_buffer, op.value, op.indices)
        return op

    def visit_buffer_region_(self, op: BufferRegion):
        new_buffer = self.mutate_buffer(op.buffer)
        op = super().visit_buffer_region_(op)
        if new_buffer is not None:
            return BufferRegion(new_buffer, op.region)
        return op

    def visit_decl_buffer_(self, op: DeclBuffer):
        new_buffer = self.mutate_buffer(op.buffer)
        op = super().visit_decl_buffer_(op)
        if new_buffer is not None:
            return DeclBuffer(new_buffer, op.span)
        return op

    def visit_array_prim_expr_(self, op: list[Expr]):
        return [self.visit_expr(expr) for expr in op]

    def visit_alloc_buffer_(self, op: AllocBuffer):
        op = super().visit_alloc_buffer_(op)
        if op.buffer in self.buffer_map:
            return AllocBuffer(self.buffer_map[op.buffer], op.annotations, op.span)
        return op

    def visit_op_call_(self, op):
        op = super().visit_op_call_(op)
        new_workspace = {}
        for key, value in op.workspace.items():
            new_buffer = self.mutate_buffer(value)
            if new_buffer is not None:
                new_workspace[key] = new_buffer
            else:
                new_workspace[key] = value
        new_config = {}
        for key, value in op.config.items():
            if is_prim_expr(value):
                new_config[key] = self.visit_expr(value)
            else:
                new_config[key] = value
        args = list()
        for arg in op.args:
            args.append(arg)
        return TilePrimitiveCall(
            *args,
            op=op.op,
            workspace=new_workspace,
            config=new_config,
            dispatch=op.dispatch,
            scope=op.scope,
        )


class KernelReplacePointSearcher(StmtMutator):
    def __init__(self, body: Stmt):
        super().__init__()
        self.body = body

    def visit_evaluate_(self, op: Evaluate):
        value = op.value
        if isinstance(value, Call) and value.op.same_as(Op.get("tirx.tvm_kernel_replace_point")):
            return self.body
        return super().visit_evaluate_(op)


def seek_kernel_replace_point(stmt: Stmt, body: Stmt) -> Stmt:
    """replace kernel replace point in stmt with body"""
    return KernelReplacePointSearcher(body)(stmt)