chore: import upstream snapshot with attribution

2026-07-13 13:36:25 +08:00
commit 26446540fa
3151 changed files with 974126 additions and 0 deletions
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+import tvm_ffi
+
+import tvm
+import tvm.testing
+from tvm import relax as rx
+from tvm import tirx
+from tvm.ir import Range
+
+
+def _check_equal(x, y, map_free_vars=False):
+    tvm.ir.assert_structural_equal(x, y, map_free_vars)
+    tvm.ir.assert_structural_equal(y, x, map_free_vars)
+
+    xhash = tvm_ffi.structural_hash(x, map_free_vars)
+    yhash = tvm_ffi.structural_hash(y, map_free_vars)
+
+    assert xhash == yhash
+
+
+def _check_json_roundtrip(x):
+    xret = tvm.ir.load_json(tvm.ir.save_json(x))
+    _check_equal(x, xret, map_free_vars=True)
+    return xret
+
+
+def test_dtensor_type():
+    n, m = tirx.Var("n", "int64"), tirx.Var("m", "int64")
+
+    tensor_ty0 = rx.TensorType([1, n + 1, m], "float32")
+    tensor_ty1 = rx.TensorType([1, n + 1, m], "float32")
+    assert tensor_ty0 == tensor_ty1
+
+    device_mesh0 = rx.distributed.DeviceMesh((2, 2), Range(0, 4))
+    device_mesh1 = rx.distributed.DeviceMesh((2, 2), Range(0, 4))
+    tvm.ir.assert_structural_equal(device_mesh0, device_mesh1)
+
+    shard0 = rx.distributed.PlacementSpec.sharding(0)
+    replica = rx.distributed.PlacementSpec.replica()
+
+    placement0 = rx.distributed.Placement([shard0, replica])
+    placement1 = rx.distributed.Placement([shard0, replica])
+    tvm.ir.assert_structural_equal(placement0, placement1)
+
+    ty0 = rx.distributed.DTensorType(tensor_ty0, device_mesh0, placement0)
+    ty1 = rx.distributed.DTensorType(tensor_ty1, device_mesh1, placement1)
+    _check_equal(ty0, ty1)
+    _check_json_roundtrip(ty0)
+    _check_json_roundtrip(ty1)
+
+    assert ty0 == ty1
+    tvm.ir.assert_structural_equal(ty0.device_mesh, device_mesh0)
+    assert ty0.device_mesh.shape == (2, 2)
+    tvm.ir.assert_structural_equal(ty0.device_mesh.device_range, Range(0, 4))
+    tvm.ir.assert_structural_equal(ty0.placement, placement0)
+    assert len(ty0.placement.dim_specs) == 2
+    assert ty0.placement.dim_specs[0] == shard0
+    assert ty0.placement.dim_specs[1] == replica
+    assert ty0.tensor_ty == tensor_ty0
+
+    # can turn into str
+    # str(ty0)
+
+    # dimension of device mesh and placement should be the same
+    shard1 = rx.distributed.PlacementSpec.sharding(1)
+    placement2 = rx.distributed.Placement([shard0, replica, shard1])
+    with pytest.raises(ValueError):
+        rx.distributed.DTensorType(tensor_ty0, device_mesh0, placement2)
+
+    # Sharding dimension should be smaller than tensor ndim
+    shard3 = rx.distributed.PlacementSpec.sharding(3)
+    placement3 = rx.distributed.Placement([shard3, replica])
+    with pytest.raises(ValueError):
+        rx.distributed.DTensorType(tensor_ty0, device_mesh0, placement3)
+
+
+if __name__ == "__main__":
+    tvm.testing.main()
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#  type: ignore
+import tvm
+import tvm.testing
+from tvm import relax
+from tvm.script.parser import ir as I
+from tvm.script.parser import relax as R
+
+
+def test_simple():
+    @I.ir_module
+    class Before:
+        I.module_attrs({"device_num": 2})
+        I.module_global_infos({"mesh": [R.device_mesh((2,), I.Range(0, 2))]})
+
+        @R.function
+        def foo(
+            x1: R.DTensor((128, 128), "float32", "mesh[0]", "R"),
+            x2: R.DTensor((128, 128), "float32", "mesh[0]", "S[0]"),
+        ):
+            R.func_attr({"num_input": 1})
+            # scatter
+            lv0 = R.dist.redistribute(x1, "mesh[0]", "S[1]")
+            # do nothing
+            lv1 = R.dist.redistribute(x2, "mesh[0]", "S[0]")
+            return (lv0, lv1)
+
+    @I.ir_module
+    class Expected:
+        I.module_attrs({"device_num": 2})
+        I.module_global_infos({"mesh": [R.device_mesh((2,), I.Range(0, 2))]})
+
+        @R.function
+        def foo(
+            x1: R.DTensor((128, 128), "float32", "mesh[0]", "R"),
+            x2: R.DTensor((128, 128), "float32", "mesh[0]", "S[0]"),
+        ) -> R.Tuple(
+            R.DTensor((128, 128), "float32", "mesh[0]", "S[1]"),
+            R.DTensor((128, 128), "float32", "mesh[0]", "S[0]"),
+        ):
+            R.func_attr({"num_input": 1})
+            lv0: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = (
+                R.dist.redistribute_replica_to_shard(x1, num_workers=2, axis=1)
+            )
+            lv1: R.DTensor((128, 128), "float32", "mesh[0]", "S[0]") = x2
+            return (lv0, lv1)
+
+    after = relax.distributed.transform.LegalizeRedistribute()(Before)
+    tvm.ir.assert_structural_equal(after, Expected)
+
+
+if __name__ == "__main__":
+    tvm.testing.main()
@@ -0,0 +1,397 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: F401, F841
+
+#  type: ignore
+import tvm
+import tvm.testing
+from tvm import relax
+from tvm.ir import assert_structural_equal
+from tvm.script.parser import ir as I
+from tvm.script.parser import relax as R
+from tvm.script.parser import tirx as T
+
+
+def test_mlp():
+    @I.ir_module(s_tir=True)
+    class MLP:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {"mesh": [R.device_mesh((2,), I.Range(0, 2)), R.device_mesh((1,), I.Range(4, 5))]}
+        )
+
+        @T.prim_func(private=True, s_tir=True)
+        def gelu1(
+            A: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+            T_multiply: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            T_multiply_1 = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            compute = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            T_multiply_2 = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            T_add = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_multiply"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(A[v_ax0, v_ax1])
+                    T.writes(T_multiply_1[v_ax0, v_ax1])
+                    T_multiply_1[v_ax0, v_ax1] = A[v_ax0, v_ax1] * T.float32(0.70710678118654757)
+            for i0, i1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("compute"):
+                    v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
+                    T.reads(T_multiply_1[v_i0, v_i1])
+                    T.writes(compute[v_i0, v_i1])
+                    compute[v_i0, v_i1] = T.erf(T_multiply_1[v_i0, v_i1])
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_multiply_1"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(compute[v_ax0, v_ax1])
+                    T.writes(T_multiply_2[v_ax0, v_ax1])
+                    T_multiply_2[v_ax0, v_ax1] = compute[v_ax0, v_ax1] * T.float32(0.5)
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_add"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(T_multiply_2[v_ax0, v_ax1])
+                    T.writes(T_add[v_ax0, v_ax1])
+                    T_add[v_ax0, v_ax1] = T.float32(0.5) + T_multiply_2[v_ax0, v_ax1]
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_multiply_2"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(A[v_ax0, v_ax1], T_add[v_ax0, v_ax1])
+                    T.writes(T_multiply[v_ax0, v_ax1])
+                    T_multiply[v_ax0, v_ax1] = A[v_ax0, v_ax1] * T_add[v_ax0, v_ax1]
+
+        @T.prim_func(private=True, s_tir=True)
+        def matmul1(
+            A: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+            B: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+            matmul_1: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            for i0, i1, k in T.grid(T.int64(128), T.int64(64), T.int64(128)):
+                with T.sblock("matmul"):
+                    v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+                    T.reads(A[v_i0, v_k], B[v_k, v_i1])
+                    T.writes(matmul_1[v_i0, v_i1])
+                    with T.init():
+                        matmul_1[v_i0, v_i1] = T.float32(0)
+                    matmul_1[v_i0, v_i1] = matmul_1[v_i0, v_i1] + A[v_i0, v_k] * B[v_k, v_i1]
+
+        @T.prim_func(private=True, s_tir=True)
+        def matmul2(
+            A: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+            B: T.Buffer((T.int64(64), T.int64(128)), "float32"),
+            matmul_1: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            for i0, i1, k in T.grid(T.int64(128), T.int64(128), T.int64(64)):
+                with T.sblock("matmul"):
+                    v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+                    T.reads(A[v_i0, v_k], B[v_k, v_i1])
+                    T.writes(matmul_1[v_i0, v_i1])
+                    with T.init():
+                        matmul_1[v_i0, v_i1] = T.float32(0)
+                    matmul_1[v_i0, v_i1] = matmul_1[v_i0, v_i1] + A[v_i0, v_k] * B[v_k, v_i1]
+
+        @R.function
+        def foo(
+            x: R.DTensor((128, 128), "float32", "mesh[0]", "R"),
+            weight1: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]"),
+            weight2: R.DTensor((128, 128), "float32", "mesh[0]", "S[0]"),
+        ) -> R.DTensor((128, 128), "float32", "mesh[0]", "R"):
+            R.func_attr({"num_input": 1})
+            cls = MLP
+            lv0: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = R.dist.call_tir_local_view(
+                cls.matmul1,
+                (x, weight1),
+                out_ty=R.DTensor((128, 128), "float32", "mesh[0]", "S[1]"),
+            )
+            lv1: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = R.dist.call_tir_local_view(
+                cls.gelu1, (lv0,), out_ty=R.DTensor((128, 128), "float32", "mesh[0]", "S[1]")
+            )
+            lv2: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = lv1
+            gv: R.DTensor((128, 128), "float32", "mesh[0]", "R") = R.dist.call_tir_local_view(
+                cls.matmul2,
+                (lv2, weight2),
+                out_ty=R.DTensor((128, 128), "float32", "mesh[0]", "R"),
+            )
+            lv3: R.DTensor((128, 128), "float32", "mesh[0]", "R") = R.ccl.allreduce(
+                gv, op_type="sum"
+            )
+            return lv3
+
+    @I.ir_module(check_well_formed=False, s_tir=True)
+    class LoweredMLP:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {"mesh": [R.device_mesh((2,), I.Range(0, 2)), R.device_mesh((1,), I.Range(4, 5))]}
+        )
+
+        @R.function
+        def foo(
+            x: R.Tensor((128, 128), dtype="float32"),
+            weight1: R.Tensor((128, 128), dtype="float32"),
+            weight2: R.Tensor((128, 128), dtype="float32"),
+        ) -> R.Tensor((128, 128), dtype="float32"):
+            R.func_attr({"num_input": 1})
+            cls = LoweredMLP
+            gv: R.Tensor((128, 128), dtype="float32") = R.ccl.broadcast_from_worker0(x)
+            gv1: R.Tensor((128, 64), dtype="float32") = R.ccl.scatter_from_worker0(
+                weight1, num_workers=2, axis=1
+            )
+            gv2: R.Tensor((64, 128), dtype="float32") = R.ccl.scatter_from_worker0(
+                weight2, num_workers=2, axis=0
+            )
+            lv0 = R.call_tir(
+                MLP.get_global_var("matmul1"),
+                (gv, gv1),
+                out_ty=R.Tensor((128, 64), dtype="float32"),
+            )
+            lv1 = R.call_tir(
+                MLP.get_global_var("gelu1"), (lv0,), out_ty=R.Tensor((128, 64), dtype="float32")
+            )
+            lv2: R.Tensor((128, 64), dtype="float32") = lv1
+            gv_1 = R.call_tir(
+                MLP.get_global_var("matmul2"),
+                (lv2, gv2),
+                out_ty=R.Tensor((128, 128), dtype="float32"),
+            )
+            lv3: R.Tensor((128, 128), dtype="float32") = R.ccl.allreduce(gv_1, op_type="sum")
+            return lv3
+
+    for gv, func in MLP.functions_items():
+        if gv.name_hint != "foo":
+            LoweredMLP[gv] = func
+
+    mod = MLP
+    mod = relax.distributed.transform.LowerDistIR()(mod)
+    tvm.ir.assert_structural_equal(mod, LoweredMLP)
+
+
+def test_mlp_with_tuple():
+    @I.ir_module(s_tir=True)
+    class MLPWithTuple:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {"mesh": [R.device_mesh((2,), I.Range(0, 2)), R.device_mesh((1,), I.Range(4, 5))]}
+        )
+
+        @T.prim_func(private=True, s_tir=True)
+        def gelu1(
+            A: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+            T_multiply: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            T_multiply_1 = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            compute = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            T_multiply_2 = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            T_add = T.sblock_alloc_buffer((T.int64(128), T.int64(64)))
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_multiply"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(A[v_ax0, v_ax1])
+                    T.writes(T_multiply_1[v_ax0, v_ax1])
+                    T_multiply_1[v_ax0, v_ax1] = A[v_ax0, v_ax1] * T.float32(0.70710678118654757)
+            for i0, i1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("compute"):
+                    v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
+                    T.reads(T_multiply_1[v_i0, v_i1])
+                    T.writes(compute[v_i0, v_i1])
+                    compute[v_i0, v_i1] = T.erf(T_multiply_1[v_i0, v_i1])
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_multiply_1"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(compute[v_ax0, v_ax1])
+                    T.writes(T_multiply_2[v_ax0, v_ax1])
+                    T_multiply_2[v_ax0, v_ax1] = compute[v_ax0, v_ax1] * T.float32(0.5)
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_add"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(T_multiply_2[v_ax0, v_ax1])
+                    T.writes(T_add[v_ax0, v_ax1])
+                    T_add[v_ax0, v_ax1] = T.float32(0.5) + T_multiply_2[v_ax0, v_ax1]
+            for ax0, ax1 in T.grid(T.int64(128), T.int64(64)):
+                with T.sblock("T_multiply_2"):
+                    v_ax0, v_ax1 = T.axis.remap("SS", [ax0, ax1])
+                    T.reads(A[v_ax0, v_ax1], T_add[v_ax0, v_ax1])
+                    T.writes(T_multiply[v_ax0, v_ax1])
+                    T_multiply[v_ax0, v_ax1] = A[v_ax0, v_ax1] * T_add[v_ax0, v_ax1]
+
+        @T.prim_func(private=True, s_tir=True)
+        def matmul11(
+            A: T.Buffer((T.int64(64), T.int64(64)), "float32"),
+            B: T.Buffer((T.int64(64), T.int64(128)), "float32"),
+            matmul: T.Buffer((T.int64(64), T.int64(128)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            for i0, i1, k in T.grid(T.int64(64), T.int64(128), T.int64(64)):
+                with T.sblock("matmul"):
+                    v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+                    T.reads(A[v_i0, v_k], B[v_k, v_i1])
+                    T.writes(matmul[v_i0, v_i1])
+                    with T.init():
+                        matmul[v_i0, v_i1] = T.float32(0)
+                    matmul[v_i0, v_i1] = matmul[v_i0, v_i1] + A[v_i0, v_k] * B[v_k, v_i1]
+
+        @T.prim_func(private=True, s_tir=True)
+        def matmul2(
+            A: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+            B: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+            matmul: T.Buffer((T.int64(128), T.int64(64)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            for i0, i1, k in T.grid(T.int64(128), T.int64(64), T.int64(128)):
+                with T.sblock("matmul"):
+                    v_i0, v_i1, v_k = T.axis.remap("SSR", [i0, i1, k])
+                    T.reads(A[v_i0, v_k], B[v_k, v_i1])
+                    T.writes(matmul[v_i0, v_i1])
+                    with T.init():
+                        matmul[v_i0, v_i1] = T.float32(0)
+                    matmul[v_i0, v_i1] = matmul[v_i0, v_i1] + A[v_i0, v_k] * B[v_k, v_i1]
+
+        @T.prim_func(private=True, s_tir=True)
+        def split11(
+            A: T.Buffer((128, 64), "float32"),
+            T_split: T.Buffer((64, 64), "float32"),
+            T_split_1: T.Buffer((64, 64), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            # with T.sblock("root"):
+            for ax1, ax2 in T.grid(64, 64):
+                with T.sblock("T_split"):
+                    v_ax1, v_ax2 = T.axis.remap("SS", [ax1, ax2])
+                    T.reads(A[v_ax1, v_ax2])
+                    T.writes(T_split[v_ax1, v_ax2])
+                    T_split[v_ax1, v_ax2] = A[v_ax1, v_ax2]
+            for ax1, ax2 in T.grid(64, 64):
+                with T.sblock("T_split_1"):
+                    v_ax1, v_ax2 = T.axis.remap("SS", [ax1, ax2])
+                    T.reads(A[v_ax1 + 64, v_ax2])
+                    T.writes(T_split_1[v_ax1, v_ax2])
+                    T_split_1[v_ax1, v_ax2] = A[v_ax1 + 64, v_ax2]
+
+        @R.function
+        def foo(
+            x: R.DTensor((128, 128), "float32", "mesh[0]", "R"),
+            weight_packed: R.Tuple(
+                R.DTensor((128, 128), "float32", "mesh[0]", "S[1]"),
+                R.DTensor((128, 128), "float32", "mesh[0]", "S[0]"),
+            ),
+        ) -> R.DTensor((64, 128), "float32", "mesh[0]", "R"):
+            cls = MLPWithTuple
+            weight1: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = weight_packed[0]
+            lv0: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = R.dist.call_tir_local_view(
+                cls.matmul2,
+                (x, weight1),
+                out_ty=R.DTensor((128, 128), "float32", "mesh[0]", "S[1]"),
+            )
+            lv1: R.DTensor((128, 128), "float32", "mesh[0]", "S[1]") = R.dist.call_tir_local_view(
+                cls.gelu1, (lv0,), out_ty=R.DTensor((128, 128), "float32", "mesh[0]", "S[1]")
+            )
+            gv: R.Tuple(
+                R.DTensor((64, 128), "float32", "mesh[0]", "S[1]"),
+                R.DTensor((64, 128), "float32", "mesh[0]", "S[1]"),
+            ) = R.dist.call_tir_local_view(
+                cls.split11,
+                (lv1,),
+                out_ty=[
+                    R.DTensor((64, 128), "float32", "mesh[0]", "S[1]"),
+                    R.DTensor((64, 128), "float32", "mesh[0]", "S[1]"),
+                ],
+            )
+            lv2: R.DTensor((64, 128), "float32", "mesh[0]", "S[1]") = gv[0]
+            lv3: R.DTensor((64, 128), "float32", "mesh[0]", "S[1]") = lv2
+            weight2: R.DTensor((128, 128), "float32", "mesh[0]", "S[0]") = weight_packed[1]
+            gv_1: R.DTensor((64, 128), "float32", "mesh[0]", "R") = R.dist.call_tir_local_view(
+                cls.matmul11,
+                (lv3, weight2),
+                out_ty=R.DTensor((64, 128), "float32", "mesh[0]", "R"),
+            )
+            lv4: R.DTensor((64, 128), "float32", "mesh[0]", "R") = R.ccl.allreduce(
+                gv_1, op_type="sum"
+            )
+            return lv4
+
+    @I.ir_module(check_well_formed=False, s_tir=True)
+    class LoweredMLPWithTuple:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {"mesh": [R.device_mesh((2,), I.Range(0, 2)), R.device_mesh((1,), I.Range(4, 5))]}
+        )
+
+        @R.function
+        def foo(
+            x: R.Tensor((128, 128), dtype="float32"),
+            weight_packed: R.Tuple(
+                R.Tensor((128, 128), dtype="float32"), R.Tensor((128, 128), dtype="float32")
+            ),
+        ) -> R.Tensor((64, 128), dtype="float32"):
+            cls = LoweredMLPWithTuple
+            gv: R.Tensor((128, 128), dtype="float32") = R.ccl.broadcast_from_worker0(x)
+            gv1: R.Tensor((128, 128), dtype="float32") = weight_packed[0]
+            gv2: R.Tensor((128, 64), dtype="float32") = R.ccl.scatter_from_worker0(
+                gv1, num_workers=2, axis=1
+            )
+            gv3: R.Tensor((128, 128), dtype="float32") = weight_packed[1]
+            gv4: R.Tensor((64, 128), dtype="float32") = R.ccl.scatter_from_worker0(
+                gv3, num_workers=2, axis=0
+            )
+            lv0 = R.call_tir(
+                MLPWithTuple.get_global_var("matmul2"),
+                (gv, gv2),
+                out_ty=R.Tensor((128, 64), dtype="float32"),
+            )
+            lv1 = R.call_tir(
+                MLPWithTuple.get_global_var("gelu1"),
+                (lv0,),
+                out_ty=R.Tensor((128, 64), dtype="float32"),
+            )
+            gv_1 = R.call_tir(
+                MLPWithTuple.get_global_var("split11"),
+                (lv1,),
+                out_ty=[
+                    R.Tensor((64, 64), dtype="float32"),
+                    R.Tensor((64, 64), dtype="float32"),
+                ],
+            )
+            lv2: R.Tensor((64, 64), dtype="float32") = gv_1[0]
+            lv3: R.Tensor((64, 64), dtype="float32") = lv2
+            gv_1_1 = R.call_tir(
+                MLPWithTuple.get_global_var("matmul11"),
+                (lv3, gv4),
+                out_ty=R.Tensor((64, 128), dtype="float32"),
+            )
+            lv4: R.Tensor((64, 128), dtype="float32") = R.ccl.allreduce(gv_1_1, op_type="sum")
+            return lv4
+
+    for gv, func in MLPWithTuple.functions_items():
+        if gv.name_hint != "foo":
+            LoweredMLPWithTuple[gv] = func
+
+    mod = MLPWithTuple
+    mod = relax.distributed.transform.LowerDistIR()(mod)
+    tvm.ir.assert_structural_equal(mod, LoweredMLPWithTuple)
+
+
+if __name__ == "__main__":
+    tvm.testing.main()
@@ -0,0 +1,191 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: F401
+
+from typing import Optional, Union
+
+import pytest
+
+import tvm
+import tvm.script
+import tvm.testing
+from tvm import IRModule, relax, tirx, topi
+from tvm.ir import Range
+from tvm.relax import Call, SeqExpr, VarBinding
+from tvm.relax.distributed import DeviceMesh
+from tvm.script.parser import ir as I
+from tvm.script.parser import relax as R
+from tvm.script.parser import tirx as T
+
+
+def _check(
+    parsed: relax.Function | IRModule,
+    expect: relax.Function | IRModule | None = None,
+):
+    test = parsed.script(show_meta=True)
+    roundtrip_mod = tvm.script.from_source(test)
+    tvm.ir.assert_structural_equal(parsed, roundtrip_mod)
+    if expect:
+        tvm.ir.assert_structural_equal(parsed, expect)
+
+
+def test_call_tir_dtensor():
+    @I.ir_module(s_tir=True)
+    class TestModule:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {
+                "mesh": [
+                    R.device_mesh((2, 2), I.Range(0, 4)),  # mesh[0]
+                    R.device_mesh((1,), I.Range(4, 5)),  # mesh[1]
+                ]
+            }
+        )
+
+        @T.prim_func(s_tir=True)
+        def tir_func(
+            x: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+            y: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            for i, j in T.grid(T.int64(128), T.int64(128)):
+                with T.sblock():
+                    vi, vj = T.axis.remap("SS", [i, j])
+                    y[vi, vj] = x[vi, vj] + 1.0
+
+        @R.function
+        def foo(
+            x: R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"),
+        ) -> R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"):
+            gv0 = R.dist.call_tir(
+                TestModule.tir_func,
+                x,
+                R.DTensor(
+                    shape=(128, 128), dtype="float32", device_mesh="mesh[0]", placement="S[0], R"
+                ),
+            )
+            return gv0
+
+    device_mesh_list = [DeviceMesh((2, 2), Range(0, 4)), DeviceMesh((1,), Range(4, 5))]
+    foo_func = TestModule["foo"]
+    params = foo_func.params
+    assert len(params) == 1
+    assert params[0].ty == R.DTensor(
+        (128, 128), "float32", device_mesh_list[0], placement="S[0], R"
+    )
+    assert foo_func.ret_ty == R.DTensor(
+        (128, 128), "float32", device_mesh_list[0], placement="S[0], R"
+    )
+    assert isinstance(foo_func.body, SeqExpr)
+    assert len(foo_func.body.blocks[0].bindings) == 1
+    assert isinstance(foo_func.body.blocks[0].bindings[0], VarBinding)
+    value = foo_func.body.blocks[0].bindings[0].value
+    assert isinstance(value, Call)
+    assert value.ty_args[0] == R.DTensor(
+        (128, 128), "float32", device_mesh_list[0], placement="S[0], R"
+    )
+    _check(TestModule)
+
+
+def test_explicit_device_id():
+    @I.ir_module(s_tir=True)
+    class TestModule:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {
+                "mesh": [
+                    R.device_mesh((2, 2), [0, 1, 2, 3]),  # mesh[0]
+                    R.device_mesh(
+                        (1,),
+                        [
+                            4,
+                        ],
+                    ),  # mesh[1]
+                ]
+            }
+        )
+
+        @T.prim_func(s_tir=True)
+        def tir_func(
+            x: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+            y: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            for i, j in T.grid(T.int64(128), T.int64(128)):
+                with T.sblock():
+                    vi, vj = T.axis.remap("SS", [i, j])
+                    y[vi, vj] = x[vi, vj] + 1.0
+
+        @R.function
+        def foo(
+            x: R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"),
+        ) -> R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"):
+            gv0 = R.dist.call_tir(
+                TestModule.tir_func,
+                x,
+                R.DTensor(
+                    shape=(128, 128), dtype="float32", device_mesh="mesh[0]", placement="S[0], R"
+                ),
+            )
+            return gv0
+
+    _check(TestModule)
+
+
+def test_constant():
+    @I.ir_module(s_tir=True)
+    class TestModule:
+        I.module_attrs({"device_num": 10})
+        I.module_global_infos(
+            {
+                "mesh": [
+                    R.device_mesh((2, 2), I.Range(0, 4)),  # mesh[0]
+                    R.device_mesh((1,), I.Range(4, 5)),  # mesh[1]
+                ]
+            }
+        )
+
+        @T.prim_func(s_tir=True)
+        def tir_func(
+            x: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+            y: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+        ):
+            T.func_attr({"tirx.noalias": True})
+            for i, j in T.grid(T.int64(128), T.int64(128)):
+                with T.sblock():
+                    vi, vj = T.axis.remap("SS", [i, j])
+                    y[vi, vj] = x[vi, vj] + 1.0
+
+        @R.function
+        def foo(
+            x: R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"),
+        ) -> R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"):
+            gv0 = R.dist.call_tir(
+                TestModule.tir_func,
+                x,
+                R.DTensor(
+                    shape=(128, 128), dtype="float32", device_mesh="mesh[0]", placement="S[0], R"
+                ),
+            )
+            gv1 = R.add(gv0, R.dist.const(1.0, ty=R.DTensor((), "float32", "mesh[0]", "R, R")))
+            return gv1
+
+    _check(TestModule)
+
+
+if __name__ == "__main__":
+    tvm.testing.main()
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: E501
+
+import tvm.testing
+from tvm.ir import Range
+from tvm.relax import TensorType
+from tvm.relax.distributed import DeviceMesh, DTensorType, Placement
+from tvm.script.parser import ir as I
+from tvm.script.parser import relax as R
+from tvm.script.parser import tirx as T
+
+
+def _assert_print(obj, expected):
+    if not isinstance(obj, str):
+        obj = obj.script(verbose_expr=True)
+    obj = obj.strip()
+    assert obj == expected.strip(), "\n" + obj
+
+
+def test_constant():
+    constant = R.dist.const(
+        1,
+        ty=R.DTensor((), "float32", device_mesh=DeviceMesh((2, 2), Range(0, 4)), placement="R, R"),
+    )
+    assert (
+        constant.__str__()
+        == """R.dist.const(1.0, R.DTensor((), "float32", R.device_mesh((2, 2), R.Range(0, 4)), "R, R"))"""
+    )
+
+
+def test_dtensor_type():
+    tensor_ty1 = TensorType((32, 32), "float32")
+    tensor_ty2 = TensorType((32, 32), None)
+    obj0 = DTensorType(tensor_ty1, DeviceMesh((2, 2), Range(0, 4)), Placement.from_text("S[1], R"))
+    assert (
+        obj0.__str__()
+        == """R.DTensor((32, 32), "float32", R.device_mesh((2, 2), R.Range(0, 4)), "S[1], R")"""
+    )
+
+    obj1 = DTensorType(tensor_ty2, DeviceMesh((2, 2), Range(0, 4)), Placement.from_text("S[1], R"))
+    assert (
+        obj1.__str__()
+        == """R.DTensor((32, 32), device_mesh=R.device_mesh((2, 2), R.Range(0, 4)), placement="S[1], R")"""
+    )
+
+    obj2 = DTensorType(tensor_ty2, DeviceMesh((2, 2), [0, 1, 2, 3]), Placement.from_text("S[1], R"))
+    assert (
+        obj2.__str__()
+        == """R.DTensor((32, 32), device_mesh=R.device_mesh((2, 2), [0, 1, 2, 3]), placement="S[1], R")"""
+    )
+
+
+@I.ir_module(s_tir=True)
+class TestModule:
+    I.module_attrs({"device_num": 10})
+    I.module_global_infos(
+        {
+            "mesh": [
+                R.device_mesh((2, 2), I.Range(0, 4)),  # mesh[0]
+                R.device_mesh((1,), I.Range(4, 5)),  # mesh[1]
+            ]
+        }
+    )
+
+    @T.prim_func(s_tir=True)
+    def tir_func(
+        x: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+        y: T.Buffer((T.int64(128), T.int64(128)), "float32"),
+    ):
+        T.func_attr({"tirx.noalias": True})
+        for i, j in T.grid(T.int64(128), T.int64(128)):
+            with T.sblock():
+                vi, vj = T.axis.remap("SS", [i, j])
+                y[vi, vj] = x[vi, vj] + 1.0
+
+    @R.function
+    def foo(
+        x: R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"),
+    ) -> R.DTensor((128, 128), "float32", device_mesh="mesh[0]", placement="S[0], R"):
+        gv0 = R.dist.call_tir(
+            TestModule.tir_func,
+            x,
+            R.DTensor(
+                shape=(128, 128), dtype="float32", device_mesh="mesh[0]", placement="S[0], R"
+            ),
+        )
+        return gv0
+
+
+def test_func():
+    _assert_print(
+        TestModule["foo"],
+        """
+# from tvm.script import relax as R
+
+@R.function
+def foo(x: R.DTensor((128, 128), "float32", R.device_mesh((2, 2), R.Range(0, 4)), "S[0], R")) -> R.DTensor((128, 128), "float32", R.device_mesh((2, 2), R.Range(0, 4)), "S[0], R"):
+    gv0 = R.dist.call_tir(tir_func, (x,), out_ty=R.DTensor((128, 128), "float32", R.device_mesh((2, 2), R.Range(0, 4)), "S[0], R"))
+    return gv0
+            """,
+    )
+
+
+def test_module():
+    _assert_print(
+        TestModule,
+        """
+# from tvm.script import ir as I
+# from tvm.script import tirx as T
+# from tvm.tirx.layout import Axis
+# from tvm.script import relax as R
+
+@I.ir_module
+class Module:
+    I.module_attrs({"device_num": 10})
+    I.module_global_infos({"mesh": [R.device_mesh((2, 2), I.Range(0, 4)), R.device_mesh((1,), I.Range(4, 5))]})
+    @T.prim_func(s_tir=True)
+    def tir_func(x: T.Buffer((T.int64(128), T.int64(128)), "float32"), y: T.Buffer((T.int64(128), T.int64(128)), "float32")):
+        T.func_attr({"tirx.noalias": True})
+        # with T.sblock("root"):
+        for i, j in T.grid(T.int64(128), T.int64(128)):
+            with T.sblock(""):
+                vi, vj = T.axis.remap("SS", [i, j])
+                T.reads(x[vi, vj])
+                T.writes(y[vi, vj])
+                y[vi, vj] = x[vi, vj] + T.float32(1.0)
+
+    @R.function
+    def foo(x: R.DTensor((128, 128), "float32", "mesh[0]", "S[0], R")) -> R.DTensor((128, 128), "float32", "mesh[0]", "S[0], R"):
+        cls = Module
+        gv0 = R.dist.call_tir(cls.tir_func, (x,), out_ty=R.DTensor((128, 128), "float32", "mesh[0]", "S[0], R"))
+        return gv0
+    """,
+    )
+
+
+if __name__ == "__main__":
+    tvm.testing.main()