# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. import numpy as np import pytest import tvm import tvm.testing from tvm.script import tirx as T from tvm.testing import env @T.prim_func(s_tir=True) def ptx_cp_async(A: T.Buffer((32, 128), "float16"), B: T.Buffer((32, 128), "float16")) -> None: T.func_attr({"global_symbol": "default_function", "tirx.noalias": True}) bx = T.env_thread("blockIdx.x") tx = T.env_thread("threadIdx.x") T.launch_thread(bx, 1) T.launch_thread(tx, 32) with T.sblock(): A_shared = T.sblock_alloc_buffer([32, 128], "float16", scope="shared") T.reads(A[0:32, 0:128]) T.writes(B[0:32, 0:128]) for i in range(16): T.evaluate( T.ptx.cp_async.legacy( A_shared.data, tx * 128 + 8 * i, A.data, tx * 128 + 8 * i, 16, dtype="float16" ) ) # TODO(masahi): Remove dtype requirement from TVMScript parser T.evaluate(T.ptx.cp_async.commit_group(dtype="")) T.evaluate(T.ptx.cp_async.wait_group(0, dtype="")) for i in range(128): B[tx, i] = A_shared[tx, i] @pytest.mark.gpu @pytest.mark.skipif(not env.has_cuda_compute(8), reason="need cuda compute >= 8.0") def test_ptx_cp_async(): f = ptx_cp_async mod = tvm.compile(f, target="cuda") A_np = np.random.rand(32, 128).astype("float16") B_np = np.zeros((32, 128)).astype("float16") def run_and_check(): dev = tvm.cuda(0) A_nd = tvm.runtime.tensor(A_np, device=dev) B_nd = tvm.runtime.tensor(B_np, device=dev) mod(A_nd, B_nd) tvm.testing.assert_allclose(B_nd.numpy(), A_np) tvm.testing.run_with_gpu_lock(run_and_check) # Note: tests for the indexed barrier API (`create_barriers`, # `ptx_init_barrier_thread_count`, `ptx_arrive_barrier`, `ptx_wait_barrier`, # `ptx_cp_async_barrier`, `ptx_arrive_barrier_expect_tx`) were removed — # fork uses `ptx_mbarrier_*` instead and those intrinsics have no # users elsewhere in this codebase. if __name__ == "__main__": test_ptx_cp_async()