chore: import upstream snapshot with attribution

tests/nightly/python/test_nnapi/__init__.py

@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Infrastructure and tests for NNAPI"""

tests/nightly/python/test_nnapi/conftest.py

@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+import pytest
+
+from tvm import rpc
+
+
+def remote():
+    required_env = ("TVM_TRACKER_HOST", "TVM_TRACKER_PORT", "RPC_DEVICE_KEY")
+    missing = [name for name in required_env if name not in os.environ]
+    if missing:
+        pytest.skip(f"NNAPI remote environment unavailable: {', '.join(missing)} not set")
+    rpc_tracker_host = os.environ["TVM_TRACKER_HOST"]
+    rpc_tracker_port = int(os.environ["TVM_TRACKER_PORT"])
+    rpc_device_key = os.environ["RPC_DEVICE_KEY"]
+    tracker = rpc.connect_tracker(rpc_tracker_host, rpc_tracker_port)
+    remote = tracker.request(rpc_device_key, priority=0, session_timeout=600)
+    return remote, tracker

tests/nightly/python/test_nnapi/infrastructure.py

@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: RUF005
+
+import numpy as np
+
+import tvm
+import tvm.script.relax as R
+from tvm.relax.backend.contrib.nnapi import partition_for_nnapi
+from tvm.support import ndk, utils
+
+
+# pylint: disable=import-outside-toplevel,missing-function-docstring
+def reshape_matmul(mod: tvm.IRModule):
+    from tvm.relax import Expr
+    from tvm.relax.dpl import DFPattern, rewrite_call
+    from tvm.relax.dpl.pattern import is_op, wildcard
+
+    input0 = wildcard()
+    input1 = wildcard()
+    pattern = is_op("relax.matmul")(input0, input1)
+
+    def _rewriter(expr: Expr, matches: dict[DFPattern, Expr]):
+        i0 = matches[input0]
+        i1 = matches[input1]
+        if len(i0.ty.shape) == 2 and len(i1.ty.shape) == 2:
+            i0_shape = [1] + [*i0.ty.shape.values]
+            i1_shape = [1] + [*i1.ty.shape.values]
+            oshape = matches[pattern].ty.shape
+            return R.reshape(R.matmul(R.reshape(i0, i0_shape), R.reshape(i1, i1_shape)), oshape)
+        return expr
+
+    mod["main"] = rewrite_call(pattern, _rewriter, mod["main"])
+    return mod
+
+
+def decompose_clip(mod: tvm.IRModule) -> tvm.IRModule:
+    from tvm.relax import Expr
+    from tvm.relax.dpl import DFPattern, rewrite_call
+    from tvm.relax.dpl.pattern import is_op, wildcard
+
+    input_pattern = wildcard()
+    min_pattern = wildcard()
+    max_pattern = wildcard()
+    pattern = is_op("relax.clip")(input_pattern, min_pattern, max_pattern)
+
+    def _rewriter(expr: Expr, matches: dict[DFPattern, Expr]) -> Expr:  # pylint: disable=unused-argument
+        dtype = matches[input_pattern].ty.dtype
+        return R.minimum(
+            R.maximum(
+                matches[input_pattern],
+                R.const(np.array(matches[min_pattern].value.value).astype(dtype), dtype),
+            ),
+            R.const(np.array(matches[max_pattern].value.value).astype(dtype), dtype),
+        )
+
+    mod["main"] = rewrite_call(pattern, _rewriter, mod["main"])
+    return mod
+
+
+def _build(mod, enable_nnapi):
+    if isinstance(mod, tvm.ir.Call):
+        mod = tvm.IRModule.from_expr(mod)
+
+    if enable_nnapi:
+        mod = tvm.relax.transform.FoldConstant()(mod)
+        mod = reshape_matmul(mod)
+        mod = decompose_clip(mod)
+        mod = partition_for_nnapi(mod)
+
+        mod = tvm.relax.transform.RunCodegen()(mod)
+    ex = tvm.compile(mod, target={"kind": "llvm", "mtriple": "aarch64-linux-android"})
+
+    return ex
+
+
+def _run(remote, tracker, ex, inputs):
+    tmp = utils.tempdir()
+    so_name = "test_mod.so"
+    so_path = tmp / so_name
+    ex.export_library(str(so_path), fcompile=ndk.create_shared, options=["-shared", "-fPIC", "-lm"])
+
+    remote.upload(so_path)
+    dev = remote.cpu(0)
+
+    try:
+        # Execute the model on the remote.
+        remote_ex = remote.load_module(so_name)
+        vm = tvm.relax.VirtualMachine(remote_ex, device=dev)
+
+        inputs = [x.copyto(dev) for x in inputs]
+
+        vm.set_input("main", *inputs)
+        vm.invoke_stateful("main")
+        output = vm.get_outputs("main")
+        output = output.numpy()
+    except Exception as e:
+        # Re-raise all exceptions
+        raise e
+    finally:
+        # Manually close the connection.
+        # See https://discuss.tvm.apache.org/t/trouble-with-rpc-session/14008/.
+        #
+        # TODO: Remove if it does not happen on Python 3.11.
+        remote._sess.get_function("CloseRPCConnection")()
+        tracker.close()
+        pass
+
+    return output
+
+
+def build_and_run(
+    remote,
+    tracker,
+    mod,
+    inputs,
+    enable_nnapi=False,
+):
+    ex = _build(mod, enable_nnapi)
+    return _run(remote, tracker, ex, inputs)

tests/nightly/python/test_nnapi/test_network.py

@@ -0,0 +1,138 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""NNAPI network tests."""
+
+import numpy as np
+import pytest
+
+pytest.importorskip("onnx")
+
+import onnx
+
+import tvm
+from test_nnapi.conftest import remote
+from test_nnapi.infrastructure import build_and_run  # , build_and_run_vm
+from tvm.contrib.download import download_testdata
+from tvm.relax.frontend.onnx import from_onnx
+from tvm.testing import env
+
+
+def _build_and_run_network(remote_obj, tracker, mod, input_data):
+    """Helper function to build and run a network."""
+
+    def execute_on_host(mod, inputs):
+        with tvm.transform.PassContext(opt_level=3):
+            ex = tvm.compile(mod, target="llvm")
+        dev = tvm.cpu(0)
+        vm = tvm.relax.VirtualMachine(ex, device=dev)
+        output = vm["main"](*inputs)
+        return output.numpy()
+
+    outputs = []
+    for nnapi in [True, False]:
+        if nnapi:
+            outputs.append(
+                build_and_run(
+                    remote_obj,
+                    tracker,
+                    mod,
+                    input_data,
+                    enable_nnapi=nnapi,
+                )
+            )
+        else:
+            outputs.append(execute_on_host(mod, input_data))
+    return outputs
+
+
+def get_network(name, dtype, input_shape=(1, 3, 224, 224)):
+    def download_model(model_url, name):
+        model_path = download_testdata(model_url, name + ".onnx", module="onnx")
+        onnx_model = onnx.load(model_path)
+
+        shape_dict = {"x": input_shape}
+        mod = from_onnx(onnx_model, shape_dict)
+        return mod
+
+    def create_model(name):
+        if "vgg11" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/vgg11_Opset18_timm/vgg11_Opset18.onnx"
+        elif "mobilenetv3" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/mobilenetv3_large_100_miil_Opset17_timm/mobilenetv3_large_100_miil_Opset17.onnx"
+        elif "alexnet" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/alexnet_Opset17_torch_hub/alexnet_Opset17.onnx"
+        elif "resnet50" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/resnet50_Opset18_timm/resnet50_Opset18.onnx"
+        elif "resnet34" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/resnet34_Opset18_timm/resnet34_Opset18.onnx"
+        elif "resnet18" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/resnet18_Opset18_timm/resnet18_Opset18.onnx"
+        elif "squeezenet" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/squeezenet1_1_Opset18_torch_hub/squeezenet1_1_Opset18.onnx"
+        elif "vgg16" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/vgg16_Opset18_timm/vgg16_Opset18.onnx"
+        elif "vgg19" == name:
+            model_url = "https://github.com/onnx/models/raw/bec48b6a70e5e9042c0badbaafefe4454e072d08/Computer_Vision/vgg19_Opset18_timm/vgg19_Opset18.onnx"
+        else:
+            assert False, f"Not supported model {name}"
+
+        return download_model(model_url, name)
+
+    mod = create_model(name)
+    return mod, {"data": (input_shape, dtype)}
+
+
+@pytest.mark.parametrize(
+    "name",
+    [
+        "alexnet",
+        "vgg11",
+        "vgg16",
+        "vgg19",
+        "resnet18",
+        "resnet34",
+        "resnet50",
+        "squeezenet",
+        "mobilenetv3",
+    ],
+)
+@pytest.mark.parametrize(
+    "dtype",
+    [
+        "float32",
+    ],
+)
+@pytest.mark.skipif(not env.build_flag_enabled("USE_NNAPI_CODEGEN"), reason="need nnapi")
+def test_network(name, dtype):
+    remote_obj, tracker = remote()
+    print(f"Network evaluating {name} with dtype {dtype}")
+    np.random.seed(0)
+    mod, inputs = get_network(name, dtype)
+    input_data = {}
+
+    for _name, (shape, _dtype) in inputs.items():
+        input_data[_name] = np.random.uniform(-1.0, 1.0, shape).astype(_dtype)
+
+    inputs_tvm: list[tvm.runtime.Tensor] = [tvm.runtime.tensor(v) for k, v in input_data.items()]
+    outputs = _build_and_run_network(remote_obj, tracker, mod, inputs_tvm)
+    nnapi_out = outputs[0]
+    expected_out = outputs[1]
+    tvm.testing.assert_allclose(nnapi_out, expected_out, rtol=1e-4, atol=1e-5)
+
+
+if __name__ == "__main__":
+    tvm.testing.main()

tests/nightly/python/test_nnapi/test_ops.py

@@ -0,0 +1,361 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# ruff: noqa: F841
+"""NNAPI integration operator tests."""
+
+import numpy as np
+import pytest
+
+import tvm
+import tvm.script
+import tvm.script.relax as R
+import tvm.script.tirx as T
+from test_nnapi.conftest import remote
+from test_nnapi.infrastructure import build_and_run
+
+
+def _build_and_run_network(remote_obj, tracker, mod, input_data):
+    """Helper function to build and run a network."""
+
+    def execute_on_host(mod, inputs):
+        with tvm.transform.PassContext(opt_level=3):
+            ex = tvm.compile(mod, target="llvm")
+        dev = tvm.cpu(0)
+        vm = tvm.relax.VirtualMachine(ex, device=dev)
+        output = vm["main"](*inputs)
+        return output.numpy()
+
+    outputs = []
+    for nnapi in [True, False]:
+        if nnapi:
+            outputs.append(
+                build_and_run(
+                    remote_obj,
+                    tracker,
+                    mod,
+                    input_data,
+                    enable_nnapi=nnapi,
+                )
+            )
+        else:
+            outputs.append(execute_on_host(mod, input_data))
+    return outputs
+
+
+@pytest.mark.parametrize(
+    "op",
+    [
+        R.exp,
+        R.log,
+        R.negative,
+        R.sqrt,
+        R.rsqrt,
+        R.floor,
+        R.nn.relu,
+        R.nn.softmax,
+        R.sigmoid,
+        R.tanh,
+        R.abs,
+    ],
+)
+def test_unary(op, input_shape=(1, 2, 8, 5)):
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(i0: R.Tensor((1, 2, 8, 5), "float32")) -> R.Tensor((1, 2, 8, 5), "float32"):
+                with R.dataflow():
+                    t0 = op(i0)
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[np.random.uniform(size=(1, 2, 8, 5)).astype("float32")],
+    )
+
+
+@pytest.mark.parametrize(
+    "op",
+    [
+        R.power,
+        R.greater,
+        R.add,
+        R.multiply,
+        R.subtract,
+        R.equal,
+        R.less,
+        R.less_equal,
+        R.not_equal,
+        R.maximum,
+        R.minimum,
+        R.greater_equal,
+    ],
+)
+def test_elementwise_binary(op, input_shape=(1, 2, 8, 5)):
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((1, 2, 8, 5), "float32"),
+                i1: R.Tensor((1, 2, 8, 5), "float32"),
+            ) -> R.Tensor((1, 2, 8, 5), "float32"):
+                with R.dataflow():
+                    t0 = op(i0, i1)
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            np.random.uniform(size=input_shape).astype("float32"),
+            np.random.uniform(size=input_shape).astype("float32"),
+        ],
+    )
+
+
+def test_divide(input_shape=(1, 2, 8, 5)):
+    remote_obj, tracker = remote()
+
+    def create_model(input_shape) -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((1, 2, 8, 5), "float32"),
+                i1: R.Tensor((1, 2, 8, 5), "float32"),
+            ) -> R.Tensor((1, 2, 8, 5), "float32"):
+                with R.dataflow():
+                    t0 = R.divide(i0, i1)
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model(input_shape)
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            np.random.uniform(size=input_shape).astype("float32"),
+            np.random.uniform(size=input_shape).astype("float32") + np.ones(input_shape, "float32"),
+        ],
+    )
+
+
+def test_matmul():
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((5, 3, 4), "float32"),
+                i1: R.Tensor((5, 4, 8), "float32"),
+            ) -> R.Tensor((5, 3, 8), "float32"):
+                with R.dataflow():
+                    t0 = R.matmul(i0, i1)
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            np.random.random(size=(5, 3, 4)).astype("float32"),
+            np.random.random(size=(5, 4, 8)).astype("float32"),
+        ],
+    )
+
+
+def test_permute_dims():
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((5, 4, 8), "float32"),
+            ) -> R.Tensor((8, 5, 4), "float32"):
+                with R.dataflow():
+                    t0 = R.permute_dims(i0, axes=[2, 0, 1])
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            np.random.random(size=(5, 4, 8)).astype("float32"),
+        ],
+    )
+
+
+def test_astype():
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((8, 10, 15), "float32"),
+            ) -> R.Tensor((8, 10, 15), "float16"):
+                with R.dataflow():
+                    t0: R.Tensor((8, 10, 15), "float16") = R.astype(i0, dtype="float16")
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            tvm.runtime.tensor(np.random.uniform(size=(8, 10, 15)).astype("float32")),
+        ],
+    )
+
+
+def test_mean():
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((1, 10, 15), "float32"),
+            ) -> R.Tensor((1, 10, 1), "float32"):
+                n = T.int64()
+                with R.dataflow():
+                    t0: R.Tensor((1, 10, 1), "float32") = R.mean(i0, axis=[-1], keepdims=True)
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            tvm.runtime.tensor(np.random.uniform(size=(1, 10, 15)).astype("float32")),
+        ],
+    )
+
+
+def test_conv2d():
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((1, 3, 224, 224), "float32"),
+                i1: R.Tensor((64, 3, 3, 3), "float32"),
+                i2: R.Tensor((1, 64, 1, 1), "float32"),
+            ):
+                with R.dataflow():
+                    t0 = R.nn.conv2d(i0, i1, strides=(1, 1), padding=(1, 1))
+                    t0 = R.add(i2, t0)
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            np.random.random(size=(1, 3, 224, 224)).astype("float32"),
+            np.random.random(size=(64, 3, 3, 3)).astype("float32"),
+            np.random.random(size=(1, 64, 1, 1)).astype("float32"),
+        ],
+    )
+
+
+def test_max_pool2d():
+    remote_obj, tracker = remote()
+
+    def create_model() -> tvm.IRModule:
+        @tvm.script.ir_module
+        class Module:
+            @R.function
+            def main(
+                i0: R.Tensor((1, 1, 28, 28), "float32"),
+            ):
+                with R.dataflow():
+                    t0 = R.nn.max_pool2d(i0, pool_size=(1, 1), strides=(1, 1), padding=(0, 0))
+                    R.output(t0)
+                return t0
+
+        return Module
+
+    mod = create_model()
+    verify(
+        remote_obj,
+        tracker,
+        mod,
+        inputs=[
+            np.random.random(size=(1, 1, 28, 28)).astype("float32"),
+        ],
+    )
+
+
+def verify(remote_obj, tracker, mod, inputs):
+    inputs_tvm: list[tvm.runtime.Tensor] = [tvm.runtime.tensor(v) for v in inputs]
+    outputs = _build_and_run_network(remote_obj, tracker, mod, inputs_tvm)
+    nnapi_out = outputs[0]
+    expected_out = outputs[1]
+    tvm.testing.assert_allclose(nnapi_out, expected_out, rtol=1e-4, atol=1e-5)
+
+
+if __name__ == "__main__":
+    tvm.testing.main()