wehub-resource-sync
417 lines
18 KiB
JSON
417 lines
18 KiB
JSON
{
|
|
"$schemaVersion": 2,
|
|
"_documentation": {
|
|
"purpose": "Declarative configuration for ./test.sh. Editing this file is the supported way to add, drop, or reconfigure tests for either the local-host or on-device Android driver.",
|
|
"top_level": {
|
|
"android": "Stages dispatched by `./test.sh android <serial>` — runs on the connected device after a cross-build push.",
|
|
"local": "Stages dispatched by `./test.sh local` — runs entirely on the host (CPU only).",
|
|
"llm": "LLM smoke-test configuration. Active in both modes when the model is provisioned."
|
|
},
|
|
"stage_object": {
|
|
"name": "Stage label, also used for the per-stage log filename.",
|
|
"filter": "Filter tag — see filter_tags below. Matched against the user's --runs selector.",
|
|
"comment": "Free-form note about why this stage exists / what it covers.",
|
|
"binary": "On-device binary: 'run_test' (default), 'v2basic' (MNNV2Basic.out), 'backendtest' (backendTest.out), or 'benchmark' (benchmark.out).",
|
|
"prefix": "First positional arg to run_test.out — test-name prefix (or 'all').",
|
|
"type": "Forward type. 0 = CPU, 3 = OpenCL, 7 = Vulkan.",
|
|
"precision": "BackendConfig::PrecisionMode. 0 = Normal, 1 = High, 2 = Low.",
|
|
"threadOrGpuMode": "CPU: thread count. GPU: gpuMode bitmask (TUNING_NONE=1, TUNING_HEAVY=2, TUNING_WIDE=4; MEMORY_BUFFER=64, MEMORY_IMAGE=128). e.g. 129 = TUNING_NONE | MEMORY_IMAGE.",
|
|
"tag": "Test-report flag forwarded to run_test.out.",
|
|
"memory": "BackendConfig::MemoryMode (0=Normal, 1=High, 2=Low). Omit when not setting.",
|
|
"dynamicOption": "RuntimeHint::dynamicQuantOption (0..7). Omit when not setting.",
|
|
"kleidiAi": "argv[8]: 1 enables KleidiAI on ARM. Omit when not setting.",
|
|
"skip": "Array of EXACT test names to skip (passed via MNN_TEST_SKIP env to MNNTestSuite::run()). Use to exclude tests that hit known device/driver bugs without losing coverage of their siblings.",
|
|
"args": "(smoke/bench only) Positional argv array; '{model}' / '{models_dir}' get per-iteration substitution."
|
|
},
|
|
"filter_tags": {
|
|
"cpu": "Plain CPU stages (also covers lowmem and llm).",
|
|
"opencl-image": "OpenCL with MNN_GPU_MEMORY_IMAGE.",
|
|
"opencl-buffer": "OpenCL with MNN_GPU_MEMORY_BUFFER (some upstream kernels are buggy on Mali Bifrost — see 'skip' lists).",
|
|
"vulkan": "Vulkan backend.",
|
|
"lowmem": "Low-memory configurations (memory=2).",
|
|
"smoke-opencl": "Smoke A/B on OpenCL.",
|
|
"smoke-vulkan": "Smoke A/B on Vulkan.",
|
|
"llm": "LLM smoke test."
|
|
},
|
|
"skip_rationale": {
|
|
"opencl_image_sequence_pollution": "On the OpenCL IMAGE path, cumprod/cumsum/ROIPooling pass standalone but fail when run after the long preceding test sequence — the OpenCL→CPU fallback path leaks state through the IMAGE-memtype tensor pool. Skipped in the bulk run; covered by per-test isolation if you need them.",
|
|
"opencl_buffer_mali_loop_bugs": "BatchMatMul, col2im, cumprod, cumsum, ROIPooling, ScatterElementsTest, ScatterNdTest all decompose to the BUFFER-mode `loop`/`gather` kernel set on OpenCL. On Mali Bifrost these kernels return zero-or-tiny outputs (BatchMatMul large-K accumulates to ~0; Col2Im fuse path zeros batch index 1; gather-based ops fall back via WrapExecution and the gather kernel writes zero). All pass on the IMAGE memtype path, exercised by the 'unit/opencl/op' stage.",
|
|
"opencl_cumsum_reshape": "op/cumsum_reshape drives the same OpenCL CumSum scan as op/cumsum (already skipped): the scan kernel returns [1,0,0,0,0] — the first element only, not the running sum — on both IMAGE and BUFFER. It is a CPU-side regression test (While-output makeFullRef) that does not need GPU coverage. Skipped on both OpenCL stages.",
|
|
"opencl_image_scatterelements": "op/ScatterElementsTest's MUL-reduction subtest with duplicate indices computes 2.2 instead of 4.62 on the OpenCL IMAGE path — only the first update is applied, the duplicate-index second update is dropped. Already skipped on BUFFER (gather/scatter WrapExecution fallback); now skipped on IMAGE for the same scatter-reduction bug.",
|
|
"vulkan_buggy_kernels": "op/binary/powInt8 returns 0 instead of 16 on Vulkan; op/binary/AddBroast returns -2 instead of 0 then SIGSEGVs the process. Both are upstream Vulkan-backend bugs unrelated to anything in this fork.",
|
|
"vulkan_device_broken_ops": "On this test device the Vulkan backend is broadly unreliable, so the Vulkan stage skips a larger set than OpenCL (all of these pass on CPU). Two classes: (1) HANGS that wedge the suite with no output — op/convolution/weighti8i4conv2d (int4 quant conv, stuck >40 min), op/convolution/depthwise_conv and op/lowMemory/mixedKernel (each stuck >5 min). (2) NUMERIC FAILURES — op/binary/pow, op/cumprod, op/cumsum (Loop/scan family, also skipped on OpenCL), op/Deconvolutionfull, op/GridSample3D, op/AvePool3d, op/ROIPooling, op/Interp + op/InterpInt8 (InterpType 3 / cubic), op/strideSliceWrite, op/unary/erfInt8 + op/unary/erfcInt8. These are upstream Vulkan-backend limitations on this device, not regressions in this fork."
|
|
}
|
|
},
|
|
|
|
"android": {
|
|
"_comment": "Stages run by `./test.sh android`. Smoke + bench stages iterate per model from android.smoke_models below.",
|
|
"stages": [
|
|
{
|
|
"name": "unit/cpu/all",
|
|
"filter": "cpu",
|
|
"comment": "Single-thread CPU full test suite at Precision_Normal — the broadest sanity check.",
|
|
"prefix": "all",
|
|
"type": 0,
|
|
"precision": 0,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "64",
|
|
"memory": 0
|
|
},
|
|
{
|
|
"name": "unit/cpu/op-mt",
|
|
"filter": "cpu",
|
|
"comment": "4-thread op-only sweep. Exercises threadpool paths that the single-thread run can't hit.",
|
|
"prefix": "op",
|
|
"type": 0,
|
|
"precision": 0,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "multi64",
|
|
"memory": 0
|
|
},
|
|
{
|
|
"name": "unit/cpu/op-fp16-conv",
|
|
"filter": "cpu",
|
|
"comment": "FP16-precision convolution sweep (precision=Low).",
|
|
"prefix": "op/convolution",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "fp16multi64",
|
|
"memory": 0
|
|
},
|
|
{
|
|
"name": "unit/cpu/op-fp16-col2im",
|
|
"filter": "cpu",
|
|
"comment": "FP16-precision Col2Im check.",
|
|
"prefix": "op/col2im",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "fp16col2im64",
|
|
"memory": 0
|
|
},
|
|
{
|
|
"name": "unit/cpu/op-fp16-roi",
|
|
"filter": "cpu",
|
|
"comment": "FP16-precision ROIAlign / ROIPooling check (prefix 'op/R').",
|
|
"prefix": "op/R",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "fp16roipooling64",
|
|
"memory": 0
|
|
},
|
|
{
|
|
"name": "unit/opencl/op",
|
|
"filter": "opencl-image",
|
|
"comment": "OpenCL TUNING_NONE | MEMORY_IMAGE = 129. Correctness sweep — TUNING_WIDE adds many seconds of per-kernel tuning that's wasted on a single-shot run.",
|
|
"prefix": "op",
|
|
"type": 3,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 129,
|
|
"tag": "64",
|
|
"memory": 0,
|
|
"skip": [
|
|
"op/cumprod",
|
|
"op/cumsum",
|
|
"op/cumsum_reshape",
|
|
"op/ROIPooling",
|
|
"op/ScatterElementsTest",
|
|
"op/ScatterNdTest",
|
|
"op/ConvInt8/winograd"
|
|
]
|
|
},
|
|
{
|
|
"name": "unit/opencl/op-buffer",
|
|
"filter": "opencl-buffer",
|
|
"comment": "OpenCL TUNING_NONE | MEMORY_BUFFER = 65. Catches regressions in BUFFER-only creators (e.g. Attention) that the IMAGE path masks via CPU fallback.",
|
|
"prefix": "op",
|
|
"type": 3,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 65,
|
|
"tag": "64",
|
|
"memory": 0,
|
|
"skip": [
|
|
"op/BatchMatMul",
|
|
"op/col2im",
|
|
"op/cumprod",
|
|
"op/cumsum",
|
|
"op/cumsum_reshape",
|
|
"op/ROIPooling",
|
|
"op/ScatterElementsTest",
|
|
"op/ScatterNdTest",
|
|
"op/ConvInt8/winograd"
|
|
]
|
|
},
|
|
{
|
|
"name": "unit/vulkan/op",
|
|
"filter": "vulkan",
|
|
"comment": "Vulkan TUNING_NONE = 1. Vulkan ignores MEMORY_* bits — they're set via the MNN_VULKAN_IMAGE CMake option.",
|
|
"prefix": "op",
|
|
"type": 7,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "64",
|
|
"memory": 0,
|
|
"skip": [
|
|
"op/binary/powInt8",
|
|
"op/binary/AddBroast",
|
|
"op/binary/pow",
|
|
"op/cumprod",
|
|
"op/cumsum",
|
|
"op/convolution/weighti8i4conv2d",
|
|
"op/convolution/depthwise_conv",
|
|
"op/lowMemory/mixedKernel",
|
|
"op/Deconvolutionfull",
|
|
"op/GridSample3D",
|
|
"op/AvePool3d",
|
|
"op/ROIPooling",
|
|
"op/Interp",
|
|
"op/InterpInt8",
|
|
"op/strideSliceWrite",
|
|
"op/unary/erfInt8",
|
|
"op/unary/erfcInt8"
|
|
]
|
|
},
|
|
{
|
|
"name": "lowmem/dyn-p1-t1",
|
|
"filter": "lowmem",
|
|
"comment": "Low-memory dynamic-quant: precision=High, single-thread, dynamicOption=2.",
|
|
"prefix": "op/lowMemory",
|
|
"type": 0,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 2
|
|
},
|
|
{
|
|
"name": "lowmem/dyn-p2-t1",
|
|
"filter": "lowmem",
|
|
"comment": "Low-memory dynamic-quant: precision=Low, single-thread, dynamicOption=2.",
|
|
"prefix": "op/lowMemory",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 2
|
|
},
|
|
{
|
|
"name": "lowmem/dyn-p1-t4",
|
|
"filter": "lowmem",
|
|
"comment": "Low-memory dynamic-quant: precision=High, 4-thread, dynamicOption=2.",
|
|
"prefix": "op/lowMemory",
|
|
"type": 0,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 2
|
|
},
|
|
{
|
|
"name": "lowmem/dyn-p2-t4",
|
|
"filter": "lowmem",
|
|
"comment": "Low-memory dynamic-quant: precision=Low, 4-thread, dynamicOption=2.",
|
|
"prefix": "op/lowMemory",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 2
|
|
},
|
|
{
|
|
"name": "lowmem/wdeq-p1",
|
|
"filter": "lowmem",
|
|
"comment": "Weight-dequant low-memory mode: precision=High, single-thread (dynamicOption omitted).",
|
|
"prefix": "op/lowMemory",
|
|
"type": 0,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "64"
|
|
},
|
|
{
|
|
"name": "lowmem/wdeq-p2",
|
|
"filter": "lowmem",
|
|
"comment": "Weight-dequant low-memory mode: precision=Low, single-thread (dynamicOption omitted).",
|
|
"prefix": "op/lowMemory",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "64"
|
|
},
|
|
{
|
|
"name": "lowmem/i8i4-d1-p2",
|
|
"filter": "lowmem",
|
|
"comment": "weighti8i4 conv2d: precision=Low, 4-thread, memory=Low, dynamicOption=1.",
|
|
"prefix": "op/convolution/weighti8i4conv2d",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 1
|
|
},
|
|
{
|
|
"name": "lowmem/i8i4-d1-p1",
|
|
"filter": "lowmem",
|
|
"comment": "weighti8i4 conv2d: precision=High, 4-thread, memory=Low, dynamicOption=1.",
|
|
"prefix": "op/convolution/weighti8i4conv2d",
|
|
"type": 0,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 1
|
|
},
|
|
{
|
|
"name": "lowmem/i8i4-d2-p2",
|
|
"filter": "lowmem",
|
|
"comment": "weighti8i4 conv2d: precision=Low, 4-thread, memory=Low, dynamicOption=2.",
|
|
"prefix": "op/convolution/weighti8i4conv2d",
|
|
"type": 0,
|
|
"precision": 2,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 2
|
|
},
|
|
{
|
|
"name": "lowmem/i8i4-d2-p1",
|
|
"filter": "lowmem",
|
|
"comment": "weighti8i4 conv2d: precision=High, 4-thread, memory=Low, dynamicOption=2.",
|
|
"prefix": "op/convolution/weighti8i4conv2d",
|
|
"type": 0,
|
|
"precision": 1,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "64",
|
|
"memory": 2,
|
|
"dynamicOption": 2
|
|
}
|
|
],
|
|
|
|
"smoke_models": [
|
|
"MobileNet/v1/mobilenet_v1.caffe.mnn",
|
|
"MobileNet/v2/mobilenet_v2.caffe.mnn",
|
|
"SqueezeNet/v1.0/squeezenet_v1.0.caffe.mnn",
|
|
"SqueezeNet/v1.1/squeezenet_v1.1.caffe.mnn"
|
|
],
|
|
|
|
"smoke_a_stages": [
|
|
{
|
|
"name": "smokeA/cpu",
|
|
"filter": "cpu",
|
|
"comment": "Forward-smoke on CPU. Catches model-load and shape-inference regressions.",
|
|
"binary": "v2basic",
|
|
"args": ["{model}", "1", "0", "0"]
|
|
},
|
|
{
|
|
"name": "smokeA/opencl",
|
|
"filter": "smoke-opencl",
|
|
"comment": "Forward-smoke on OpenCL with default gpuMode.",
|
|
"binary": "v2basic",
|
|
"args": ["{model}", "1", "0", "3"]
|
|
},
|
|
{
|
|
"name": "smokeA/vulkan",
|
|
"filter": "smoke-vulkan",
|
|
"comment": "Forward-smoke on Vulkan.",
|
|
"binary": "v2basic",
|
|
"args": ["{model}", "1", "0", "7"]
|
|
}
|
|
],
|
|
|
|
"smoke_b_stages": [
|
|
{
|
|
"name": "smokeB/opencl",
|
|
"filter": "smoke-opencl",
|
|
"comment": "Numeric CPU-vs-OpenCL comparison; tolerance 0.05.",
|
|
"binary": "backendtest",
|
|
"args": ["{model}", "3", "0.05"]
|
|
},
|
|
{
|
|
"name": "smokeB/vulkan",
|
|
"filter": "smoke-vulkan",
|
|
"comment": "Numeric CPU-vs-Vulkan comparison; tolerance 0.05.",
|
|
"binary": "backendtest",
|
|
"args": ["{model}", "7", "0.05"]
|
|
}
|
|
],
|
|
|
|
"bench_stages": [
|
|
{
|
|
"name": "bench/cpu",
|
|
"filter": "cpu",
|
|
"comment": "10-iter benchmark on CPU, 4-thread, Precision_High.",
|
|
"binary": "benchmark",
|
|
"args": ["{models_dir}", "10", "2", "0", "4", "1"]
|
|
},
|
|
{
|
|
"name": "bench/opencl",
|
|
"filter": "cpu",
|
|
"comment": "10-iter benchmark on OpenCL with TUNING_WIDE | MEMORY_IMAGE = 132. TUNING_WIDE intentional — perf is the point.",
|
|
"binary": "benchmark",
|
|
"args": ["{models_dir}", "10", "2", "3", "132", "1"]
|
|
},
|
|
{
|
|
"name": "bench/vulkan",
|
|
"filter": "cpu",
|
|
"comment": "10-iter benchmark on Vulkan with TUNING_WIDE.",
|
|
"binary": "benchmark",
|
|
"args": ["{models_dir}", "10", "2", "7", "4", "1"]
|
|
}
|
|
]
|
|
},
|
|
|
|
"local": {
|
|
"_comment": "Stages run by `./test.sh local`. Local mode is CPU-only by design — Stage B (CPU-vs-backend) is meaningless without a GPU build.",
|
|
"stages": [
|
|
{
|
|
"name": "unit/cpu",
|
|
"filter": "cpu",
|
|
"comment": "Single-thread full unit-test suite on the host build.",
|
|
"prefix": "",
|
|
"type": 0,
|
|
"precision": 0,
|
|
"threadOrGpuMode": 1,
|
|
"tag": "",
|
|
"memory": 0
|
|
},
|
|
{
|
|
"name": "unit/cpu-mt",
|
|
"filter": "cpu",
|
|
"comment": "4-thread op-only sweep on the host build.",
|
|
"prefix": "op",
|
|
"type": 0,
|
|
"precision": 0,
|
|
"threadOrGpuMode": 4,
|
|
"tag": "",
|
|
"memory": 0
|
|
}
|
|
],
|
|
|
|
"smoke_a_stages": [
|
|
{
|
|
"name": "smokeA/cpu",
|
|
"filter": "cpu",
|
|
"comment": "Forward-smoke on CPU — catches model-load / shape-inference regressions.",
|
|
"binary": "v2basic",
|
|
"args": ["{model}", "1", "0", "0"]
|
|
}
|
|
]
|
|
},
|
|
|
|
"llm": {
|
|
"_comment": "LLM smoke test. Provisioning is lazy (deferred to the llm stage, so other stages run offline) and configured via env vars: LLM_MODEL_DIR (use an on-disk model, no download), LLM_MODEL_SOURCE=huggingface|modelscope, LLM_MODEL_URL_BASE. Skipped automatically when the model is unavailable.",
|
|
"model_repo": "taobao-mnn/Qwen2.5-0.5B-Instruct-MNN",
|
|
"config_file": "config.json",
|
|
"prompt_file": "prompt.txt",
|
|
"stage": {
|
|
"name": "llm/{model_name}",
|
|
"filter": "llm",
|
|
"comment": "Runs llm_demo against the provisioned config.json + prompt.txt."
|
|
}
|
|
}
|
|
}
|