chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,552 @@
|
||||
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "hytlass/gemm_coord.h"
|
||||
|
||||
namespace ap {
|
||||
|
||||
constexpr int kNumConfigsHalf = 28;
|
||||
constexpr int kNumConfigsFloat = 13;
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct SwizzleWrapper {
|
||||
using Type =
|
||||
hytlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<SwizzleFactor>;
|
||||
};
|
||||
|
||||
#define AP_AUTOTUNE(func, stream_ptr, count, ...) \
|
||||
{ \
|
||||
using FuncType = decltype(func<0>); \
|
||||
static int selected_config_id = -1; \
|
||||
static std::vector<std::function<FuncType>> matmul_functions = \
|
||||
[]<std::size_t... Is>(std::index_sequence<Is...>) { \
|
||||
return std::vector<std::function<FuncType>>{func<Is>...}; \
|
||||
} \
|
||||
(std::make_index_sequence<count>()); \
|
||||
if (selected_config_id == -1) { \
|
||||
selected_config_id = \
|
||||
ap::ProfileBestConfig(matmul_functions, stream_ptr, ##__VA_ARGS__); \
|
||||
} \
|
||||
matmul_functions[selected_config_id](__VA_ARGS__); \
|
||||
}
|
||||
|
||||
#define AP_AUTOTUNE_half(func, stream_ptr, ...) \
|
||||
AP_AUTOTUNE(func, stream_ptr, ap::kNumConfigsHalf, __VA_ARGS__)
|
||||
#define AP_AUTOTUNE_float(func, stream_ptr, ...) \
|
||||
AP_AUTOTUNE(func, stream_ptr, ap::kNumConfigsFloat, __VA_ARGS__)
|
||||
#define AP_AUTOTUNE_bfloat16(func, stream_ptr, ...) \
|
||||
AP_AUTOTUNE_half(func, stream_ptr, __VA_ARGS__)
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched, int Id = 0>
|
||||
struct GemmTuningConfigs {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 2;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = Id;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 1> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 1;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 2> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 2;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 3> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 64, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 3;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 4> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 4;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 5> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 5;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 6> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 6;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 7> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 64, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 7;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 8> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 8;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 9> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 9;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 10> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 32, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 10;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 11> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 11;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 12> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 12;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 13> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 64, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 13;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 14> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 14;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 15> {
|
||||
using TShape = hytlass::gemm::GemmShape<32, 64, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<16, 32, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 15;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 16> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 16;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 17> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 17;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 18> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 18;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 19> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 6;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 19;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 20> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 6;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 20;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 21> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 10;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 21;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 22> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 256, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 2;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 22;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 23> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 256, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 23;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 24> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 24;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 25> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 25;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 26> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 128, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 26;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 27> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 64, 64>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 64>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 27;
|
||||
};
|
||||
|
||||
// Specialization for float
|
||||
template <int SwizzleFactor, bool Batched, int Id>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, Id> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 64, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = Id;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 1> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 32, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 1;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 2> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 2;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 3> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 256, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 3;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 4> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 256, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 4;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 5> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 5;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 6> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 6;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 7> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 7;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 8> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 64, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 8;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 9> {
|
||||
using TShape = hytlass::gemm::GemmShape<256, 64, 32>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 32>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 9;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 10> {
|
||||
using TShape = hytlass::gemm::GemmShape<64, 128, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 10;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 11> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 64, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<64, 32, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 11;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 12> {
|
||||
using TShape = hytlass::gemm::GemmShape<128, 128, 16>;
|
||||
using WShape = hytlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = hytlass::gemm::GemmShape<16, 16, 8>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 12;
|
||||
};
|
||||
|
||||
struct DefaultConfig {
|
||||
static constexpr int kConfigId = 0;
|
||||
static constexpr int kSwizzleFactor = 1;
|
||||
static constexpr bool kBatched = false;
|
||||
};
|
||||
|
||||
} // namespace ap
|
||||
@@ -0,0 +1,254 @@
|
||||
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "hytlass/epilogue/thread/linear_combination_bias_elementwise.h"
|
||||
#include "hytlass/gemm/device/gemm_universal.h"
|
||||
#include "hytlass/gemm/device/gemm_universal_with_broadcast.h"
|
||||
#include "hytlass/util/device_memory.h"
|
||||
|
||||
#include "cutlass_patch/batched_matrix_coord.h"
|
||||
#include "cutlass_patch/epilogue/thread/linear_combination_unary.h"
|
||||
#include "cutlass_patch/epilogue/thread/linear_combination_variadic.h"
|
||||
#include "cutlass_patch/gemm/device/gemm_universal_with_variadic.h"
|
||||
#include "cutlass_patch/hip/all_tuning_configs.h"
|
||||
|
||||
#include "params.h" // NOLINT
|
||||
|
||||
#define CHECK_HYTLASS(status) \
|
||||
{ \
|
||||
auto error = status; \
|
||||
if (error != hytlass::Status::kSuccess) { \
|
||||
std::cerr << "HYTLASS error = " << int(error) << " (" \
|
||||
<< hytlassGetStatusString(error) << ")" \
|
||||
<< " at line " << __LINE__ << std::endl; \
|
||||
std::abort(); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace ap {
|
||||
|
||||
using MatrixCoord = cutlass_patch::BatchedMatrixCoord;
|
||||
using bfloat16 = __hip_bfloat16;
|
||||
|
||||
// Operation performed by GEMM
|
||||
template <typename ElementT>
|
||||
struct GemmOperation {
|
||||
using Type = hytlass::arch::OpMultiplyAdd;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct GemmOperation<float> {
|
||||
using Type = hytlass::arch::OpMultiplyAddFastF32;
|
||||
};
|
||||
|
||||
static hytlass::gemm::GemmUniversalMode GetGemmMode(int batch_count) {
|
||||
return batch_count > 1 ? hytlass::gemm::GemmUniversalMode::kBatched
|
||||
: hytlass::gemm::GemmUniversalMode::kGemm;
|
||||
}
|
||||
|
||||
static void *GetWorkspace(size_t workspace_size) {
|
||||
static hytlass::device_memory::allocation<uint8_t> workspace;
|
||||
if (workspace.size() < workspace_size) {
|
||||
workspace.reset(workspace_size);
|
||||
}
|
||||
return workspace.get();
|
||||
}
|
||||
|
||||
template <typename GemmFunc>
|
||||
hytlass::Status SetMaxDynamicSharedMemorySize() {
|
||||
hipError_t hiprt_result;
|
||||
|
||||
// If requires more than 48KB: configure for extended, dynamic shared memory
|
||||
if constexpr (GemmFunc::kSharedStorageSize >= (48 << 10)) {
|
||||
hiprt_result = hipFuncSetAttribute(
|
||||
(const void *)hytlass::Kernel2<typename GemmFunc::GemmKernel>,
|
||||
hipFuncAttributeMaxDynamicSharedMemorySize,
|
||||
GemmFunc::kSharedStorageSize);
|
||||
if (hiprt_result != hipSuccess) {
|
||||
HYTLASS_TRACE_HOST("hipFuncSetAttribute() returned error "
|
||||
<< hipGetErrorString(hiprt_result));
|
||||
return hytlass::Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
#if AP_ENABLE_DEBUG
|
||||
// Update SM occupancy member
|
||||
int sm_occupancy = -1;
|
||||
hiprt_result = hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(
|
||||
&sm_occupancy,
|
||||
hytlass::Kernel2<typename GemmFunc::GemmKernel>,
|
||||
GemmFunc::GemmKernel::kThreadCount,
|
||||
GemmFunc::kSharedStorageSize,
|
||||
hipOccupancyDisableCachingOverride);
|
||||
if (hiprt_result != hipSuccess) {
|
||||
HYTLASS_TRACE_HOST(
|
||||
"hipOccupancyMaxActiveBlocksPerMultiprocessorWithFlags() returned "
|
||||
"error "
|
||||
<< hipGetErrorString(hiprt_result));
|
||||
return hytlass::Status::kErrorInternal;
|
||||
}
|
||||
HYTLASS_TRACE_HOST("sm_occupancy: (" << sm_occupancy
|
||||
<< ") "
|
||||
"smem_size: ("
|
||||
<< GemmFunc::kSharedStorageSize
|
||||
<< ") "
|
||||
"GemmKernel::kThreadCount: ("
|
||||
<< GemmFunc::GemmKernel::kThreadCount
|
||||
<< ")");
|
||||
#endif
|
||||
return hytlass::Status::kSuccess;
|
||||
}
|
||||
|
||||
// Convert HIP data type to hytlass data type
|
||||
template <typename T>
|
||||
struct HytlassDataType {
|
||||
using Type = T;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct HytlassDataType<half> {
|
||||
using Type = hytlass::half_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct HytlassDataType<__hip_bfloat16> {
|
||||
using Type = hytlass::bfloat16_t;
|
||||
};
|
||||
|
||||
// Convert to hytlass layout
|
||||
template <bool Transposed>
|
||||
struct MatrixLayout {
|
||||
using Type = hytlass::layout::RowMajor;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MatrixLayout<true> {
|
||||
using Type = hytlass::layout::ColumnMajor;
|
||||
};
|
||||
|
||||
template <typename ElementT,
|
||||
typename ElementComputeT,
|
||||
template <typename T>
|
||||
class VariadicFunctor,
|
||||
int AlignA = 128 / hytlass::sizeof_bits<ElementT>::value,
|
||||
int AlignB = 128 / hytlass::sizeof_bits<ElementT>::value,
|
||||
int ConfigId = DefaultConfig::kConfigId,
|
||||
int SwizzleFactor = DefaultConfig::kSwizzleFactor,
|
||||
bool Batched = DefaultConfig::kBatched>
|
||||
void MatmulAddVariadic(
|
||||
const GemmEpilogueParams ¶ms,
|
||||
const typename VariadicFunctor<ElementComputeT>::Arguments &variadic_args) {
|
||||
// <- data type of accumulator
|
||||
using ElementAccumulator = typename HytlassDataType<ElementComputeT>::Type;
|
||||
// <- data type of epilogue operations
|
||||
using ElementComputeEpilogue = ElementAccumulator;
|
||||
// <- data type of elements in input matrix A
|
||||
using ElementInputA = typename HytlassDataType<ElementT>::Type;
|
||||
// <- data type of elements in input matrix B
|
||||
using ElementInputB = typename HytlassDataType<ElementT>::Type;
|
||||
// <- data type of elements in output matrix D
|
||||
using ElementOutput = typename HytlassDataType<ElementT>::Type;
|
||||
|
||||
constexpr int AlignC = AlignB;
|
||||
|
||||
// Epilogue operation as LinearCombination:
|
||||
// alpha * accumulator + beta * source
|
||||
using EpilogueOutputOp =
|
||||
cutlass_patch::epilogue::thread::LinearCombinationVariadic<
|
||||
VariadicFunctor,
|
||||
ElementOutput,
|
||||
AlignC,
|
||||
ElementAccumulator,
|
||||
ElementComputeEpilogue,
|
||||
hytlass::epilogue::thread::ScaleType::NoBetaScaling>;
|
||||
|
||||
using GemmFunc = cutlass_patch::gemm::device::GemmUniversalWithVariadic<
|
||||
ElementInputA,
|
||||
hytlass::layout::RowMajor,
|
||||
ElementInputB,
|
||||
hytlass::layout::RowMajor,
|
||||
ElementOutput,
|
||||
hytlass::layout::RowMajor,
|
||||
ElementAccumulator,
|
||||
hytlass::arch::OpClassTensorOp,
|
||||
hytlass::arch::Gfx928,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
TShape,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
WShape,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
IShape,
|
||||
EpilogueOutputOp,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
SwizzleThreadBlock,
|
||||
GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::kNumStages,
|
||||
AlignA,
|
||||
AlignB,
|
||||
typename GemmOperation<ElementT>::Type>;
|
||||
|
||||
CHECK_HYTLASS(SetMaxDynamicSharedMemorySize<GemmFunc>());
|
||||
|
||||
/// Arguments
|
||||
hytlass::gemm::GemmCoord problem_size{params.m, params.n, params.k};
|
||||
|
||||
const ElementInputA *input =
|
||||
reinterpret_cast<const ElementInputA *>(params.input);
|
||||
const ElementInputB *weight =
|
||||
reinterpret_cast<const ElementInputB *>(params.weight);
|
||||
const ElementOutput *bias =
|
||||
reinterpret_cast<const ElementOutput *>(params.bias);
|
||||
ElementOutput *output = reinterpret_cast<ElementOutput *>(params.output);
|
||||
|
||||
ElementComputeEpilogue alpha = static_cast<ElementComputeEpilogue>(1);
|
||||
ElementComputeEpilogue beta = bias ? static_cast<ElementComputeEpilogue>(1)
|
||||
: static_cast<ElementComputeEpilogue>(0);
|
||||
|
||||
typename GemmFunc::Arguments arguments{
|
||||
GetGemmMode(params.batch_count),
|
||||
problem_size, // <- problem size of matrix multiplication
|
||||
params.batch_count, // <- batch_count or k-dimension split factor
|
||||
{alpha, beta, variadic_args}, // <- epilogue params, alpha, beta
|
||||
input, // <- input, ptr_A, A, shape={M, K}
|
||||
weight, // <- input, ptr_B, B, shape={K, N}
|
||||
bias, // <- input, ptr_C, shape={M, N} or {1, N}
|
||||
output, // <- output, ptr_D, Z, shape={M, N}
|
||||
params.shape_args.batch_stride_A,
|
||||
params.shape_args.batch_stride_B,
|
||||
params.shape_args.batch_stride_C,
|
||||
params.shape_args.batch_stride_D,
|
||||
params.shape_args.lda,
|
||||
params.shape_args.ldb,
|
||||
params.shape_args.ldc_bias,
|
||||
params.shape_args.ldd};
|
||||
|
||||
size_t workspace_size = GemmFunc::get_workspace_size(arguments);
|
||||
void *workspace = workspace_size > 0 ? GetWorkspace(workspace_size) : nullptr;
|
||||
|
||||
GemmFunc device_gemm;
|
||||
|
||||
hipStream_t *stream_ptr = reinterpret_cast<hipStream_t *>(params.stream_ptr);
|
||||
|
||||
CHECK_HYTLASS(device_gemm.can_implement(arguments));
|
||||
CHECK_HYTLASS(device_gemm.initialize(arguments, workspace, *stream_ptr));
|
||||
|
||||
// Run the GEMM
|
||||
CHECK_HYTLASS(device_gemm(*stream_ptr));
|
||||
#if AP_ENABLE_DEBUG
|
||||
CHECK_HIP(hipStreamSynchronize(*stream_ptr));
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace ap
|
||||
Reference in New Issue
Block a user