chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,496 @@
|
||||
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// auto-generated by generate_configs.py
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/gemm_coord.h"
|
||||
|
||||
namespace ap {
|
||||
|
||||
constexpr int kNumConfigsHalf = 23;
|
||||
constexpr int kNumConfigsFloat = 13;
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct SwizzleWrapper {
|
||||
using Type =
|
||||
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle<SwizzleFactor>;
|
||||
};
|
||||
|
||||
// template <int SwizzleFactor>
|
||||
// struct SwizzleWrapper<SwizzleFactor, true> {
|
||||
// using Type =
|
||||
// cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle;
|
||||
// };
|
||||
|
||||
#define AP_AUTOTUNE(func, stream_ptr, count, ...) \
|
||||
{ \
|
||||
using FuncType = decltype(func<0>); \
|
||||
static int selected_config_id = -1; \
|
||||
static std::vector<std::function<FuncType>> matmul_functions = \
|
||||
[]<std::size_t... Is>(std::index_sequence<Is...>) { \
|
||||
return std::vector<std::function<FuncType>>{func<Is>...}; \
|
||||
} \
|
||||
(std::make_index_sequence<count>()); \
|
||||
\
|
||||
if (selected_config_id == -1) { \
|
||||
selected_config_id = \
|
||||
ap::ProfileBestConfig(matmul_functions, stream_ptr, ##__VA_ARGS__); \
|
||||
} \
|
||||
\
|
||||
matmul_functions[selected_config_id](__VA_ARGS__); \
|
||||
}
|
||||
|
||||
#define AP_AUTOTUNE_half(func, stream_ptr, ...) \
|
||||
AP_AUTOTUNE(func, stream_ptr, ap::kNumConfigsHalf, __VA_ARGS__)
|
||||
#define AP_AUTOTUNE_float(func, stream_ptr, ...) \
|
||||
AP_AUTOTUNE(func, stream_ptr, ap::kNumConfigsFloat, __VA_ARGS__)
|
||||
#define AP_AUTOTUNE_bfloat16(func, stream_ptr, ...) \
|
||||
AP_AUTOTUNE_half(func, stream_ptr, __VA_ARGS__)
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched, int Id = 0>
|
||||
struct GemmTuningConfigs {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 2;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = Id;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 1> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 1;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 2> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 2;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 3> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 64, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 32, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 3;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 4> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 4;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 5> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 5;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 6> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 6;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 7> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 64, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 7;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 8> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 8;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 9> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 9;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 10> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 32, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 32, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 10;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 11> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 11;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 12> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 12;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 13> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 64, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 13;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 14> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 14;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 15> {
|
||||
using TShape = cutlass::gemm::GemmShape<32, 64, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<16, 32, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 15;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 16> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 32, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 16;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 17> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 17;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 18> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 64>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 64, 64>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 5;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 18;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 19> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 6;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 19;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 20> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 32, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 6;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 20;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 21> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 32, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 32, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 7;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 21;
|
||||
};
|
||||
|
||||
template <typename ElementT, int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<ElementT, SwizzleFactor, Batched, 22> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 32, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 16>;
|
||||
static constexpr int kNumStages = 10;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 22;
|
||||
};
|
||||
|
||||
// Specialization for float
|
||||
template <int SwizzleFactor, bool Batched, int Id>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, Id> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 64, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 32, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = Id;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 1> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 32, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 1;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 2> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 2;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 3> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 256, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 3;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 4> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 256, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 4;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 5> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 32, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 5;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 6> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 6;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 7> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 7;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 8> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 64, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 8;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 9> {
|
||||
using TShape = cutlass::gemm::GemmShape<256, 64, 32>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 32, 32>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 3;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 9;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 10> {
|
||||
using TShape = cutlass::gemm::GemmShape<64, 128, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 10;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 11> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 64, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<64, 32, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 11;
|
||||
};
|
||||
|
||||
template <int SwizzleFactor, bool Batched>
|
||||
struct GemmTuningConfigs<float, SwizzleFactor, Batched, 12> {
|
||||
using TShape = cutlass::gemm::GemmShape<128, 128, 16>;
|
||||
using WShape = cutlass::gemm::GemmShape<32, 64, 16>;
|
||||
using IShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
static constexpr int kNumStages = 4;
|
||||
|
||||
using SwizzleThreadBlock =
|
||||
typename SwizzleWrapper<SwizzleFactor, Batched>::Type;
|
||||
static constexpr int kId = 12;
|
||||
};
|
||||
|
||||
} // namespace ap
|
||||
@@ -0,0 +1,273 @@
|
||||
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/gemm_coord.h"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination_bias_elementwise.h"
|
||||
#include "cutlass/util/device_memory.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_with_broadcast.h"
|
||||
|
||||
#include "cutlass_patch/batched_matrix_coord.h"
|
||||
#include "cutlass_patch/cuda/default_config_id.h"
|
||||
#include "cutlass_patch/epilogue/thread/linear_combination_unary.h"
|
||||
#include "cutlass_patch/epilogue/thread/linear_combination_variadic.h"
|
||||
#include "cutlass_patch/gemm/device/gemm_universal_with_variadic.h"
|
||||
|
||||
#include "params.h" // NOLINT
|
||||
|
||||
#define CHECK_CUTLASS(status) \
|
||||
{ \
|
||||
cutlass::Status error = status; \
|
||||
if (error != cutlass::Status::kSuccess) { \
|
||||
std::cerr << "Got cutlass error: " << cutlassGetStatusString(error) \
|
||||
<< " at: " << __LINE__ << std::endl; \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
}
|
||||
|
||||
namespace ap {
|
||||
using bfloat16 = nv_bfloat16;
|
||||
|
||||
template <typename T, int N>
|
||||
using Array = cutlass::Array<T, N>;
|
||||
|
||||
using MatrixCoord = cutlass_patch::BatchedMatrixCoord;
|
||||
|
||||
// Convert CUDA data type to cutlass data type
|
||||
template <typename T>
|
||||
struct CutlassDataType {
|
||||
using Type = T;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct CutlassDataType<half> {
|
||||
using Type = cutlass::half_t;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct CutlassDataType<__nv_bfloat16> {
|
||||
using Type = cutlass::bfloat16_t;
|
||||
};
|
||||
|
||||
// Convert to cutlass layout
|
||||
template <bool Transposed>
|
||||
struct MatrixLayout {
|
||||
using Type = cutlass::layout::RowMajor;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct MatrixLayout<true> {
|
||||
using Type = cutlass::layout::ColumnMajor;
|
||||
};
|
||||
|
||||
// Operation performed by GEMM
|
||||
template <typename ElementT>
|
||||
struct GemmOperation {
|
||||
using Type = cutlass::arch::OpMultiplyAdd;
|
||||
};
|
||||
|
||||
template <>
|
||||
struct GemmOperation<float> {
|
||||
using Type = cutlass::arch::OpMultiplyAddFastF32;
|
||||
};
|
||||
|
||||
static cutlass::gemm::GemmUniversalMode GetGemmMode(int batch_count) {
|
||||
return batch_count > 1 ? cutlass::gemm::GemmUniversalMode::kBatched
|
||||
: cutlass::gemm::GemmUniversalMode::kGemm;
|
||||
}
|
||||
|
||||
static void *GetWorkspace(size_t workspace_size) {
|
||||
static cutlass::device_memory::allocation<uint8_t> workspace;
|
||||
if (workspace.size() < workspace_size) {
|
||||
workspace.reset(workspace_size);
|
||||
}
|
||||
return workspace.get();
|
||||
}
|
||||
|
||||
template <typename GemmFunc>
|
||||
cutlass::Status SetMaxDynamicSharedMemorySize() {
|
||||
cudaError_t cudart_result;
|
||||
|
||||
// If requires more than 48KB: configure for extended, dynamic shared memory
|
||||
if constexpr (GemmFunc::kSharedStorageSize >= (48 << 10)) {
|
||||
cudart_result =
|
||||
cudaFuncSetAttribute(cutlass::Kernel2<typename GemmFunc::GemmKernel>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
GemmFunc::kSharedStorageSize);
|
||||
if (cudart_result != cudaSuccess) {
|
||||
CUTLASS_TRACE_HOST("cudaFuncSetAttribute() returned error "
|
||||
<< cudaGetErrorString(cudart_result));
|
||||
return cutlass::Status::kErrorInternal;
|
||||
}
|
||||
}
|
||||
|
||||
#if AP_ENABLE_DEBUG
|
||||
// Update SM occupancy member
|
||||
int sm_occupancy = -1;
|
||||
cudart_result = cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(
|
||||
&sm_occupancy,
|
||||
cutlass::Kernel2<typename GemmFunc::GemmKernel>,
|
||||
GemmFunc::GemmKernel::kThreadCount,
|
||||
GemmFunc::kSharedStorageSize,
|
||||
cudaOccupancyDisableCachingOverride);
|
||||
if (cudart_result != cudaSuccess) {
|
||||
CUTLASS_TRACE_HOST(
|
||||
"cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags() returned "
|
||||
"error "
|
||||
<< cudaGetErrorString(cudart_result));
|
||||
return cutlass::Status::kErrorInternal;
|
||||
}
|
||||
CUTLASS_TRACE_HOST("sm_occupancy: (" << sm_occupancy
|
||||
<< ") "
|
||||
"smem_size: ("
|
||||
<< GemmFunc::kSharedStorageSize
|
||||
<< ") "
|
||||
"GemmKernel::kThreadCount: ("
|
||||
<< GemmFunc::GemmKernel::kThreadCount
|
||||
<< ")");
|
||||
#endif
|
||||
return cutlass::Status::kSuccess;
|
||||
}
|
||||
|
||||
template <typename ElementT,
|
||||
typename ElementComputeT,
|
||||
template <typename T>
|
||||
class VariadicFunctor,
|
||||
int AlignA = 128 / cutlass::sizeof_bits<ElementT>::value,
|
||||
int AlignB = 128 / cutlass::sizeof_bits<ElementT>::value,
|
||||
int ConfigId = DefaultConfig::kConfigId,
|
||||
int SwizzleFactor = DefaultConfig::kSwizzleFactor,
|
||||
bool Batched = DefaultConfig::kBatched>
|
||||
void MatmulAddVariadic(
|
||||
const GemmEpilogueParams ¶ms,
|
||||
const typename VariadicFunctor<ElementComputeT>::Arguments &variadic_args) {
|
||||
using ElementAccumulator =
|
||||
typename CutlassDataType<ElementComputeT>::Type; // <- data type of
|
||||
// accumulator
|
||||
using ElementComputeEpilogue =
|
||||
ElementAccumulator; // <- data type of epilogue operations
|
||||
using ElementInputA =
|
||||
typename CutlassDataType<ElementT>::Type; // <- data type of elements in
|
||||
// input matrix A
|
||||
using ElementInputB =
|
||||
typename CutlassDataType<ElementT>::Type; // <- data type of elements in
|
||||
// input matrix B
|
||||
using ElementOutput =
|
||||
typename CutlassDataType<ElementT>::Type; // <- data type of elements in
|
||||
// output matrix D
|
||||
|
||||
constexpr int AlignC = AlignB;
|
||||
|
||||
// Epilogue operation as LinearCombination:
|
||||
// alpha * accumulator + beta * source
|
||||
using EpilogueOutputOp =
|
||||
cutlass_patch::epilogue::thread::LinearCombinationVariadic<
|
||||
VariadicFunctor,
|
||||
ElementOutput,
|
||||
AlignC,
|
||||
ElementAccumulator,
|
||||
ElementComputeEpilogue,
|
||||
cutlass::epilogue::thread::ScaleType::NoBetaScaling>; // <- alpha x
|
||||
// AB + bias
|
||||
|
||||
using GemmFunc = cutlass_patch::gemm::device::GemmUniversalWithVariadic<
|
||||
ElementInputA,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementInputB,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementOutput,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementAccumulator,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm80,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
TShape,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
WShape,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
IShape,
|
||||
EpilogueOutputOp,
|
||||
typename GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::
|
||||
SwizzleThreadBlock,
|
||||
GemmTuningConfigs<ElementT, SwizzleFactor, Batched, ConfigId>::kNumStages,
|
||||
AlignA,
|
||||
AlignB,
|
||||
typename GemmOperation<ElementT>::Type>;
|
||||
|
||||
CHECK_CUTLASS(SetMaxDynamicSharedMemorySize<GemmFunc>());
|
||||
|
||||
/// Arguments
|
||||
cutlass::gemm::GemmCoord problem_size{params.m, params.n, params.k};
|
||||
|
||||
const ElementInputA *input =
|
||||
reinterpret_cast<const ElementInputA *>(params.input);
|
||||
const ElementInputB *weight =
|
||||
reinterpret_cast<const ElementInputB *>(params.weight);
|
||||
const ElementOutput *bias =
|
||||
reinterpret_cast<const ElementOutput *>(params.bias);
|
||||
ElementOutput *output = reinterpret_cast<ElementOutput *>(params.output);
|
||||
|
||||
ElementComputeEpilogue alpha = static_cast<ElementComputeEpilogue>(1);
|
||||
ElementComputeEpilogue beta = bias ? static_cast<ElementComputeEpilogue>(1)
|
||||
: static_cast<ElementComputeEpilogue>(0);
|
||||
|
||||
typename GemmFunc::Arguments arguments{
|
||||
GetGemmMode(params.batch_count),
|
||||
problem_size, // <- problem size of matrix multiplication
|
||||
params.batch_count, // <- batch_count or k-dimension split factor
|
||||
{alpha, beta, variadic_args}, // <- epilogue params, alpha, beta
|
||||
input, // <- input, ptr_A, A, shape={M, K}
|
||||
weight, // <- input, ptr_B, B, shape={K, N}
|
||||
bias, // <- input, ptr_C, shape={M, N} or {1, N}
|
||||
output, // <- output, ptr_D, Z, shape={M, N}
|
||||
params.shape_args.batch_stride_A,
|
||||
params.shape_args.batch_stride_B,
|
||||
params.shape_args.batch_stride_C,
|
||||
params.shape_args.batch_stride_D,
|
||||
params.shape_args.lda,
|
||||
params.shape_args.ldb,
|
||||
params.shape_args.ldc_bias,
|
||||
params.shape_args.ldd};
|
||||
|
||||
size_t workspace_size = GemmFunc::get_workspace_size(arguments);
|
||||
void *workspace = workspace_size > 0 ? GetWorkspace(workspace_size) : nullptr;
|
||||
|
||||
GemmFunc device_gemm;
|
||||
|
||||
cudaStream_t *stream_ptr =
|
||||
reinterpret_cast<cudaStream_t *>(params.stream_ptr);
|
||||
|
||||
CHECK_CUTLASS(device_gemm.can_implement(arguments));
|
||||
CHECK_CUTLASS(device_gemm.initialize(arguments, workspace, *stream_ptr));
|
||||
|
||||
//
|
||||
// Run the GEMM
|
||||
//
|
||||
CHECK_CUTLASS(device_gemm(*stream_ptr));
|
||||
#if AP_ENABLE_DEBUG
|
||||
CHECK_CUDA(cudaStreamSynchronize(*stream_ptr));
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace ap
|
||||
@@ -0,0 +1,27 @@
|
||||
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "all_tuning_configs.h" // NOLINT
|
||||
|
||||
namespace ap {
|
||||
|
||||
struct DefaultConfig {
|
||||
static constexpr int kConfigId = 0;
|
||||
static constexpr int kSwizzleFactor = 1;
|
||||
static constexpr bool kBatched = false;
|
||||
};
|
||||
|
||||
} // namespace ap
|
||||
Reference in New Issue
Block a user