chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,58 @@
|
||||
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
|
||||
namespace c10 {
|
||||
|
||||
enum class DeviceType : int8_t {
|
||||
CPU = 0,
|
||||
CUDA = 1,
|
||||
XPU = 12,
|
||||
IPU = 18,
|
||||
CUSTOM = 20,
|
||||
PrivateUse1 = CUSTOM,
|
||||
};
|
||||
|
||||
constexpr DeviceType kCUDA = DeviceType::CUDA;
|
||||
constexpr DeviceType kCPU = DeviceType::CPU;
|
||||
constexpr DeviceType kCUSTOM = DeviceType::CUSTOM;
|
||||
constexpr DeviceType kXPU = DeviceType::XPU;
|
||||
constexpr DeviceType kIPU = DeviceType::IPU;
|
||||
constexpr DeviceType kPrivateUse1 = DeviceType::PrivateUse1;
|
||||
|
||||
} // namespace c10
|
||||
|
||||
namespace std {
|
||||
template <>
|
||||
struct hash<c10::DeviceType> {
|
||||
std::size_t operator()(c10::DeviceType k) const noexcept {
|
||||
return std::hash<int>()(static_cast<int>(k));
|
||||
}
|
||||
};
|
||||
} // namespace std
|
||||
|
||||
namespace at {
|
||||
using c10::DeviceType;
|
||||
using c10::kCPU;
|
||||
using c10::kCUDA;
|
||||
using c10::kCUSTOM;
|
||||
using c10::kIPU;
|
||||
using c10::kPrivateUse1;
|
||||
using c10::kXPU;
|
||||
} // namespace at
|
||||
@@ -0,0 +1,337 @@
|
||||
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The file has been adapted from pytorch project
|
||||
// Licensed under BSD-style license -
|
||||
// https://github.com/pytorch/pytorch/blob/main/LICENSE
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <c10/util/BFloat16.h>
|
||||
#include <c10/util/Float4_e2m1fn_x2.h>
|
||||
#include <c10/util/Float8_e4m3fn.h>
|
||||
#include <c10/util/Float8_e4m3fnuz.h>
|
||||
#include <c10/util/Float8_e5m2.h>
|
||||
#include <c10/util/Float8_e5m2fnuz.h>
|
||||
#include <c10/util/Float8_e8m0fnu.h>
|
||||
#include <c10/util/Half.h>
|
||||
#include <c10/util/bits.h>
|
||||
#include <c10/util/complex.h>
|
||||
#include <c10/util/qint32.h>
|
||||
#include <c10/util/qint8.h>
|
||||
#include <c10/util/quint2x4.h>
|
||||
#include <c10/util/quint4x2.h>
|
||||
#include <c10/util/quint8.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <ostream>
|
||||
#include <type_traits>
|
||||
|
||||
#include "paddle/common/macros.h"
|
||||
|
||||
namespace c10 {
|
||||
|
||||
// dummy struct for uint1 to uint7, actual functionality
|
||||
// of these dtypes will be implemented in python with Tensor subclass
|
||||
template <unsigned int N>
|
||||
struct dummy_uint1_7_t {};
|
||||
|
||||
// dummy struct for int1 to int7, actual functionality
|
||||
// of these dtypes will be implemented in python with Tensor subclass
|
||||
template <unsigned int N>
|
||||
struct dummy_int1_7_t {};
|
||||
|
||||
#define AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(_) \
|
||||
_(uint8_t, UINT8, Byte) /* 0 */ \
|
||||
_(int8_t, INT8, Char) /* 1 */ \
|
||||
_(int16_t, INT16, Short) /* 2 */ \
|
||||
_(int, INT32, Int) /* 3 */ \
|
||||
_(int64_t, INT64, Long) /* 4 */ \
|
||||
_(at::Half, FLOAT16, Half) /* 5 */ \
|
||||
_(float, FLOAT32, Float) /* 6 */ \
|
||||
_(double, FLOAT64, Double) /* 7 */ \
|
||||
_(c10::complex<at::Half>, ComplexHalf, ComplexHalf) /* 8 */ \
|
||||
_(c10::complex<float>, COMPLEX64, ComplexFloat) /* 9 */ \
|
||||
_(c10::complex<double>, COMPLEX128, ComplexDouble) /* 10 */ \
|
||||
_(bool, BOOL, Bool) /* 11 */ \
|
||||
_(c10::qint8, QInt8, QInt8) /* 12 */ \
|
||||
_(c10::quint8, QUInt8, QUInt8) /* 13 */ \
|
||||
_(c10::qint32, QInt32, QInt32) /* 14 */ \
|
||||
_(at::BFloat16, BFLOAT16, BFloat16) /* 15 */ \
|
||||
_(c10::quint4x2, QUInt4x2, QUInt4x2) /* 16 */ \
|
||||
_(c10::quint2x4, QUInt2x4, QUInt2x4) /* 17 */ \
|
||||
_(c10::bits1x8, Bits1x8, Bits1x8) /* 18 */ \
|
||||
_(c10::bits2x4, Bits2x4, Bits2x4) /* 19 */ \
|
||||
_(c10::bits4x2, Bits4x2, Bits4x2) /* 20 */ \
|
||||
_(c10::bits8, Bits8, Bits8) /* 21 */ \
|
||||
_(c10::bits16, Bits16, Bits16) /* 22 */ \
|
||||
_(c10::Float8_e5m2, FLOAT8_E5M2, Float8_e5m2) /* 23 */ \
|
||||
_(c10::Float8_e4m3fn, FLOAT8_E4M3FN, Float8_e4m3fn) /* 24 */ \
|
||||
_(c10::Float8_e5m2fnuz, Float8_e5m2fnuz, Float8_e5m2fnuz) /* 25 */ \
|
||||
_(c10::Float8_e4m3fnuz, Float8_e4m3fnuz, Float8_e4m3fnuz) /* 26 */ \
|
||||
_(uint16_t, UINT16, UInt16) /* 27 */ \
|
||||
_(uint32_t, UINT32, UInt32) /* 28 */ \
|
||||
_(uint64_t, UINT64, UInt64) /* 29 */ \
|
||||
_(c10::dummy_uint1_7_t<1>, UInt1, UInt1) /* 30 */ \
|
||||
_(c10::dummy_uint1_7_t<2>, UInt2, UInt2) /* 31 */ \
|
||||
_(c10::dummy_uint1_7_t<3>, UInt3, UInt3) /* 32 */ \
|
||||
_(c10::dummy_uint1_7_t<4>, UInt4, UInt4) /* 33 */ \
|
||||
_(c10::dummy_uint1_7_t<5>, UInt5, UInt5) /* 34 */ \
|
||||
_(c10::dummy_uint1_7_t<6>, UInt6, UInt6) /* 35 */ \
|
||||
_(c10::dummy_uint1_7_t<7>, UInt7, UInt7) /* 36 */ \
|
||||
_(c10::dummy_int1_7_t<1>, Int1, Int1) /* 37 */ \
|
||||
_(c10::dummy_int1_7_t<2>, Int2, Int2) /* 38 */ \
|
||||
_(c10::dummy_int1_7_t<3>, Int3, Int3) /* 39 */ \
|
||||
_(c10::dummy_int1_7_t<4>, Int4, Int4) /* 40 */ \
|
||||
_(c10::dummy_int1_7_t<5>, Int5, Int5) /* 41 */ \
|
||||
_(c10::dummy_int1_7_t<6>, Int6, Int6) /* 42 */ \
|
||||
_(c10::dummy_int1_7_t<7>, Int7, Int7) /* 43 */ \
|
||||
_(c10::Float8_e8m0fnu, Float8_e8m0fnu, Float8_e8m0fnu) /* 44 */ \
|
||||
_(c10::Float4_e2m1fn_x2, Float4_e2m1fn_x2, Float4_e2m1fn_x2) /* 45 */
|
||||
|
||||
#define AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_EXCEPT_COMPLEX_HALF_F8NZ(_) \
|
||||
_(uint8_t, Byte) \
|
||||
_(int8_t, Char) \
|
||||
_(int16_t, Short) \
|
||||
_(int, Int) \
|
||||
_(int64_t, Long) \
|
||||
_(at::Half, Half) \
|
||||
_(float, Float) \
|
||||
_(double, Double) \
|
||||
_(c10::complex<float>, ComplexFloat) \
|
||||
_(c10::complex<double>, ComplexDouble) \
|
||||
_(bool, Bool) \
|
||||
_(at::BFloat16, BFloat16) \
|
||||
_(c10::Float8_e5m2, Float8_e5m2) \
|
||||
_(c10::Float8_e4m3fn, Float8_e4m3fn)
|
||||
|
||||
#define AT_FORALL_SCALAR_TYPES_WITH_COMPLEX(_) \
|
||||
_(uint8_t, Byte) \
|
||||
_(int8_t, Char) \
|
||||
_(int16_t, Short) \
|
||||
_(int, Int) \
|
||||
_(int64_t, Long) \
|
||||
_(float, Float) \
|
||||
_(double, Double) \
|
||||
_(c10::complex<float>, ComplexFloat) \
|
||||
_(c10::complex<double>, ComplexDouble) \
|
||||
_(bool, Bool) \
|
||||
_(at::BFloat16, BFloat16) \
|
||||
_(c10::Float8_e5m2, Float8_e5m2) \
|
||||
_(c10::Float8_e4m3fn, Float8_e4m3fn)
|
||||
|
||||
#define AT_FORALL_QINT_TYPES(_) \
|
||||
_(c10::qint8, QInt8) \
|
||||
_(c10::quint8, QUInt8) \
|
||||
_(c10::qint32, QInt32) \
|
||||
_(c10::quint4x2, QUInt4x2) \
|
||||
_(c10::quint2x4, QUInt2x4)
|
||||
|
||||
#define FOREACH_PADDLE_AND_TORCH_DTYPES(_) \
|
||||
_(uint8_t, UINT8, Byte) \
|
||||
_(int8_t, INT8, Char) \
|
||||
_(int16_t, INT16, Short) \
|
||||
_(int32_t, INT32, Int) \
|
||||
_(int64_t, INT64, Long) \
|
||||
_(at::Half, FLOAT16, Half) \
|
||||
_(float, FLOAT32, Float) \
|
||||
_(double, FLOAT64, Double) \
|
||||
_(c10::complex<float>, COMPLEX64, ComplexFloat) \
|
||||
_(c10::complex<double>, COMPLEX128, ComplexDouble) \
|
||||
_(bool, BOOL, Bool) \
|
||||
_(at::BFloat16, BFLOAT16, BFloat16) \
|
||||
_(c10::Float8_e5m2, FLOAT8_E5M2, Float8_e5m2) \
|
||||
_(c10::Float8_e4m3fn, FLOAT8_E4M3FN, Float8_e4m3fn) \
|
||||
_(uint16_t, UINT16, UInt16) \
|
||||
_(uint32_t, UINT32, UInt32)
|
||||
|
||||
enum class PADDLE_API ScalarType : int8_t {
|
||||
Byte = 0,
|
||||
Char = 1,
|
||||
Short = 2,
|
||||
Int = 3,
|
||||
Long = 4,
|
||||
Half = 5,
|
||||
Float = 6,
|
||||
Double = 7,
|
||||
ComplexHalf = 8,
|
||||
ComplexFloat = 9,
|
||||
ComplexDouble = 10,
|
||||
Bool = 11,
|
||||
QInt8 = 12,
|
||||
QUInt8 = 13,
|
||||
QInt32 = 14,
|
||||
BFloat16 = 15,
|
||||
QUInt4x2 = 16,
|
||||
QUInt2x4 = 17,
|
||||
Bits1x8 = 18,
|
||||
Bits2x4 = 19,
|
||||
Bits4x2 = 20,
|
||||
Bits8 = 21,
|
||||
Bits16 = 22,
|
||||
Float8_e5m2 = 23,
|
||||
Float8_e4m3fn = 24,
|
||||
Float8_e5m2fnuz = 25,
|
||||
Float8_e4m3fnuz = 26,
|
||||
UInt16 = 27,
|
||||
UInt32 = 28,
|
||||
UInt64 = 29,
|
||||
UInt1 = 30,
|
||||
UInt2 = 31,
|
||||
UInt3 = 32,
|
||||
UInt4 = 33,
|
||||
UInt5 = 34,
|
||||
UInt6 = 35,
|
||||
UInt7 = 36,
|
||||
Int1 = 37,
|
||||
Int2 = 38,
|
||||
Int3 = 39,
|
||||
Int4 = 40,
|
||||
Int5 = 41,
|
||||
Int6 = 42,
|
||||
Int7 = 43,
|
||||
Float8_e8m0fnu = 44,
|
||||
Float4_e2m1fn_x2 = 45,
|
||||
Undefined = 46,
|
||||
NumOptions = 47
|
||||
};
|
||||
|
||||
constexpr uint16_t NumScalarTypes =
|
||||
static_cast<uint16_t>(ScalarType::NumOptions);
|
||||
|
||||
namespace impl {
|
||||
|
||||
template <c10::ScalarType N>
|
||||
struct ScalarTypeToCPPType;
|
||||
|
||||
#define SPECIALIZE_ScalarTypeToCPPType(cpp_type, _2, scalar_type) \
|
||||
template <> \
|
||||
struct ScalarTypeToCPPType<c10::ScalarType::scalar_type> { \
|
||||
using type = cpp_type; \
|
||||
\
|
||||
static type t; \
|
||||
};
|
||||
|
||||
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(SPECIALIZE_ScalarTypeToCPPType)
|
||||
|
||||
#undef SPECIALIZE_ScalarTypeToCPPType
|
||||
|
||||
template <c10::ScalarType N>
|
||||
using ScalarTypeToCPPTypeT = typename ScalarTypeToCPPType<N>::type;
|
||||
|
||||
} // namespace impl
|
||||
|
||||
template <typename T>
|
||||
struct CppTypeToScalarType;
|
||||
|
||||
#define SPECIALIZE_CppTypeToScalarType(cpp_type, _2, scalar_type) \
|
||||
template <> \
|
||||
struct CppTypeToScalarType<cpp_type> \
|
||||
: std::integral_constant<c10::ScalarType, \
|
||||
c10::ScalarType::scalar_type> {};
|
||||
|
||||
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(SPECIALIZE_CppTypeToScalarType)
|
||||
|
||||
#undef SPECIALIZE_CppTypeToScalarType
|
||||
|
||||
#define AT_FORALL_SCALAR_TYPES_AND(SCALARTYPE, _) \
|
||||
_(uint8_t, Byte) \
|
||||
_(int8_t, Char) \
|
||||
_(int16_t, Short) \
|
||||
_(int, Int) \
|
||||
_(int64_t, Long) \
|
||||
_(float, Float) \
|
||||
_(double, Double) \
|
||||
_(decltype(::c10::impl::ScalarTypeToCPPType< \
|
||||
::c10::ScalarType::SCALARTYPE>::t), \
|
||||
SCALARTYPE)
|
||||
|
||||
#define AT_FORALL_SCALAR_TYPES_AND2(SCALARTYPE1, SCALARTYPE2, _) \
|
||||
_(uint8_t, Byte) \
|
||||
_(int8_t, Char) \
|
||||
_(int16_t, Short) \
|
||||
_(int, Int) \
|
||||
_(int64_t, Long) \
|
||||
_(float, Float) \
|
||||
_(double, Double) \
|
||||
_(decltype(::c10::impl::ScalarTypeToCPPType< \
|
||||
::c10::ScalarType::SCALARTYPE1>::t), \
|
||||
SCALARTYPE1) \
|
||||
_(decltype(::c10::impl::ScalarTypeToCPPType< \
|
||||
::c10::ScalarType::SCALARTYPE2>::t), \
|
||||
SCALARTYPE2)
|
||||
|
||||
#define AT_FORALL_SCALAR_TYPES_AND3(SCALARTYPE1, SCALARTYPE2, SCALARTYPE3, _) \
|
||||
_(uint8_t, Byte) \
|
||||
_(int8_t, Char) \
|
||||
_(int16_t, Short) \
|
||||
_(int, Int) \
|
||||
_(int64_t, Long) \
|
||||
_(float, Float) \
|
||||
_(double, Double) \
|
||||
_(decltype(::c10::impl::ScalarTypeToCPPType< \
|
||||
::c10::ScalarType::SCALARTYPE1>::t), \
|
||||
SCALARTYPE1) \
|
||||
_(decltype(::c10::impl::ScalarTypeToCPPType< \
|
||||
::c10::ScalarType::SCALARTYPE2>::t), \
|
||||
SCALARTYPE2) \
|
||||
_(decltype(::c10::impl::ScalarTypeToCPPType< \
|
||||
::c10::ScalarType::SCALARTYPE3>::t), \
|
||||
SCALARTYPE3)
|
||||
|
||||
#define AT_FORALL_COMPLEX_TYPES(_) \
|
||||
_(c10::complex<float>, ComplexFloat) \
|
||||
_(c10::complex<double>, ComplexDouble)
|
||||
|
||||
inline const char* toString(ScalarType t) {
|
||||
#define DEFINE_CASE(_1, _2, name) \
|
||||
case ScalarType::name: \
|
||||
return #name;
|
||||
|
||||
switch (t) {
|
||||
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_CASE)
|
||||
case ScalarType::Undefined:
|
||||
return "Undefined";
|
||||
default:
|
||||
return "UNKNOWN_SCALAR";
|
||||
}
|
||||
#undef DEFINE_CASE
|
||||
}
|
||||
|
||||
inline std::ostream& operator<<(std::ostream& stream, ScalarType scalar_type) {
|
||||
return stream << toString(scalar_type);
|
||||
}
|
||||
|
||||
inline bool isQIntType(ScalarType t) {
|
||||
return t == ScalarType::QInt8 || t == ScalarType::QUInt8 ||
|
||||
t == ScalarType::QInt32 || t == ScalarType::QUInt4x2 ||
|
||||
t == ScalarType::QUInt2x4;
|
||||
}
|
||||
|
||||
inline ScalarType toUnderlying(ScalarType t) {
|
||||
switch (t) {
|
||||
case ScalarType::QUInt8:
|
||||
case ScalarType::QUInt4x2:
|
||||
case ScalarType::QUInt2x4:
|
||||
return ScalarType::Byte;
|
||||
case ScalarType::QInt8:
|
||||
return ScalarType::Char;
|
||||
case ScalarType::QInt32:
|
||||
return ScalarType::Int;
|
||||
default:
|
||||
return t;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace c10
|
||||
@@ -0,0 +1,399 @@
|
||||
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// The file has been adapted from pytorch project
|
||||
// Licensed under BSD-style license -
|
||||
// https://github.com/pytorch/pytorch/blob/main/LICENSE
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <c10/macros/Macros.h>
|
||||
#include <c10/util/ArrayRef.h>
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
|
||||
namespace torch::headeronly {
|
||||
|
||||
// The PtrTraits argument to the TensorAccessor/GenericPackedTensorAccessor
|
||||
// is used to enable the __restrict__ keyword/modifier for the data
|
||||
// passed to cuda.
|
||||
template <typename T>
|
||||
struct DefaultPtrTraits {
|
||||
typedef T* PtrType;
|
||||
};
|
||||
|
||||
#if defined(__CUDACC__) || defined(__HIPCC__)
|
||||
template <typename T>
|
||||
struct RestrictPtrTraits {
|
||||
typedef T* __restrict__ PtrType;
|
||||
};
|
||||
#endif
|
||||
|
||||
namespace detail {
|
||||
|
||||
// TensorAccessorBase and TensorAccessor are used for both CPU and CUDA tensors.
|
||||
// For CUDA tensors it is used in device code (only). This means that we
|
||||
// restrict ourselves to functions and types available there (e.g. IntArrayRef
|
||||
// isn't).
|
||||
|
||||
// The PtrTraits argument is only relevant to cuda to support `__restrict__`
|
||||
// pointers.
|
||||
template <class ArrayRefCls,
|
||||
typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
class TensorAccessorBase {
|
||||
public:
|
||||
typedef typename PtrTraits<T>::PtrType PtrType;
|
||||
|
||||
C10_HOST_DEVICE TensorAccessorBase(PtrType data,
|
||||
const index_t* sizes,
|
||||
const index_t* strides)
|
||||
: data_(data), sizes_(sizes), strides_(strides) {}
|
||||
C10_HOST ArrayRefCls sizes() const { return ArrayRefCls(sizes_, N); }
|
||||
C10_HOST ArrayRefCls strides() const { return ArrayRefCls(strides_, N); }
|
||||
C10_HOST_DEVICE index_t stride(index_t i) const { return strides_[i]; }
|
||||
C10_HOST_DEVICE index_t size(index_t i) const { return sizes_[i]; }
|
||||
C10_HOST_DEVICE PtrType data() { return data_; }
|
||||
C10_HOST_DEVICE const PtrType data() const { return data_; }
|
||||
|
||||
protected:
|
||||
PtrType data_;
|
||||
const index_t* sizes_;
|
||||
const index_t* strides_;
|
||||
};
|
||||
|
||||
// The `TensorAccessor` is typically instantiated for CPU `Tensor`s using
|
||||
// `Tensor.accessor<T, N>()`.
|
||||
// For CUDA `Tensor`s, `GenericPackedTensorAccessor` is used on the host and
|
||||
// only indexing on the device uses `TensorAccessor`s.
|
||||
template <class ArrayRefCls,
|
||||
typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
class TensorAccessor
|
||||
: public TensorAccessorBase<ArrayRefCls, T, N, PtrTraits, index_t> {
|
||||
public:
|
||||
typedef typename PtrTraits<T>::PtrType PtrType;
|
||||
|
||||
C10_HOST_DEVICE TensorAccessor(PtrType data,
|
||||
const index_t* sizes,
|
||||
const index_t* strides)
|
||||
: TensorAccessorBase<ArrayRefCls, T, N, PtrTraits, index_t>(
|
||||
data, sizes, strides) {}
|
||||
|
||||
C10_HOST_DEVICE TensorAccessor<ArrayRefCls, T, N - 1, PtrTraits, index_t>
|
||||
operator[](index_t i) {
|
||||
return TensorAccessor<ArrayRefCls, T, N - 1, PtrTraits, index_t>(
|
||||
this->data_ + this->strides_[0] * i,
|
||||
this->sizes_ + 1,
|
||||
this->strides_ + 1);
|
||||
}
|
||||
|
||||
C10_HOST_DEVICE const
|
||||
TensorAccessor<ArrayRefCls, T, N - 1, PtrTraits, index_t>
|
||||
operator[](index_t i) const {
|
||||
return TensorAccessor<ArrayRefCls, T, N - 1, PtrTraits, index_t>(
|
||||
this->data_ + this->strides_[0] * i,
|
||||
this->sizes_ + 1,
|
||||
this->strides_ + 1);
|
||||
}
|
||||
};
|
||||
|
||||
template <class ArrayRefCls,
|
||||
typename T,
|
||||
template <typename U>
|
||||
class PtrTraits,
|
||||
typename index_t>
|
||||
class TensorAccessor<ArrayRefCls, T, 1, PtrTraits, index_t>
|
||||
: public TensorAccessorBase<ArrayRefCls, T, 1, PtrTraits, index_t> {
|
||||
public:
|
||||
typedef typename PtrTraits<T>::PtrType PtrType;
|
||||
|
||||
C10_HOST_DEVICE TensorAccessor(PtrType data,
|
||||
const index_t* sizes,
|
||||
const index_t* strides)
|
||||
: TensorAccessorBase<ArrayRefCls, T, 1, PtrTraits, index_t>(
|
||||
data, sizes, strides) {}
|
||||
C10_HOST_DEVICE T& operator[](index_t i) {
|
||||
return this->data_[this->strides_[0] * i];
|
||||
}
|
||||
C10_HOST_DEVICE const T& operator[](index_t i) const {
|
||||
return this->data_[this->strides_[0] * i];
|
||||
}
|
||||
};
|
||||
|
||||
// GenericPackedTensorAccessorBase and GenericPackedTensorAccessor are used on
|
||||
// for CUDA `Tensor`s on the host and as in contrast to `TensorAccessor`s, they
|
||||
// copy the strides and sizes on instantiation (on the host) in order to
|
||||
// transfer them on the device when calling kernels. On the device, indexing of
|
||||
// multidimensional tensors gives to `TensorAccessor`s. Use RestrictPtrTraits as
|
||||
// PtrTraits if you want the tensor's data pointer to be marked as __restrict__.
|
||||
// Instantiation from data, sizes, strides is only needed on the host and
|
||||
// std::copy isn't available on the device, so those functions are host only.
|
||||
template <typename IndexBoundsCheck,
|
||||
typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
class GenericPackedTensorAccessorBase {
|
||||
public:
|
||||
typedef typename PtrTraits<T>::PtrType PtrType;
|
||||
C10_HOST GenericPackedTensorAccessorBase(PtrType data,
|
||||
const index_t* sizes,
|
||||
const index_t* strides)
|
||||
: data_(data) {
|
||||
std::copy(sizes, sizes + N, std::begin(this->sizes_));
|
||||
std::copy(strides, strides + N, std::begin(this->strides_));
|
||||
}
|
||||
|
||||
// if index_t is not int64_t, we want to have an int64_t constructor
|
||||
template <typename source_index_t,
|
||||
class = std::enable_if_t<std::is_same_v<source_index_t, int64_t>>>
|
||||
C10_HOST GenericPackedTensorAccessorBase(PtrType data,
|
||||
const source_index_t* sizes,
|
||||
const source_index_t* strides)
|
||||
: data_(data) {
|
||||
for (size_t i = 0; i < N; ++i) {
|
||||
this->sizes_[i] = sizes[i];
|
||||
this->strides_[i] = strides[i];
|
||||
}
|
||||
}
|
||||
|
||||
C10_HOST_DEVICE index_t stride(index_t i) const { return strides_[i]; }
|
||||
C10_HOST_DEVICE index_t size(index_t i) const { return sizes_[i]; }
|
||||
C10_HOST_DEVICE PtrType data() { return data_; }
|
||||
C10_HOST_DEVICE const PtrType data() const { return data_; }
|
||||
|
||||
protected:
|
||||
PtrType data_;
|
||||
// NOLINTNEXTLINE(runtime/arrays)
|
||||
index_t sizes_[N];
|
||||
// NOLINTNEXTLINE(runtime/arrays)
|
||||
index_t strides_[N];
|
||||
C10_HOST void bounds_check_(index_t i) const { IndexBoundsCheck _(i); }
|
||||
};
|
||||
|
||||
template <typename ItemAccessor,
|
||||
typename IndexBoundsCheck,
|
||||
typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
class GenericPackedTensorAccessor
|
||||
: public GenericPackedTensorAccessorBase<IndexBoundsCheck,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t> {
|
||||
public:
|
||||
typedef typename PtrTraits<T>::PtrType PtrType;
|
||||
|
||||
C10_HOST GenericPackedTensorAccessor(PtrType data,
|
||||
const index_t* sizes,
|
||||
const index_t* strides)
|
||||
: GenericPackedTensorAccessorBase<IndexBoundsCheck,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t>(data, sizes, strides) {}
|
||||
|
||||
// if index_t is not int64_t, we want to have an int64_t constructor
|
||||
template <typename source_index_t,
|
||||
class = std::enable_if_t<std::is_same_v<source_index_t, int64_t>>>
|
||||
C10_HOST GenericPackedTensorAccessor(PtrType data,
|
||||
const source_index_t* sizes,
|
||||
const source_index_t* strides)
|
||||
: GenericPackedTensorAccessorBase<IndexBoundsCheck,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t>(data, sizes, strides) {}
|
||||
|
||||
C10_DEVICE ItemAccessor operator[](index_t i) {
|
||||
index_t* new_sizes = this->sizes_ + 1;
|
||||
index_t* new_strides = this->strides_ + 1;
|
||||
return ItemAccessor(
|
||||
this->data_ + this->strides_[0] * i, new_sizes, new_strides);
|
||||
}
|
||||
|
||||
C10_DEVICE const ItemAccessor operator[](index_t i) const {
|
||||
const index_t* new_sizes = this->sizes_ + 1;
|
||||
const index_t* new_strides = this->strides_ + 1;
|
||||
return ItemAccessor(
|
||||
this->data_ + this->strides_[0] * i, new_sizes, new_strides);
|
||||
}
|
||||
|
||||
/// Returns a PackedTensorAccessor of the same dimension after transposing the
|
||||
/// two dimensions given. Does not actually move elements; transposition is
|
||||
/// made by permuting the size/stride arrays. If the dimensions are not valid,
|
||||
/// asserts.
|
||||
C10_HOST GenericPackedTensorAccessor<ItemAccessor,
|
||||
IndexBoundsCheck,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t>
|
||||
transpose(index_t dim1, index_t dim2) const {
|
||||
this->bounds_check_(dim1);
|
||||
this->bounds_check_(dim2);
|
||||
GenericPackedTensorAccessor<ItemAccessor,
|
||||
IndexBoundsCheck,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t>
|
||||
result(this->data_, this->sizes_, this->strides_);
|
||||
std::swap(result.strides_[dim1], result.strides_[dim2]);
|
||||
std::swap(result.sizes_[dim1], result.sizes_[dim2]);
|
||||
return result;
|
||||
}
|
||||
};
|
||||
|
||||
template <typename ItemAccessor,
|
||||
typename IndexBoundsCheck,
|
||||
typename T,
|
||||
template <typename U>
|
||||
class PtrTraits,
|
||||
typename index_t>
|
||||
class GenericPackedTensorAccessor<ItemAccessor,
|
||||
IndexBoundsCheck,
|
||||
T,
|
||||
1,
|
||||
PtrTraits,
|
||||
index_t>
|
||||
: public GenericPackedTensorAccessorBase<IndexBoundsCheck,
|
||||
T,
|
||||
1,
|
||||
PtrTraits,
|
||||
index_t> {
|
||||
public:
|
||||
typedef typename PtrTraits<T>::PtrType PtrType;
|
||||
C10_HOST GenericPackedTensorAccessor(PtrType data,
|
||||
const index_t* sizes,
|
||||
const index_t* strides)
|
||||
: GenericPackedTensorAccessorBase<IndexBoundsCheck,
|
||||
T,
|
||||
1,
|
||||
PtrTraits,
|
||||
index_t>(data, sizes, strides) {}
|
||||
|
||||
// if index_t is not int64_t, we want to have an int64_t constructor
|
||||
template <typename source_index_t,
|
||||
class = std::enable_if_t<std::is_same_v<source_index_t, int64_t>>>
|
||||
C10_HOST GenericPackedTensorAccessor(PtrType data,
|
||||
const source_index_t* sizes,
|
||||
const source_index_t* strides)
|
||||
: GenericPackedTensorAccessorBase<IndexBoundsCheck,
|
||||
T,
|
||||
1,
|
||||
PtrTraits,
|
||||
index_t>(data, sizes, strides) {}
|
||||
|
||||
C10_DEVICE T& operator[](index_t i) {
|
||||
return this->data_[this->strides_[0] * i];
|
||||
}
|
||||
C10_DEVICE const T& operator[](index_t i) const {
|
||||
return this->data_[this->strides_[0] * i];
|
||||
}
|
||||
|
||||
// Same as in the general N-dimensional case, but note that in the
|
||||
// 1-dimensional case the returned PackedTensorAccessor will always be an
|
||||
// identical copy of the original
|
||||
C10_HOST GenericPackedTensorAccessor<ItemAccessor,
|
||||
IndexBoundsCheck,
|
||||
T,
|
||||
1,
|
||||
PtrTraits,
|
||||
index_t>
|
||||
transpose(index_t dim1, index_t dim2) const {
|
||||
this->bounds_check_(dim1);
|
||||
this->bounds_check_(dim2);
|
||||
return GenericPackedTensorAccessor<ItemAccessor,
|
||||
IndexBoundsCheck,
|
||||
T,
|
||||
1,
|
||||
PtrTraits,
|
||||
index_t>(
|
||||
this->data_, this->sizes_, this->strides_);
|
||||
}
|
||||
};
|
||||
|
||||
template <size_t N, typename index_t>
|
||||
struct HeaderOnlyIndexBoundsCheck {
|
||||
explicit HeaderOnlyIndexBoundsCheck(index_t i) {
|
||||
TORCH_CHECK(0 <= i && i < index_t{N},
|
||||
"Index ",
|
||||
i,
|
||||
" is not within bounds of a tensor of dimension ",
|
||||
N);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
// HeaderOnlyTensorAccessorBase is same as at::TensorAccessorBase.
|
||||
template <typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
using HeaderOnlyTensorAccessorBase =
|
||||
detail::TensorAccessorBase<c10::IntArrayRef, T, N, PtrTraits, index_t>;
|
||||
|
||||
// HeaderOnlyTensorAccessor is same as at::TensorAccessor.
|
||||
template <typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
using HeaderOnlyTensorAccessor =
|
||||
detail::TensorAccessor<c10::IntArrayRef, T, N, PtrTraits, index_t>;
|
||||
|
||||
// HeaderOnlyGenericPackedTensorAccessorBase is same as
|
||||
// at::GenericPackedTensorAccessorBase.
|
||||
template <typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
using HeaderOnlyGenericPackedTensorAccessorBase =
|
||||
detail::GenericPackedTensorAccessorBase<
|
||||
detail::HeaderOnlyIndexBoundsCheck<N, index_t>,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t>;
|
||||
|
||||
// HeaderOnlyGenericPackedTensorAccessor is same as
|
||||
// at::GenericPackedTensorAccessor.
|
||||
template <typename T,
|
||||
size_t N,
|
||||
template <typename U> class PtrTraits = DefaultPtrTraits,
|
||||
typename index_t = int64_t>
|
||||
using HeaderOnlyGenericPackedTensorAccessor =
|
||||
detail::GenericPackedTensorAccessor<
|
||||
HeaderOnlyTensorAccessor<T, N - 1, PtrTraits, index_t>,
|
||||
detail::HeaderOnlyIndexBoundsCheck<N, index_t>,
|
||||
T,
|
||||
N,
|
||||
PtrTraits,
|
||||
index_t>;
|
||||
|
||||
} // namespace torch::headeronly
|
||||
@@ -0,0 +1,85 @@
|
||||
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <c10/macros/Macros.h>
|
||||
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
#ifndef C10_UNLIKELY
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
#define C10_UNLIKELY(expr) (__builtin_expect(static_cast<bool>(expr), 0))
|
||||
#else
|
||||
#define C10_UNLIKELY(expr) (expr)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace c10 {
|
||||
|
||||
// Keep constexpr-friendly control flow when the check condition is constant
|
||||
// under nvcc/hipcc, matching upstream headeronly behavior.
|
||||
#if defined(__CUDACC__) || defined(__HIPCC__)
|
||||
#define C10_UNLIKELY_OR_CONST(e) e
|
||||
#else
|
||||
#define C10_UNLIKELY_OR_CONST(e) C10_UNLIKELY(e)
|
||||
#endif
|
||||
|
||||
} // namespace c10
|
||||
|
||||
#ifdef STRIP_ERROR_MESSAGES
|
||||
#define STD_TORCH_CHECK_MSG(cond, type, ...) \
|
||||
(#cond #type " CHECK FAILED at " C10_STRINGIZE(__FILE__))
|
||||
#else
|
||||
namespace torch::headeronly::detail {
|
||||
|
||||
template <typename... Args>
|
||||
inline std::string stdTorchCheckMsgImpl(const char* /*msg*/,
|
||||
const Args&... args) {
|
||||
std::ostringstream oss;
|
||||
((oss << args), ...);
|
||||
return oss.str();
|
||||
}
|
||||
|
||||
inline const char* stdTorchCheckMsgImpl(const char* msg) { return msg; }
|
||||
|
||||
inline const char* stdTorchCheckMsgImpl(const char* /*msg*/, const char* args) {
|
||||
return args;
|
||||
}
|
||||
|
||||
} // namespace torch::headeronly::detail
|
||||
|
||||
#define STD_TORCH_CHECK_MSG(cond, type, ...) \
|
||||
(torch::headeronly::detail::stdTorchCheckMsgImpl( \
|
||||
"Expected " #cond \
|
||||
" to be true, but got false. " \
|
||||
"(Could this error message be improved? If so, " \
|
||||
"please report an enhancement request to PyTorch.)", \
|
||||
##__VA_ARGS__))
|
||||
#endif
|
||||
|
||||
#define STD_TORCH_CHECK(cond, ...) \
|
||||
if (C10_UNLIKELY_OR_CONST(!(cond))) { \
|
||||
throw std::runtime_error(STD_TORCH_CHECK_MSG(cond, \
|
||||
"", \
|
||||
__func__, \
|
||||
", ", \
|
||||
__FILE__, \
|
||||
":", \
|
||||
__LINE__, \
|
||||
", ", \
|
||||
##__VA_ARGS__)); \
|
||||
}
|
||||
Reference in New Issue
Block a user