chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
@@ -0,0 +1,213 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <c10/util/Exception.h>
#include <cstdint>
#include <functional>
#include <iterator>
#include <vector>
#include "paddle/phi/common/int_array.h"
namespace c10 {
#define TORCH_CHECK_CONSTEXPR(COND, MSG) \
((COND) ? void(0) : throw std::runtime_error(MSG))
template <typename T>
class ArrayRef {
private:
/// The start of the array, in an external buffer.
const T* Data;
/// The number of elements.
size_t Length;
public:
using iterator = const T*;
using const_iterator = const T*;
using size_type = size_t;
using value_type = T;
using reverse_iterator = std::reverse_iterator<iterator>;
/* implicit */ constexpr ArrayRef() : Data(nullptr), Length(0) {}
constexpr ArrayRef(const T& OneElt) : Data(&OneElt), Length(1) {} // NOLINT
constexpr ArrayRef(const T* data, size_t length)
: Data(data), Length(length) {}
constexpr ArrayRef(const T* begin, const T* end)
: Data(begin), Length(end - begin) {}
template <typename Container,
typename U = decltype(std::declval<Container>().data()),
typename = std::enable_if_t<(std::is_same_v<U, T*> ||
std::is_same_v<U, T const*>)>>
/* implicit */ ArrayRef(const Container& container) // NOLINT
: Data(container.data()), Length(container.size()) {}
template <typename A>
/* implicit */ ArrayRef(const std::vector<T, A>& Vec) // NOLINT
: Data(Vec.data()), Length(Vec.size()) {
static_assert(!std::is_same_v<T, bool>,
"ArrayRef<bool> cannot be constructed from a "
"std::vector<bool> bitfield.");
}
template <size_t N>
/* implicit */ constexpr ArrayRef(const std::array<T, N>& Arr) // NOLINT
: Data(Arr.data()), Length(N) {}
template <size_t N>
/* implicit */ constexpr ArrayRef(const T (&Arr)[N]) // NOLINT
: Data(Arr), Length(N) {}
/* implicit */ constexpr ArrayRef(const std::initializer_list<T>& Vec)
: Data(std::begin(Vec) == std::end(Vec) ? static_cast<T*>(nullptr)
: std::begin(Vec)),
Length(Vec.size()) {}
constexpr iterator begin() const { return Data; }
constexpr iterator end() const { return Data + Length; }
constexpr const_iterator cbegin() const { return Data; }
constexpr const_iterator cend() const { return Data + Length; }
constexpr reverse_iterator rbegin() const { return reverse_iterator(end()); }
constexpr reverse_iterator rend() const { return reverse_iterator(begin()); }
constexpr bool allMatch(const std::function<bool(const T&)>& pred) const {
return std::all_of(cbegin(), cend(), pred);
}
constexpr bool empty() const { return Length == 0; }
constexpr const T* data() const { return Data; }
constexpr size_t size() const { return Length; }
constexpr const T& front() const {
TORCH_CHECK_CONSTEXPR(
!empty(), "ArrayRef: attempted to access front() of empty list");
return Data[0];
}
constexpr const T& back() const {
TORCH_CHECK_CONSTEXPR(!empty(),
"ArrayRef: attempted to access back() of empty list");
return Data[Length - 1];
}
constexpr bool equals(ArrayRef RHS) const {
return Length == RHS.Length && std::equal(begin(), end(), RHS.begin());
}
/// slice(n, m) - Take M elements of the array starting at element N
constexpr ArrayRef<T> slice(size_t N, size_t M) const {
TORCH_CHECK_CONSTEXPR(N + M <= size(), "ArrayRef: invalid slice");
return ArrayRef<T>(data() + N, M);
}
/// slice(n) - Chop off the first N elements of the array.
constexpr ArrayRef<T> slice(size_t N) const {
TORCH_CHECK_CONSTEXPR(N <= size(), "ArrayRef: invalid slice");
return slice(N, size() - N);
}
constexpr const T& operator[](size_t Index) const { return Data[Index]; }
/// Vector compatibility
constexpr const T& at(size_t Index) const {
TORCH_CHECK_CONSTEXPR(Index < Length, "ArrayRef: invalid index");
return Data[Index];
}
template <typename U>
std::enable_if_t<std::is_same_v<U, T>, ArrayRef<T>>& operator=(
U&& Temporary) = delete;
template <typename U>
std::enable_if_t<std::is_same_v<U, T>, ArrayRef<T>>& operator=(
std::initializer_list<U>) = delete;
std::vector<T> vec() const { return std::vector<T>(Data, Data + Length); }
const paddle::experimental::IntArray _PD_ToPaddleIntArray() const {
return paddle::experimental::IntArray(
reinterpret_cast<const int64_t*>(Data), Length);
}
};
template <typename T>
std::ostream& operator<<(std::ostream& out, ArrayRef<T> list) {
int i = 0;
out << "[";
for (const auto& e : list) {
if (i++ > 0) out << ", ";
out << e;
}
out << "]";
return out;
}
template <typename T>
bool operator==(c10::ArrayRef<T> a1, c10::ArrayRef<T> a2) {
return a1.equals(a2);
}
template <typename T>
bool operator!=(c10::ArrayRef<T> a1, c10::ArrayRef<T> a2) {
return !a1.equals(a2);
}
template <typename T>
bool operator==(const std::vector<T>& a1, c10::ArrayRef<T> a2) {
return c10::ArrayRef<T>(a1).equals(a2);
}
template <typename T>
bool operator!=(const std::vector<T>& a1, c10::ArrayRef<T> a2) {
return !c10::ArrayRef<T>(a1).equals(a2);
}
template <typename T>
bool operator==(c10::ArrayRef<T> a1, const std::vector<T>& a2) {
return a1.equals(c10::ArrayRef<T>(a2));
}
template <typename T>
bool operator!=(c10::ArrayRef<T> a1, const std::vector<T>& a2) {
return !a1.equals(c10::ArrayRef<T>(a2));
}
using IntArrayRef = ArrayRef<int64_t>;
} // namespace c10
namespace at {
using c10::ArrayRef;
using c10::IntArrayRef;
} // namespace at
namespace torch {
using c10::ArrayRef;
using c10::IntArrayRef;
} // namespace torch
@@ -0,0 +1,29 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/phi/common/bfloat16.h"
namespace c10 {
using BFloat16 = ::phi::dtype::bfloat16;
} // namespace c10
namespace at {
using c10::BFloat16;
} // namespace at
namespace torch {
using c10::BFloat16;
} // namespace torch
@@ -0,0 +1,170 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <c10/macros/Macros.h>
#include <torch/headeronly/util/Exception.h>
#include <cstdint>
#include <exception>
#include <iostream>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <string>
#include <tuple>
#include <variant>
#include <vector>
#include "paddle/common/enforce.h"
#include "paddle/common/errors.h"
#include "paddle/common/exception.h"
#include "paddle/common/macros.h"
namespace c10 {
#define TORCH_CHECK(COND, ...) PD_CHECK(COND, ##__VA_ARGS__);
#define TORCH_INTERNAL_ASSERT(COND, ...) PD_CHECK(COND, ##__VA_ARGS__);
#define TORCH_CHECK_OP(val1, val2, op) \
do { \
auto&& _val1 = (val1); \
auto&& _val2 = (val2); \
if (!(_val1 op _val2)) { \
std::ostringstream _result; \
_result << "Check failed: " #val1 " " #op " " #val2 " (" << _val1 \
<< " vs. " << _val2 << "). "; \
PD_THROW(_result.str()); \
} \
} while (false);
// Check for a given boolean condition.
#ifndef CHECK
#define CHECK(condition) PD_CHECK(condition, "CHECK failed : ", #condition)
#endif
// TORCH_CHECK_OP macro definitions
#define TORCH_CHECK_EQ(val1, val2) TORCH_CHECK_OP(val1, val2, ==)
#define TORCH_CHECK_NE(val1, val2) TORCH_CHECK_OP(val1, val2, !=)
#define TORCH_CHECK_LE(val1, val2) TORCH_CHECK_OP(val1, val2, <=)
#define TORCH_CHECK_LT(val1, val2) TORCH_CHECK_OP(val1, val2, <)
#define TORCH_CHECK_GE(val1, val2) TORCH_CHECK_OP(val1, val2, >=)
#define TORCH_CHECK_GT(val1, val2) TORCH_CHECK_OP(val1, val2, >)
} // namespace c10
enum class C10ErrorType {
NotImplementedError,
Error,
};
constexpr auto NotImplementedError = C10ErrorType::NotImplementedError;
constexpr auto Error = C10ErrorType::Error;
inline void C10ThrowImpl(C10ErrorType err_type, const std::string& msg) {
switch (err_type) {
case C10ErrorType::NotImplementedError:
PADDLE_THROW(common::errors::Unimplemented(msg));
break;
case C10ErrorType::Error:
PADDLE_THROW(common::errors::InvalidArgument(msg));
break;
default:
PADDLE_THROW(common::errors::Fatal("Unknown error type: " + msg));
}
}
#define C10_THROW_ERROR(err_type, msg) C10ThrowImpl(err_type, msg)
// Warning support - simplified implementation compatible with PyTorch API
namespace c10 {
// Warning types
struct UserWarning {};
struct DeprecationWarning {};
// Simple Warning class
class Warning {
public:
template <typename WarningType, typename... Args>
Warning(WarningType type,
const std::tuple<const char*, const char*, uint32_t>& location,
const std::string& msg,
bool verbatim)
: msg_(msg), verbatim_(verbatim) {
(void)type;
(void)location;
(void)verbatim;
}
const std::string& msg() const { return msg_; }
private:
std::string msg_;
bool verbatim_;
};
// Warning handler - prints to stderr
inline void warn(const Warning& warning) {
std::cerr << "Warning: " << warning.msg() << std::endl;
}
// Helper to concatenate message arguments
template <typename... Args>
inline std::string torch_warn_msg_impl(const Args&... args) {
std::ostringstream oss;
(oss << ... << args);
return oss.str();
}
inline std::string torch_warn_msg_impl() { return ""; }
} // namespace c10
// TORCH_WARN macros
#ifdef DISABLE_WARN
#define _TORCH_WARN_WITH(...) ((void)0)
#else
#define _TORCH_WARN_WITH(warning_t, ...) \
do { \
::c10::warn(::c10::Warning( \
warning_t{}, \
std::make_tuple(__func__, __FILE__, static_cast<uint32_t>(__LINE__)), \
::c10::torch_warn_msg_impl(__VA_ARGS__), \
false)); \
} while (0)
#endif
#define TORCH_WARN(...) _TORCH_WARN_WITH(::c10::UserWarning, __VA_ARGS__)
#define TORCH_WARN_DEPRECATION(...) \
_TORCH_WARN_WITH(::c10::DeprecationWarning, __VA_ARGS__)
// TORCH_WARN_ONCE - only warns once per call site
#ifdef DISABLE_WARN
#define TORCH_WARN_ONCE(...) ((void)0)
#else
#define TORCH_WARN_ONCE(...) \
do { \
static bool C10_ANONYMOUS_VARIABLE(torch_warn_once_) = [] { \
TORCH_WARN(__VA_ARGS__); \
return true; \
}(); \
} while (0)
#endif
// Deprecated attribute macro
#define C10_DEPRECATED_MESSAGE(msg) [[deprecated(msg)]]
@@ -0,0 +1,65 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
/// Defines the Float4_e2m1fn_x2 type (4-bit floating-point, two elements packed
/// into one byte). This is the FP4 dtype from the OCP MX format spec
/// (https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf,
/// Section 5.3.3)
///
/// Given two high precision values val0 and val1, here is the
/// binary configuration of their packed representation, from MSB to LSB:
///
/// original value | val1 : val0
/// ========================================
/// bit index (MSB==7, LSB==0) | 7654 : 3210
/// sign/exponent/mantissa | seem : seem
struct alignas(1) Float4_e2m1fn_x2 {
uint8_t val_;
Float4_e2m1fn_x2() = default;
explicit constexpr Float4_e2m1fn_x2(uint8_t val) : val_(val) {}
};
/// Comparison operators
inline bool operator==(const Float4_e2m1fn_x2& a, const Float4_e2m1fn_x2& b) {
return a.val_ == b.val_;
}
inline bool operator!=(const Float4_e2m1fn_x2& a, const Float4_e2m1fn_x2& b) {
return a.val_ != b.val_;
}
} // namespace c10
namespace at {
using c10::Float4_e2m1fn_x2;
using c10::operator!=;
using c10::operator==;
} // namespace at
namespace torch {
using c10::Float4_e2m1fn_x2;
using c10::operator!=;
using c10::operator==;
} // namespace torch
@@ -0,0 +1,27 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/phi/common/float8_e4m3fn.h"
namespace c10 {
using Float8_e4m3fn = ::phi::dtype::float8_e4m3fn;
} // namespace c10
namespace at {
using c10::Float8_e4m3fn;
} // namespace at
namespace torch {
using c10::Float8_e4m3fn;
} // namespace torch
@@ -0,0 +1,40 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
struct Float8_e4m3fnuz {
constexpr Float8_e4m3fnuz() = default;
explicit constexpr Float8_e4m3fnuz(uint8_t value) : x(value) {}
uint8_t x{0};
};
} // namespace c10
namespace at {
using c10::Float8_e4m3fnuz;
} // namespace at
namespace torch {
using c10::Float8_e4m3fnuz;
} // namespace torch
@@ -0,0 +1,28 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/phi/common/float8_e5m2.h"
namespace c10 {
using Float8_e5m2 = ::phi::dtype::float8_e5m2;
} // namespace c10
namespace at {
using c10::Float8_e5m2;
} // namespace at
namespace torch {
using c10::Float8_e5m2;
} // namespace torch
@@ -0,0 +1,40 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
struct Float8_e5m2fnuz {
constexpr Float8_e5m2fnuz() = default;
explicit constexpr Float8_e5m2fnuz(uint8_t value) : x(value) {}
uint8_t x{0};
};
} // namespace c10
namespace at {
using c10::Float8_e5m2fnuz;
} // namespace at
namespace torch {
using c10::Float8_e5m2fnuz;
} // namespace torch
@@ -0,0 +1,40 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
struct Float8_e8m0fnu {
constexpr Float8_e8m0fnu() = default;
explicit constexpr Float8_e8m0fnu(uint8_t value) : x(value) {}
uint8_t x{0};
};
} // namespace c10
namespace at {
using c10::Float8_e8m0fnu;
} // namespace at
namespace torch {
using c10::Float8_e8m0fnu;
} // namespace torch
@@ -0,0 +1,29 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/phi/common/float16.h"
namespace c10 {
using Half = ::phi::dtype::float16;
} // namespace c10
namespace at {
using c10::Half;
} // namespace at
namespace torch {
using c10::Half;
} // namespace torch
@@ -0,0 +1,26 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <optional>
namespace c10 {
// Aliases from C++17 std::optional
using std::bad_optional_access;
using std::make_optional;
using std::nullopt;
using std::nullopt_t;
using std::optional;
} // namespace c10
@@ -0,0 +1,234 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <c10/util/ArrayRef.h>
#include <cstdint>
#include <optional>
#include <vector>
namespace c10 {
template <typename T>
class OptionalArrayRef final {
public:
// Constructors
constexpr OptionalArrayRef() noexcept = default;
constexpr OptionalArrayRef(std::nullopt_t) noexcept {}
OptionalArrayRef(const OptionalArrayRef& other) = default;
OptionalArrayRef(OptionalArrayRef&& other) noexcept = default;
constexpr OptionalArrayRef(const std::optional<ArrayRef<T>>& other) noexcept
: wrapped_opt_array_ref(other) {}
constexpr OptionalArrayRef(std::optional<ArrayRef<T>>&& other) noexcept
: wrapped_opt_array_ref(std::move(other)) {}
constexpr OptionalArrayRef(const T& value) noexcept
: wrapped_opt_array_ref(value) {}
template <
typename U = ArrayRef<T>,
std::enable_if_t<!std::is_same_v<std::decay_t<U>, OptionalArrayRef> &&
!std::is_same_v<std::decay_t<U>, std::in_place_t> &&
std::is_constructible_v<ArrayRef<T>, U&&> &&
std::is_convertible_v<U&&, ArrayRef<T>> &&
!std::is_convertible_v<U&&, T>,
bool> = false>
constexpr OptionalArrayRef(U&& value) noexcept(
std::is_nothrow_constructible_v<ArrayRef<T>, U&&>)
: wrapped_opt_array_ref(std::forward<U>(value)) {}
template <
typename U = ArrayRef<T>,
std::enable_if_t<!std::is_same_v<std::decay_t<U>, OptionalArrayRef> &&
!std::is_same_v<std::decay_t<U>, std::in_place_t> &&
std::is_constructible_v<ArrayRef<T>, U&&> &&
!std::is_convertible_v<U&&, ArrayRef<T>>,
bool> = false>
constexpr explicit OptionalArrayRef(U&& value) noexcept(
std::is_nothrow_constructible_v<ArrayRef<T>, U&&>)
: wrapped_opt_array_ref(std::forward<U>(value)) {}
template <typename... Args>
constexpr explicit OptionalArrayRef(std::in_place_t ip,
Args&&... args) noexcept
: wrapped_opt_array_ref(ip, std::forward<Args>(args)...) {}
template <typename U, typename... Args>
constexpr explicit OptionalArrayRef(std::in_place_t ip,
std::initializer_list<U> il,
Args&&... args)
: wrapped_opt_array_ref(ip, il, std::forward<Args>(args)...) {}
constexpr OptionalArrayRef(const std::initializer_list<T>& Vec)
: wrapped_opt_array_ref(ArrayRef<T>(Vec)) {}
// Destructor
~OptionalArrayRef() = default;
// Assignment
constexpr OptionalArrayRef& operator=(std::nullopt_t) noexcept {
wrapped_opt_array_ref = std::nullopt;
return *this;
}
OptionalArrayRef& operator=(const OptionalArrayRef& other) = default;
OptionalArrayRef& operator=(OptionalArrayRef&& other) noexcept = default;
constexpr OptionalArrayRef& operator=(
const std::optional<ArrayRef<T>>& other) noexcept {
wrapped_opt_array_ref = other;
return *this;
}
constexpr OptionalArrayRef& operator=(
std::optional<ArrayRef<T>>&& other) noexcept {
wrapped_opt_array_ref = std::move(other);
return *this;
}
template <typename U = ArrayRef<T>,
typename = std::enable_if_t<
!std::is_same_v<std::decay_t<U>, OptionalArrayRef> &&
std::is_constructible_v<ArrayRef<T>, U&&> &&
std::is_assignable_v<ArrayRef<T>&, U&&>>>
constexpr OptionalArrayRef& operator=(U&& value) noexcept(
std::is_nothrow_constructible_v<ArrayRef<T>, U&&>&&
std::is_nothrow_assignable_v<ArrayRef<T>&, U&&>) {
wrapped_opt_array_ref = std::forward<U>(value);
return *this;
}
// Observers
constexpr ArrayRef<T>* operator->() noexcept {
return &wrapped_opt_array_ref.value();
}
constexpr const ArrayRef<T>* operator->() const noexcept {
return &wrapped_opt_array_ref.value();
}
constexpr ArrayRef<T>& operator*() & noexcept {
return wrapped_opt_array_ref.value();
}
constexpr const ArrayRef<T>& operator*() const& noexcept {
return wrapped_opt_array_ref.value();
}
constexpr ArrayRef<T>&& operator*() && noexcept {
return std::move(wrapped_opt_array_ref.value());
}
constexpr const ArrayRef<T>&& operator*() const&& noexcept {
return std::move(wrapped_opt_array_ref.value());
}
constexpr explicit operator bool() const noexcept {
return wrapped_opt_array_ref.has_value();
}
constexpr bool has_value() const noexcept {
return wrapped_opt_array_ref.has_value();
}
constexpr ArrayRef<T>& value() & { return wrapped_opt_array_ref.value(); }
constexpr const ArrayRef<T>& value() const& {
return wrapped_opt_array_ref.value();
}
constexpr ArrayRef<T>&& value() && {
return std::move(wrapped_opt_array_ref.value());
}
constexpr const ArrayRef<T>&& value() const&& {
return std::move(wrapped_opt_array_ref.value());
}
template <typename U>
constexpr std::enable_if_t<std::is_convertible_v<U&&, ArrayRef<T>>,
ArrayRef<T>>
value_or(U&& default_value) const& {
return wrapped_opt_array_ref.value_or(std::forward<U>(default_value));
}
template <typename U>
constexpr std::enable_if_t<std::is_convertible_v<U&&, ArrayRef<T>>,
ArrayRef<T>>
value_or(U&& default_value) && {
return wrapped_opt_array_ref.value_or(std::forward<U>(default_value));
}
// Modifiers
constexpr void swap(OptionalArrayRef& other) noexcept {
std::swap(wrapped_opt_array_ref, other.wrapped_opt_array_ref);
}
constexpr void reset() noexcept { wrapped_opt_array_ref.reset(); }
template <typename... Args>
constexpr std::enable_if_t<std::is_constructible_v<ArrayRef<T>, Args&&...>,
ArrayRef<T>&>
emplace(Args&&... args) noexcept(
std::is_nothrow_constructible_v<ArrayRef<T>, Args&&...>) {
return wrapped_opt_array_ref.emplace(std::forward<Args>(args)...);
}
template <typename U, typename... Args>
constexpr ArrayRef<T>& emplace(std::initializer_list<U> il,
Args&&... args) noexcept {
return wrapped_opt_array_ref.emplace(il, std::forward<Args>(args)...);
}
private:
std::optional<ArrayRef<T>> wrapped_opt_array_ref;
};
using OptionalIntArrayRef = OptionalArrayRef<int64_t>;
inline bool operator==(const OptionalIntArrayRef& a1,
const IntArrayRef& other) {
if (!a1.has_value()) {
return false;
}
return a1.value() == other;
}
inline bool operator==(const c10::IntArrayRef& a1,
const c10::OptionalIntArrayRef& a2) {
return a2 == a1;
}
} // namespace c10
namespace at {
using c10::OptionalIntArrayRef;
} // namespace at
namespace torch {
using c10::OptionalIntArrayRef;
} // namespace torch
@@ -0,0 +1,95 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
#include <string_view>
#include <type_traits>
namespace c10 {
namespace util {
class type_index final {
public:
constexpr explicit type_index(uint64_t checksum = 0) : checksum_(checksum) {}
constexpr uint64_t underlyingId() const noexcept { return checksum_; }
friend constexpr bool operator==(type_index lhs, type_index rhs) noexcept {
return lhs.checksum_ == rhs.checksum_;
}
friend constexpr bool operator!=(type_index lhs, type_index rhs) noexcept {
return !(lhs == rhs);
}
friend constexpr bool operator<(type_index lhs, type_index rhs) noexcept {
return lhs.checksum_ < rhs.checksum_;
}
private:
uint64_t checksum_;
};
namespace detail {
constexpr uint64_t kFnvOffsetBasis = 14695981039346656037ULL;
constexpr uint64_t kFnvPrime = 1099511628211ULL;
constexpr uint64_t fnv1a64(const char* data, size_t n) {
uint64_t hash = kFnvOffsetBasis;
for (size_t i = 0; i < n; ++i) {
hash ^= static_cast<uint64_t>(static_cast<unsigned char>(data[i]));
hash *= kFnvPrime;
}
return hash;
}
template <typename T>
constexpr std::string_view type_signature() {
#if defined(_MSC_VER) && !defined(__clang__)
constexpr std::string_view sig = __FUNCSIG__;
#else
constexpr std::string_view sig = __PRETTY_FUNCTION__;
#endif
return sig;
}
template <typename T>
constexpr uint64_t type_index_impl() {
constexpr std::string_view sig = type_signature<T>();
return fnv1a64(sig.data(), sig.size());
}
} // namespace detail
template <typename T>
constexpr type_index get_type_index() {
return type_index(detail::type_index_impl<std::decay_t<T>>());
}
} // namespace util
} // namespace c10
namespace std {
template <>
struct hash<c10::util::type_index> {
size_t operator()(c10::util::type_index v) const noexcept {
return static_cast<size_t>(v.underlyingId());
}
};
} // namespace std
@@ -0,0 +1,139 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <c10/macros/Macros.h>
#include <cstddef>
#include <memory>
#include <utility>
namespace c10 {
using DeleterFnPtr = void (*)(void*);
namespace detail {
// Does not delete anything
inline void deleteNothing(void* /*unused*/) {}
// A detail::UniqueVoidPtr is an owning smart pointer like unique_ptr, but
// with three major differences:
//
// 1) It is specialized to void
//
// 2) It is specialized for a function pointer deleter
// void(void* ctx); i.e., the deleter doesn't take a
// reference to the data, just to a context pointer
// (erased as void*). In fact, internally, this pointer
// is implemented as having an owning reference to
// context, and a non-owning reference to data; this is why
// you release_context(), not release() (the conventional
// API for release() wouldn't give you enough information
// to properly dispose of the object later.)
//
// 3) The deleter is guaranteed to be called when the unique
// pointer is destructed and the context is non-null; this is different
// from std::unique_ptr where the deleter is not called if the
// data pointer is null.
//
// Some of the methods have slightly different types than std::unique_ptr
// to reflect this.
//
class UniqueVoidPtr {
private:
// Lifetime tied to ctx_
void* data_;
std::unique_ptr<void, DeleterFnPtr> ctx_;
public:
UniqueVoidPtr() : data_(nullptr), ctx_(nullptr, &deleteNothing) {}
explicit UniqueVoidPtr(void* data)
: data_(data), ctx_(nullptr, &deleteNothing) {}
UniqueVoidPtr(void* data, void* ctx, DeleterFnPtr ctx_deleter)
: data_(data), ctx_(ctx, ctx_deleter ? ctx_deleter : &deleteNothing) {}
void* operator->() const { return data_; }
void clear() {
ctx_ = nullptr;
data_ = nullptr;
}
void* get() const { return data_; }
bool /* success */ unsafe_reset_data_and_ctx(void* new_data_and_ctx) {
if (__builtin_expect(
static_cast<bool>((ctx_.get_deleter() != &deleteNothing)), 0)) {
return false;
}
// seems quicker than calling the no-op deleter when we reset
(void)ctx_.release();
ctx_.reset(new_data_and_ctx);
data_ = new_data_and_ctx;
return true;
}
void* get_context() const { return ctx_.get(); }
void* release_context() { return ctx_.release(); }
std::unique_ptr<void, DeleterFnPtr>&& move_context() {
return std::move(ctx_);
}
[[nodiscard]] bool compare_exchange_deleter(DeleterFnPtr expected_deleter,
DeleterFnPtr new_deleter) {
if (get_deleter() != expected_deleter) return false;
ctx_ = std::unique_ptr<void, DeleterFnPtr>(ctx_.release(), new_deleter);
return true;
}
template <typename T>
T* cast_context(DeleterFnPtr expected_deleter) const {
if (get_deleter() != expected_deleter) return nullptr;
return static_cast<T*>(get_context());
}
operator bool() const { return data_ || ctx_; }
DeleterFnPtr get_deleter() const { return ctx_.get_deleter(); }
};
// Note [How UniqueVoidPtr is implemented]
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// UniqueVoidPtr solves a common problem for allocators of tensor data, which
// is that the data pointer (e.g., float*) which you are interested in, is not
// the same as the context pointer (e.g., DLManagedTensor) which you need
// to actually deallocate the data. Under a conventional deleter design, you
// have to store extra context in the deleter itself so that you can actually
// delete the right thing. Implementing this with standard C++ is somewhat
// error-prone: if you use a std::unique_ptr to manage tensors, the deleter will
// not be called if the data pointer is nullptr, which can cause a leak if the
// context pointer is non-null (and the deleter is responsible for freeing both
// the data pointer and the context pointer).
//
// So, in our reimplementation of unique_ptr, which just store the context
// directly in the unique pointer, and attach the deleter to the context
// pointer itself. In simple cases, the context pointer is just the pointer
// itself.
inline bool operator==(const UniqueVoidPtr& sp, std::nullptr_t) noexcept {
return !sp;
}
inline bool operator==(std::nullptr_t, const UniqueVoidPtr& sp) noexcept {
return !sp;
}
inline bool operator!=(const UniqueVoidPtr& sp, std::nullptr_t) noexcept {
return sp;
}
inline bool operator!=(std::nullptr_t, const UniqueVoidPtr& sp) noexcept {
return sp;
}
} // namespace detail
} // namespace c10
@@ -0,0 +1,106 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <c10/util/Exception.h>
#include <cstdint>
#include <functional>
#include <iterator>
#include <numeric>
#include <type_traits>
#include <utility>
namespace c10 {
template <typename C,
std::enable_if_t<std::is_integral_v<typename C::value_type>, int> = 0>
inline int64_t sum_integers(const C& container) {
return std::accumulate(
container.begin(), container.end(), static_cast<int64_t>(0));
}
template <typename Iter,
std::enable_if_t<std::is_integral_v<
typename std::iterator_traits<Iter>::value_type>,
int> = 0>
inline int64_t sum_integers(Iter begin, Iter end) {
return std::accumulate(begin, end, static_cast<int64_t>(0));
}
template <typename C,
std::enable_if_t<std::is_integral_v<typename C::value_type>, int> = 0>
inline int64_t multiply_integers(const C& container) {
return std::accumulate(container.begin(),
container.end(),
static_cast<int64_t>(1),
std::multiplies<>());
}
template <typename Iter,
std::enable_if_t<std::is_integral_v<
typename std::iterator_traits<Iter>::value_type>,
int> = 0>
inline int64_t multiply_integers(Iter begin, Iter end) {
return std::accumulate(
begin, end, static_cast<int64_t>(1), std::multiplies<>());
}
template <typename C,
std::enable_if_t<std::is_integral_v<typename C::value_type>, int> = 0>
inline int64_t numelements_from_dim(const int k, const C& dims) {
if (k > static_cast<int>(dims.size())) {
return 1;
} else {
auto cbegin = dims.cbegin();
std::advance(cbegin, k);
return multiply_integers(cbegin, dims.cend());
}
}
template <typename C,
std::enable_if_t<std::is_integral_v<typename C::value_type>, int> = 0>
inline int64_t numelements_to_dim(const int k, const C& dims) {
TORCH_INTERNAL_ASSERT(0 <= k);
TORCH_INTERNAL_ASSERT((unsigned)k <= dims.size());
auto cend = dims.cbegin();
std::advance(cend, k);
return multiply_integers(dims.cbegin(), cend);
}
template <typename C,
std::enable_if_t<std::is_integral_v<typename C::value_type>, int> = 0>
inline int64_t numelements_between_dim(int k, int l, const C& dims) {
TORCH_INTERNAL_ASSERT(0 <= k);
TORCH_INTERNAL_ASSERT(0 <= l);
if (k > l) {
std::swap(k, l);
}
TORCH_INTERNAL_ASSERT((unsigned)l < dims.size());
auto cbegin = dims.cbegin();
auto cend = dims.cbegin();
std::advance(cbegin, k);
std::advance(cend, l);
return multiply_integers(cbegin, cend);
}
} // namespace c10
@@ -0,0 +1,76 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
struct bits1x8 {
constexpr bits1x8() = default;
explicit constexpr bits1x8(uint8_t value) : val_(value) {}
uint8_t val_{0};
};
struct bits2x4 {
constexpr bits2x4() = default;
explicit constexpr bits2x4(uint8_t value) : val_(value) {}
uint8_t val_{0};
};
struct bits4x2 {
constexpr bits4x2() = default;
explicit constexpr bits4x2(uint8_t value) : val_(value) {}
uint8_t val_{0};
};
struct bits8 {
constexpr bits8() = default;
explicit constexpr bits8(uint8_t value) : val_(value) {}
uint8_t val_{0};
};
struct bits16 {
constexpr bits16() = default;
explicit constexpr bits16(uint16_t value) : val_(value) {}
uint16_t val_{0};
};
} // namespace c10
namespace at {
using c10::bits16;
using c10::bits1x8;
using c10::bits2x4;
using c10::bits4x2;
using c10::bits8;
} // namespace at
namespace torch {
using c10::bits16;
using c10::bits1x8;
using c10::bits2x4;
using c10::bits4x2;
using c10::bits8;
} // namespace torch
@@ -0,0 +1,29 @@
// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "paddle/phi/common/complex.h"
namespace c10 {
template <typename T>
using complex = ::phi::dtype::complex<T>;
} // namespace c10
namespace at {
using c10::complex;
} // namespace at
namespace torch {
using c10::complex;
} // namespace torch
@@ -0,0 +1,442 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from the PyTorch project.
// Licensed under BSD-style license:
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <atomic>
#include <cstdint>
#include <type_traits>
#include <utility>
namespace c10 {
// Forward declarations
class intrusive_ptr_target;
namespace raw {
namespace intrusive_ptr {
inline void incref(intrusive_ptr_target* self);
inline void decref(intrusive_ptr_target* self);
} // namespace intrusive_ptr
namespace weak_intrusive_ptr {
inline void incref(intrusive_ptr_target* self);
inline void decref(intrusive_ptr_target* self);
} // namespace weak_intrusive_ptr
struct DontIncreaseRefcount {};
} // namespace raw
namespace detail {
constexpr uint64_t kImpracticallyHugeReferenceCount = 0x0FFFFFFF;
constexpr uint64_t kImpracticallyHugeWeakReferenceCount =
(kImpracticallyHugeReferenceCount << 32);
constexpr uint64_t kReferenceCountOne = 1;
constexpr uint64_t kWeakReferenceCountOne = (kReferenceCountOne << 32);
constexpr uint64_t kUniqueRef = (kReferenceCountOne | kWeakReferenceCountOne);
inline uint32_t refcount(uint64_t combined_refcount) {
return static_cast<uint32_t>(combined_refcount);
}
inline uint32_t weakcount(uint64_t combined_refcount) {
// Bit 63 is reserved for kHasPyObject in PyTorch (a flag indicating a live
// Python wrapper). This compat layer does not implement the PyObject path,
// so the bit will never be set, but we mask it out here to match PyTorch's
// extraction logic and remain numerically correct if the bit were ever set.
return static_cast<uint32_t>((combined_refcount & ~(uint64_t(1) << 63)) >>
32);
}
inline uint64_t atomic_combined_refcount_increment(
std::atomic<uint64_t>* combined_refcount, uint64_t inc) {
return combined_refcount->fetch_add(inc, std::memory_order_relaxed) + inc;
}
inline uint64_t atomic_combined_refcount_decrement(
std::atomic<uint64_t>* combined_refcount, uint64_t dec) {
return combined_refcount->fetch_sub(dec, std::memory_order_acq_rel) - dec;
}
inline uint32_t atomic_weakcount_increment(
std::atomic<uint64_t>* combined_refcount) {
return weakcount(atomic_combined_refcount_increment(combined_refcount,
kWeakReferenceCountOne));
}
inline uint32_t atomic_weakcount_decrement(
std::atomic<uint64_t>* combined_refcount) {
return weakcount(atomic_combined_refcount_decrement(combined_refcount,
kWeakReferenceCountOne));
}
template <class T>
struct intrusive_target_default_null_type final {
static constexpr T* singleton() noexcept { return nullptr; }
};
} // namespace detail
class intrusive_ptr_target {
public:
intrusive_ptr_target() noexcept : combined_refcount_(0) {}
intrusive_ptr_target(intrusive_ptr_target&& /*other*/) noexcept
: intrusive_ptr_target() {}
intrusive_ptr_target& operator=(intrusive_ptr_target&& /*other*/) noexcept {
return *this;
}
intrusive_ptr_target(const intrusive_ptr_target& /*other*/) noexcept
: intrusive_ptr_target() {}
intrusive_ptr_target& operator=(
const intrusive_ptr_target& /*other*/) noexcept {
return *this;
}
uint32_t refcount() const {
return detail::refcount(combined_refcount_.load(std::memory_order_relaxed));
}
uint32_t weakcount() const {
return detail::weakcount(
combined_refcount_.load(std::memory_order_relaxed));
}
protected:
virtual ~intrusive_ptr_target() = default;
private:
mutable std::atomic<uint64_t> combined_refcount_;
template <typename T, typename NullType>
friend class intrusive_ptr;
template <typename T, typename NullType>
friend class weak_intrusive_ptr;
friend inline void raw::intrusive_ptr::incref(intrusive_ptr_target* self);
friend inline void raw::intrusive_ptr::decref(intrusive_ptr_target* self);
friend inline void raw::weak_intrusive_ptr::incref(
intrusive_ptr_target* self);
friend inline void raw::weak_intrusive_ptr::decref(
intrusive_ptr_target* self);
};
namespace raw {
namespace intrusive_ptr {
inline void incref(intrusive_ptr_target* self) {
if (self) {
detail::atomic_combined_refcount_increment(&self->combined_refcount_,
detail::kReferenceCountOne);
}
}
inline void decref(intrusive_ptr_target* self) {
if (self) {
uint64_t new_count = detail::atomic_combined_refcount_decrement(
&self->combined_refcount_, detail::kReferenceCountOne);
if (detail::refcount(new_count) == 0) {
// All strong references gone; release the implicit weak reference
// (strong refs count as +1 to weakcount per the kUniqueRef invariant).
if (detail::atomic_weakcount_decrement(&self->combined_refcount_) == 0) {
delete self;
}
}
}
}
} // namespace intrusive_ptr
namespace weak_intrusive_ptr {
inline void incref(intrusive_ptr_target* self) {
if (self) {
detail::atomic_weakcount_increment(&self->combined_refcount_);
}
}
inline void decref(intrusive_ptr_target* self) {
if (self) {
if (detail::atomic_weakcount_decrement(&self->combined_refcount_) == 0) {
delete self;
}
}
}
} // namespace weak_intrusive_ptr
} // namespace raw
template <class TTarget, class NullType>
class weak_intrusive_ptr;
template <class TTarget,
class NullType = detail::intrusive_target_default_null_type<TTarget>>
class intrusive_ptr final {
private:
static_assert(
std::is_base_of_v<TTarget,
std::remove_pointer_t<decltype(NullType::singleton())>>,
"NullType::singleton() must return a element_type* pointer");
TTarget* target_;
template <class TTarget2, class NullType2>
friend class intrusive_ptr;
friend class weak_intrusive_ptr<TTarget, NullType>;
void retain_() noexcept {
if (target_ != NullType::singleton()) {
detail::atomic_combined_refcount_increment(&target_->combined_refcount_,
detail::kReferenceCountOne);
}
}
void reset_() noexcept {
if (target_ != NullType::singleton()) {
uint64_t new_count = detail::atomic_combined_refcount_decrement(
&target_->combined_refcount_, detail::kReferenceCountOne);
if (detail::refcount(new_count) == 0) {
// All strong references gone; release the implicit weak reference
// (strong refs count as +1 to weakcount per the kUniqueRef invariant).
if (detail::atomic_weakcount_decrement(&target_->combined_refcount_) ==
0) {
delete target_;
}
}
target_ = NullType::singleton();
}
}
public:
using element_type = TTarget;
using pointer = TTarget*;
intrusive_ptr() noexcept : target_(NullType::singleton()) {}
intrusive_ptr(std::nullptr_t) noexcept : target_(NullType::singleton()) {}
explicit intrusive_ptr(TTarget* raw) : target_(raw) {
if (target_ != NullType::singleton()) {
target_->combined_refcount_.store(detail::kUniqueRef,
std::memory_order_relaxed);
}
}
intrusive_ptr(const intrusive_ptr& rhs) : target_(rhs.target_) { retain_(); }
intrusive_ptr(intrusive_ptr&& rhs) noexcept : target_(rhs.target_) {
rhs.target_ = NullType::singleton();
}
template <typename From, typename FromNullType>
/* implicit */ intrusive_ptr(
const intrusive_ptr<From, FromNullType>& rhs) noexcept
: target_(rhs.target_) {
static_assert(std::is_convertible_v<From*, TTarget*>,
"Source type must be convertible to target type");
retain_();
}
template <typename From, typename FromNullType>
/* implicit */ intrusive_ptr(intrusive_ptr<From, FromNullType>&& rhs) noexcept
: target_(rhs.target_) {
static_assert(std::is_convertible_v<From*, TTarget*>,
"Source type must be convertible to target type");
rhs.target_ = FromNullType::singleton();
}
~intrusive_ptr() { reset_(); }
intrusive_ptr& operator=(const intrusive_ptr& rhs) {
if (this != &rhs) {
reset_();
target_ = rhs.target_;
retain_();
}
return *this;
}
intrusive_ptr& operator=(intrusive_ptr&& rhs) noexcept {
if (this != &rhs) {
reset_();
target_ = rhs.target_;
rhs.target_ = NullType::singleton();
}
return *this;
}
// Takes ownership of a raw pointer without incrementing the refcount.
static intrusive_ptr reclaim(TTarget* raw_ptr) {
intrusive_ptr result;
result.target_ = raw_ptr;
return result;
}
// unsafe_adopt is a PyTorch API compatibility alias for reclaim().
// Both adopt a raw pointer without incrementing the refcount; prefer
// reclaim() in new code.
static intrusive_ptr unsafe_adopt(TTarget* raw_ptr) {
return reclaim(raw_ptr);
}
TTarget* get() const noexcept { return target_; }
TTarget& operator*() const { return *target_; }
TTarget* operator->() const { return target_; }
explicit operator bool() const noexcept {
return target_ != NullType::singleton();
}
uint32_t use_count() const noexcept {
if (target_ == NullType::singleton()) {
return 0;
}
return target_->refcount();
}
bool defined() const noexcept { return target_ != NullType::singleton(); }
bool unique() const noexcept { return use_count() == 1; }
void reset() noexcept { reset_(); }
void swap(intrusive_ptr& other) noexcept {
using std::swap;
swap(target_, other.target_);
}
[[deprecated(
"intrusive_ptr::release is unsafe; use reclaim() or explicit ownership "
"transfer instead")]] TTarget*
release() noexcept {
TTarget* result = target_;
target_ = NullType::singleton();
return result;
}
bool operator==(const intrusive_ptr& rhs) const noexcept {
return target_ == rhs.target_;
}
bool operator!=(const intrusive_ptr& rhs) const noexcept {
return target_ != rhs.target_;
}
bool operator==(std::nullptr_t) const noexcept {
return target_ == NullType::singleton();
}
bool operator!=(std::nullptr_t) const noexcept {
return target_ != NullType::singleton();
}
};
template <class TTarget,
class NullType = detail::intrusive_target_default_null_type<TTarget>>
class weak_intrusive_ptr final {
private:
TTarget* target_;
template <class TTarget2, class NullType2>
friend class weak_intrusive_ptr;
friend class intrusive_ptr<TTarget, NullType>;
void retain_() {
if (target_ != NullType::singleton()) {
detail::atomic_weakcount_increment(&target_->combined_refcount_);
}
}
void reset_() noexcept {
if (target_ != NullType::singleton()) {
if (detail::atomic_weakcount_decrement(&target_->combined_refcount_) ==
0) {
delete target_;
}
target_ = NullType::singleton();
}
}
public:
using element_type = TTarget;
weak_intrusive_ptr() noexcept : target_(NullType::singleton()) {}
weak_intrusive_ptr(const intrusive_ptr<TTarget, NullType>& p)
: target_(p.target_) {
retain_();
}
weak_intrusive_ptr(const weak_intrusive_ptr& rhs) : target_(rhs.target_) {
retain_();
}
weak_intrusive_ptr(weak_intrusive_ptr&& rhs) noexcept : target_(rhs.target_) {
rhs.target_ = NullType::singleton();
}
~weak_intrusive_ptr() { reset_(); }
weak_intrusive_ptr& operator=(const weak_intrusive_ptr& rhs) {
if (this != &rhs) {
reset_();
target_ = rhs.target_;
retain_();
}
return *this;
}
weak_intrusive_ptr& operator=(weak_intrusive_ptr&& rhs) {
if (this != &rhs) {
reset_();
target_ = rhs.target_;
rhs.target_ = NullType::singleton();
}
return *this;
}
intrusive_ptr<TTarget, NullType> lock() const {
if (target_ == NullType::singleton()) {
return intrusive_ptr<TTarget, NullType>();
}
auto& atomic = target_->combined_refcount_;
uint64_t count = atomic.load(std::memory_order_relaxed);
while (true) {
if (detail::refcount(count) == 0) {
return intrusive_ptr<TTarget, NullType>();
}
if (atomic.compare_exchange_weak(count,
count + detail::kReferenceCountOne,
std::memory_order_acq_rel,
std::memory_order_relaxed)) {
return intrusive_ptr<TTarget, NullType>::unsafe_adopt(target_);
}
}
}
uint32_t use_count() const {
if (target_ == NullType::singleton()) {
return 0;
}
return target_->refcount();
}
bool expired() const { return use_count() == 0; }
void reset() { reset_(); }
};
// Creates a new T with an initial strong refcount of 1.
template <typename T, typename... Args>
intrusive_ptr<T> make_intrusive(Args&&... args) {
return intrusive_ptr<T>(new T(std::forward<Args>(args)...));
}
} // namespace c10
@@ -0,0 +1,43 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
/**
* qint32 is for signed 32 bit quantized Tensors
*/
struct alignas(4) qint32 {
using underlying = int32_t;
int32_t val_;
qint32() = default;
explicit constexpr qint32(int32_t val) : val_(val) {}
};
} // namespace c10
namespace at {
using c10::qint32;
} // namespace at
namespace torch {
using c10::qint32;
} // namespace torch
@@ -0,0 +1,45 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
/**
* This is the data type for quantized Tensors. Right now we only have
* qint8 which is for 8 bit Tensors, and qint32 for 32 bit int Tensors,
* we might have 4 bit, 2 bit or 1 bit data types in the future.
*/
struct alignas(1) qint8 {
using underlying = int8_t;
int8_t val_;
qint8() = default;
explicit constexpr qint8(int8_t val) : val_(val) {}
};
} // namespace c10
namespace at {
using c10::qint8;
} // namespace at
namespace torch {
using c10::qint8;
} // namespace torch
@@ -0,0 +1,44 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
/**
* quint2x4 is for un-signed 2 bit quantized Tensors that are packed to byte
* boundary.
*/
struct alignas(1) quint2x4 {
using underlying = uint8_t;
uint8_t val_;
quint2x4() = default;
explicit constexpr quint2x4(uint8_t val) : val_(val) {}
};
} // namespace c10
namespace at {
using c10::quint2x4;
} // namespace at
namespace torch {
using c10::quint2x4;
} // namespace torch
@@ -0,0 +1,44 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
/**
* quint4x2 is for un-signed 4 bit quantized Tensors that are packed to byte
* boundary.
*/
struct alignas(1) quint4x2 {
using underlying = uint8_t;
uint8_t val_;
quint4x2() = default;
explicit constexpr quint4x2(uint8_t val) : val_(val) {}
};
} // namespace c10
namespace at {
using c10::quint4x2;
} // namespace at
namespace torch {
using c10::quint4x2;
} // namespace torch
@@ -0,0 +1,43 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <cstdint>
namespace c10 {
/**
* quint8 is for unsigned 8 bit quantized Tensors
*/
struct alignas(1) quint8 {
using underlying = uint8_t;
uint8_t val_;
quint8() = default;
explicit constexpr quint8(uint8_t val) : val_(val) {}
};
} // namespace c10
namespace at {
using c10::quint8;
} // namespace at
namespace torch {
using c10::quint8;
} // namespace torch
@@ -0,0 +1,98 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#include <c10/util/typeid.h>
#include <algorithm>
namespace caffe2 {
std::mutex& TypeMeta::getTypeMetaDatasLock() {
static std::mutex lock;
return lock;
}
uint16_t TypeMeta::nextTypeIndex(
static_cast<uint16_t>(c10::ScalarType::NumOptions));
detail::TypeMetaData* TypeMeta::typeMetaDatas() {
static detail::TypeMetaData instances[kMaxTypeIndex + 1] = {
#define SCALAR_TYPE_META(T, _2, name) \
detail::TypeMetaData(sizeof(T), \
detail::_PickNew<T>(), \
detail::_PickPlacementNew<T>(), \
detail::_PickCopy<T>(), \
detail::_PickPlacementDelete<T>(), \
detail::_PickDelete<T>(), \
TypeIdentifier::Get<T>(), \
#name),
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(SCALAR_TYPE_META)
#undef SCALAR_TYPE_META
// Remaining entries default-initialize to empty TypeMetaData.
};
static std::once_flag init_once;
std::call_once(init_once, [] {
instances[static_cast<uint16_t>(c10::ScalarType::Undefined)] =
detail::TypeMetaData{0,
nullptr,
nullptr,
nullptr,
nullptr,
nullptr,
TypeIdentifier::Get<detail::_Uninitialized>(),
"Undefined"};
});
return instances;
}
uint16_t TypeMeta::existingMetaDataIndexForType(TypeIdentifier identifier) {
auto* meta_datas = typeMetaDatas();
const auto end = meta_datas + nextTypeIndex;
auto it = std::find_if(meta_datas, end, [identifier](const auto& meta_data) {
return meta_data.id_ == identifier;
});
if (it == end) {
return kMaxTypeIndex;
}
return static_cast<uint16_t>(it - meta_datas);
}
CAFFE_DEFINE_KNOWN_TYPE(std::string, std_string)
CAFFE_DEFINE_KNOWN_TYPE(char, char)
CAFFE_DEFINE_KNOWN_TYPE(std::unique_ptr<std::mutex>, std_unique_ptr_std_mutex)
CAFFE_DEFINE_KNOWN_TYPE(std::unique_ptr<std::atomic<bool>>,
std_unique_ptr_std_atomic_bool)
CAFFE_DEFINE_KNOWN_TYPE(std::vector<int32_t>, std_vector_int32_t)
CAFFE_DEFINE_KNOWN_TYPE(std::vector<int64_t>, std_vector_int64_t)
CAFFE_DEFINE_KNOWN_TYPE(std::vector<unsigned long>, // NOLINT(runtime/int)
std_vector_unsigned_long)
CAFFE_DEFINE_KNOWN_TYPE(bool*, bool_ptr)
CAFFE_DEFINE_KNOWN_TYPE(char*, char_ptr)
CAFFE_DEFINE_KNOWN_TYPE(int*, int_ptr)
CAFFE_DEFINE_KNOWN_TYPE(
detail::_guard_long_unique<long>, // NOLINT(runtime/int)
detail_guard_long_unique_long)
CAFFE_DEFINE_KNOWN_TYPE(
detail::_guard_long_unique<std::vector<long>>, // NOLINT(runtime/int)
detail_guard_long_unique_std_vector_long)
CAFFE_DEFINE_KNOWN_TYPE(float*, float_ptr)
CAFFE_DEFINE_KNOWN_TYPE(at::Half*, at_Half)
} // namespace caffe2
@@ -0,0 +1,652 @@
// Copyright (c) 2026 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The file has been adapted from pytorch project
// Licensed under BSD-style license -
// https://github.com/pytorch/pytorch/blob/main/LICENSE
#pragma once
#include <c10/macros/Macros.h>
#include <c10/util/BFloat16.h>
#include <c10/util/Exception.h>
#include <c10/util/Float8_e4m3fn.h>
#include <c10/util/Float8_e5m2.h>
#include <c10/util/Half.h>
#include <c10/util/TypeIndex.h>
#include <c10/core/ScalarType.h>
#include <array>
#include <atomic>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <mutex>
#include <ostream>
#include <string>
#include <string_view>
#include <type_traits>
#include <vector>
#include "paddle/common/enforce.h"
// ---------------------------------------------------------------------------
// Auxiliary macros not provided by the compat Macros.h
// ---------------------------------------------------------------------------
#ifndef C10_LIKELY
#ifdef _MSC_VER
#define C10_LIKELY(val) (val)
#else
#define C10_LIKELY(val) (__builtin_expect(static_cast<bool>(val), 1))
#endif
#endif
#ifndef C10_UNLIKELY
#ifdef _MSC_VER
#define C10_UNLIKELY(val) (val)
#else
#define C10_UNLIKELY(val) (__builtin_expect(static_cast<bool>(val), 0))
#endif
#endif
#ifndef C10_API
#define C10_API PADDLE_API
#endif
#ifndef C10_EXPORT
#ifdef _MSC_VER
#define C10_EXPORT __declspec(dllexport)
#else
#define C10_EXPORT __attribute__((visibility("default")))
#endif
#endif
#ifndef C10_ALWAYS_INLINE
#ifdef _MSC_VER
#define C10_ALWAYS_INLINE __forceinline
#else
#define C10_ALWAYS_INLINE __attribute__((always_inline)) inline
#endif
#endif
#ifndef TORCH_INTERNAL_ASSERT_DEBUG_ONLY
#ifdef NDEBUG
#define TORCH_INTERNAL_ASSERT_DEBUG_ONLY(...) ((void)0)
#else
#define TORCH_INTERNAL_ASSERT_DEBUG_ONLY(...) TORCH_INTERNAL_ASSERT(__VA_ARGS__)
#endif
#endif
// ---------------------------------------------------------------------------
// caffe2::TypeIdentifier
// ---------------------------------------------------------------------------
namespace caffe2 {
/**
* A unique run-time type identifier.
*/
class C10_API TypeIdentifier final {
public:
friend std::ostream& operator<<(std::ostream& stream, TypeIdentifier typeId);
friend bool operator<(TypeIdentifier lhs, TypeIdentifier rhs) noexcept;
template <typename T>
static constexpr TypeIdentifier Get() noexcept {
return TypeIdentifier(c10::util::get_type_index<T>());
}
static constexpr TypeIdentifier uninitialized() noexcept {
return TypeIdentifier(c10::util::type_index{0});
}
uint64_t underlyingId() const noexcept { return id_.underlyingId(); }
bool operator==(TypeIdentifier other) const noexcept {
return id_ == other.id_;
}
bool operator!=(TypeIdentifier other) const noexcept {
return id_ != other.id_;
}
private:
constexpr explicit TypeIdentifier(c10::util::type_index id) noexcept
: id_(id) {}
c10::util::type_index id_;
};
inline bool operator<(TypeIdentifier lhs, TypeIdentifier rhs) noexcept {
return lhs.id_ < rhs.id_;
}
inline std::ostream& operator<<(std::ostream& stream, TypeIdentifier typeId) {
return stream << typeId.underlyingId();
}
} // namespace caffe2
// ---------------------------------------------------------------------------
// at::DataType alias
// ---------------------------------------------------------------------------
namespace at {
using DataType = caffe2::TypeIdentifier;
} // namespace at
// ---------------------------------------------------------------------------
// std::hash specialisation so TypeIdentifier can be used in unordered maps
// ---------------------------------------------------------------------------
namespace std {
template <>
struct hash<caffe2::TypeIdentifier> {
std::size_t operator()(caffe2::TypeIdentifier id) const noexcept {
return id.underlyingId();
}
};
} // namespace std
// ---------------------------------------------------------------------------
// caffe2::detail TypeMetaData + helper templates
// ---------------------------------------------------------------------------
namespace caffe2 {
namespace detail {
/**
* Per-type metadata record. One instance lives per registered type.
*/
struct TypeMetaData final {
using New = void*();
using PlacementNew = void(void*, size_t);
using Copy = void(const void*, void*, size_t);
using PlacementDelete = void(void*, size_t);
using Delete = void(void*);
constexpr TypeMetaData() noexcept
: itemsize_(0),
new_(nullptr),
placementNew_(nullptr),
copy_(nullptr),
placementDelete_(nullptr),
delete_(nullptr),
id_(TypeIdentifier::uninitialized()),
name_("nullptr (uninitialized)") {}
constexpr TypeMetaData(size_t itemsize,
New* newFn,
PlacementNew* placementNew,
Copy* copy,
PlacementDelete* placementDelete,
Delete* deleteFn,
TypeIdentifier id,
std::string_view name) noexcept
: itemsize_(itemsize),
new_(newFn),
placementNew_(placementNew),
copy_(copy),
placementDelete_(placementDelete),
delete_(deleteFn),
id_(id),
name_(name) {}
size_t itemsize_;
New* new_;
PlacementNew* placementNew_;
Copy* copy_;
PlacementDelete* placementDelete_;
Delete* delete_;
TypeIdentifier id_;
std::string_view name_;
};
// Error helper keeps this header free of heavy includes.
[[noreturn]] inline void _ThrowRuntimeTypeLogicError(const std::string& msg) {
PADDLE_THROW(common::errors::InvalidArgument(msg));
}
// ---------------------------------------------------------------------------
// Trait: treat reduced-precision scalars as "fundamental" (skip ctor/dtor).
// ---------------------------------------------------------------------------
template <typename T>
struct is_paddle_fundamental : std::is_fundamental<T> {};
template <>
struct is_paddle_fundamental<at::Half> : std::true_type {};
template <>
struct is_paddle_fundamental<at::BFloat16> : std::true_type {};
template <>
struct is_paddle_fundamental<c10::Float8_e4m3fn> : std::true_type {};
template <>
struct is_paddle_fundamental<c10::Float8_e5m2> : std::true_type {};
// ---------------------------------------------------------------------------
// PlacementNew helpers
// ---------------------------------------------------------------------------
template <typename T>
inline void _PlacementNew(void* ptr, size_t n) {
T* typed_ptr = static_cast<T*>(ptr);
for (size_t i = 0; i < n; ++i) new (typed_ptr + i) T;
}
template <typename T>
inline void _PlacementNewNotDefault(void* /*ptr*/, size_t /*n*/) {
_ThrowRuntimeTypeLogicError(std::string("Type ") + typeid(T).name() +
" is not default-constructible.");
}
template <typename T,
std::enable_if_t<std::is_default_constructible_v<T>>* = nullptr>
inline constexpr TypeMetaData::PlacementNew* _PickPlacementNew() {
return (is_paddle_fundamental<T>::value || std::is_pointer_v<T>)
? nullptr
: &_PlacementNew<T>;
}
template <typename T,
std::enable_if_t<!std::is_default_constructible_v<T>>* = nullptr>
inline constexpr TypeMetaData::PlacementNew* _PickPlacementNew() {
return &_PlacementNewNotDefault<T>;
}
// ---------------------------------------------------------------------------
// New helpers
// ---------------------------------------------------------------------------
template <typename T>
inline void* _New() {
return new T;
}
template <typename T>
inline void* _NewNotDefault() {
_ThrowRuntimeTypeLogicError(std::string("Type ") + typeid(T).name() +
" is not default-constructible.");
}
template <typename T,
std::enable_if_t<std::is_default_constructible_v<T>>* = nullptr>
inline constexpr TypeMetaData::New* _PickNew() {
return &_New<T>;
}
template <typename T,
std::enable_if_t<!std::is_default_constructible_v<T>>* = nullptr>
inline constexpr TypeMetaData::New* _PickNew() {
return &_NewNotDefault<T>;
}
// ---------------------------------------------------------------------------
// Copy helpers
// ---------------------------------------------------------------------------
template <typename T>
inline void _Copy(const void* src, void* dst, size_t n) {
const T* typed_src = static_cast<const T*>(src);
T* typed_dst = static_cast<T*>(dst);
for (size_t i = 0; i < n; ++i) typed_dst[i] = typed_src[i];
}
template <typename T>
inline void _CopyNotAllowed(const void* /*src*/, void* /*dst*/, size_t /*n*/) {
_ThrowRuntimeTypeLogicError(std::string("Type ") + typeid(T).name() +
" does not allow assignment.");
}
template <typename T, std::enable_if_t<std::is_copy_assignable_v<T>>* = nullptr>
inline constexpr TypeMetaData::Copy* _PickCopy() {
return (is_paddle_fundamental<T>::value || std::is_pointer_v<T>) ? nullptr
: &_Copy<T>;
}
template <typename T,
std::enable_if_t<!std::is_copy_assignable_v<T>>* = nullptr>
inline constexpr TypeMetaData::Copy* _PickCopy() {
return &_CopyNotAllowed<T>;
}
// ---------------------------------------------------------------------------
// PlacementDelete helpers
// ---------------------------------------------------------------------------
template <typename T>
inline void _PlacementDelete(void* ptr, size_t n) {
T* typed_ptr = static_cast<T*>(ptr);
for (size_t i = 0; i < n; ++i) typed_ptr[i].~T();
}
template <typename T>
inline constexpr TypeMetaData::PlacementDelete* _PickPlacementDelete() {
return (is_paddle_fundamental<T>::value || std::is_pointer_v<T>)
? nullptr
: &_PlacementDelete<T>;
}
// ---------------------------------------------------------------------------
// Delete helpers
// ---------------------------------------------------------------------------
template <typename T>
inline void _Delete(void* ptr) {
delete static_cast<T*>(ptr);
}
template <class T>
inline constexpr TypeMetaData::Delete* _PickDelete() noexcept {
return &_Delete<T>;
}
// Sentinel type for uninitialized TypeMeta.
class _Uninitialized final {};
} // namespace detail
// ---------------------------------------------------------------------------
// caffe2::TypeMeta
// ---------------------------------------------------------------------------
/**
* TypeMeta is a thin class that stores the type of a container (e.g. Blob or
* Tensor elements) with a unique run-time id. It mirrors the PyTorch / Caffe2
* TypeMeta API so that code written against libtorch's typeid.h compiles
* against this compat header without changes.
*/
class C10_API TypeMeta final {
public:
using New = detail::TypeMetaData::New;
using PlacementNew = detail::TypeMetaData::PlacementNew;
using Copy = detail::TypeMetaData::Copy;
using PlacementDelete = detail::TypeMetaData::PlacementDelete;
using Delete = detail::TypeMetaData::Delete;
// Default-constructs to "Undefined / uninitialized".
// NOTE: body is defined AFTER the _Uninitialized specialization below
// to avoid "specialization after instantiation" errors.
TypeMeta() noexcept;
~TypeMeta() = default;
TypeMeta(const TypeMeta& src) noexcept = default;
TypeMeta& operator=(const TypeMeta& src) noexcept = default;
TypeMeta(TypeMeta&& src) noexcept = default;
TypeMeta& operator=(TypeMeta&& src) noexcept = default;
inline TypeMeta& operator=(c10::ScalarType scalar_type) noexcept {
index_ = static_cast<uint16_t>(scalar_type);
return *this;
}
// ------------------------------------------------------------------
// Accessors
// ------------------------------------------------------------------
TypeIdentifier id() const noexcept { return data().id_; }
inline bool isScalarType() const noexcept {
return index_ <= static_cast<uint16_t>(c10::ScalarType::Undefined);
}
inline bool isScalarType(c10::ScalarType scalar_type) const noexcept {
return index_ == static_cast<uint16_t>(scalar_type);
}
inline size_t itemsize() const noexcept { return data().itemsize_; }
New* newFn() const noexcept { return data().new_; }
PlacementNew* placementNew() const noexcept { return data().placementNew_; }
Copy* copy() const noexcept { return data().copy_; }
PlacementDelete* placementDelete() const noexcept {
return data().placementDelete_;
}
Delete* deleteFn() const noexcept { return data().delete_; }
std::string_view name() const noexcept { return data().name_; }
friend bool operator==(const TypeMeta& lhs, const TypeMeta& rhs) noexcept;
template <typename T>
bool Match() const noexcept {
return *this == Make<T>();
}
// ------------------------------------------------------------------
// Static helpers
// ------------------------------------------------------------------
template <class T>
static constexpr TypeIdentifier Id() noexcept {
return TypeIdentifier::Get<T>();
}
template <class T>
static std::string_view TypeName() noexcept {
return typeid(T).name();
}
template <class T>
static constexpr size_t ItemSize() noexcept {
return sizeof(T);
}
/** Returns a TypeMeta for type T. */
template <typename T>
static TypeMeta Make() {
return TypeMeta(_typeMetaData<T>());
}
/** Convert ScalarType enum → TypeMeta. */
static inline TypeMeta fromScalarType(c10::ScalarType scalar_type) {
const auto index = static_cast<uint16_t>(scalar_type);
TORCH_INTERNAL_ASSERT_DEBUG_ONLY(
index <= static_cast<uint16_t>(c10::ScalarType::Undefined),
"Unrecognized ScalarType");
return TypeMeta(index);
}
/** Convert TypeMeta → ScalarType enum. */
inline c10::ScalarType toScalarType() const {
if (C10_LIKELY(isScalarType())) {
return static_cast<c10::ScalarType>(index_);
}
PADDLE_THROW(common::errors::InvalidArgument(
"Unsupported TypeMeta in toScalarType (index=%d)",
static_cast<int>(index_)));
}
// ------------------------------------------------------------------
// Dynamic type registration (mirrors CAFFE_KNOWN_TYPE machinery)
// ------------------------------------------------------------------
template <class T>
static uint16_t addTypeMetaData() {
const auto identifier = TypeIdentifier::Get<T>();
std::lock_guard<std::mutex> lock(getTypeMetaDatasLock());
// Check whether already registered (e.g. from another DSO).
const uint16_t existing = existingMetaDataIndexForType(identifier);
if (existing != kMaxTypeIndex) return existing;
const uint16_t index = nextTypeIndex++;
TORCH_CHECK(index <= kMaxTypeIndex,
"Maximum number of CAFFE_KNOWN_TYPE declarations exceeded.");
typeMetaDatas()[index] =
detail::TypeMetaData{sizeof(T),
detail::_PickNew<T>(),
detail::_PickPlacementNew<T>(),
detail::_PickCopy<T>(),
detail::_PickPlacementDelete<T>(),
detail::_PickDelete<T>(),
identifier,
typeid(T).name()};
return index;
}
private:
explicit TypeMeta(uint16_t index) noexcept : index_(index) {}
// Maximum number of type-metadata slots (scalar + custom).
static constexpr uint16_t kMaxTypeIndex = 255;
static std::mutex& getTypeMetaDatasLock();
static uint16_t nextTypeIndex;
static detail::TypeMetaData* typeMetaDatas();
static uint16_t existingMetaDataIndexForType(TypeIdentifier identifier);
// Template specialisations return indexes into typeMetaDatas().
// Defined below the class for scalar types; compiled-in for custom types.
template <class T>
static uint16_t _typeMetaData() noexcept;
uint16_t index_;
inline const detail::TypeMetaData& data() const {
return typeMetaDatas()[index_];
}
};
// ---------------------------------------------------------------------------
// Specialisations of TypeMeta::_typeMetaData for ScalarType types
// ---------------------------------------------------------------------------
// 3-argument AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS
#define DEFINE_SCALAR_METADATA_INSTANCE(T, _2, name) \
template <> \
inline constexpr uint16_t TypeMeta::_typeMetaData<T>() noexcept { \
return static_cast<uint16_t>(c10::ScalarType::name); \
}
AT_FORALL_SCALAR_TYPES_WITH_COMPLEX_AND_QINTS(DEFINE_SCALAR_METADATA_INSTANCE)
#undef DEFINE_SCALAR_METADATA_INSTANCE
// _Uninitialized → Undefined slot
template <>
inline constexpr uint16_t
TypeMeta::_typeMetaData<detail::_Uninitialized>() noexcept {
return static_cast<uint16_t>(c10::ScalarType::Undefined);
}
// Default constructor defined here so that _typeMetaData<_Uninitialized> is
// already specialised before it is first called.
inline TypeMeta::TypeMeta() noexcept
: index_(_typeMetaData<detail::_Uninitialized>()) {}
// ---------------------------------------------------------------------------
// Comparison / stream operators
// ---------------------------------------------------------------------------
inline bool operator==(const TypeMeta& lhs, const TypeMeta& rhs) noexcept {
return lhs.index_ == rhs.index_;
}
inline bool operator!=(const TypeMeta& lhs, const TypeMeta& rhs) noexcept {
return !(lhs == rhs);
}
inline std::ostream& operator<<(std::ostream& stream,
caffe2::TypeMeta typeMeta) {
return stream << typeMeta.name();
}
// ---------------------------------------------------------------------------
// CAFFE_KNOWN_TYPE macros (compat versions)
// ---------------------------------------------------------------------------
#if defined(_MSC_VER) || defined(__clang__)
#define EXPORT_IF_NOT_GCC C10_EXPORT
#else
#define EXPORT_IF_NOT_GCC
#endif
#if defined(_MSC_VER)
#define C10_TEMPLATE_API C10_API
#else
#define C10_TEMPLATE_API
#endif
// For use in a .cpp file.
#define CAFFE_KNOWN_TYPE(T) \
template C10_TEMPLATE_API uint16_t TypeMeta::addTypeMetaData<T>(); \
template <> \
EXPORT_IF_NOT_GCC uint16_t TypeMeta::_typeMetaData<T>() noexcept { \
static const uint16_t index = addTypeMetaData<T>(); \
return index; \
}
// For use in a .cpp file when a declaration in the header is provided.
#define CAFFE_DEFINE_KNOWN_TYPE(T, ident) \
template C10_TEMPLATE_API uint16_t TypeMeta::addTypeMetaData<T>(); \
namespace detail { \
EXPORT_IF_NOT_GCC extern const uint16_t ident##_metadata_index = \
TypeMeta::addTypeMetaData<T>(); \
} /* namespace detail */
// Declaration counterpart: provides an inline fast-path via a detail var.
// NOTE: On MSVC, directly referencing cross-DLL const data symbols is fragile
// and can cause unresolved externals during libpaddle linking. Use a
// function-local static cache there and keep non-MSVC behavior aligned with
// upstream declare/define model.
#if defined(_MSC_VER)
#define CAFFE_DECLARE_KNOWN_TYPE(T, ident) \
extern template C10_API uint16_t TypeMeta::addTypeMetaData<T>(); \
namespace detail { \
extern C10_API const uint16_t ident##_metadata_index; \
} /* namespace detail */ \
template <> \
C10_ALWAYS_INLINE uint16_t TypeMeta::_typeMetaData<T>() noexcept { \
static const uint16_t index = addTypeMetaData<T>(); \
return index; \
}
#else
#define CAFFE_DECLARE_KNOWN_TYPE(T, ident) \
extern template uint16_t TypeMeta::addTypeMetaData<T>(); \
namespace detail { \
extern C10_API const uint16_t ident##_metadata_index; \
} /* namespace detail */ \
template <> \
EXPORT_IF_NOT_GCC C10_ALWAYS_INLINE uint16_t \
TypeMeta::_typeMetaData<T>() noexcept { \
return detail::ident##_metadata_index; \
}
#endif
// Header-safe variant: lazy static, no external .cpp needed.
#define CAFFE_KNOWN_TYPE_NOEXPORT(T) \
template <> \
inline uint16_t TypeMeta::_typeMetaData<T>() noexcept { \
static const uint16_t index = addTypeMetaData<T>(); \
return index; \
}
// ---------------------------------------------------------------------------
// Built-in known types
// ---------------------------------------------------------------------------
namespace detail {
template <class T>
class _guard_long_unique_dummy final {};
template <class T>
using _guard_long_unique =
std::conditional_t<std::is_same_v<long, int32_t> || // NOLINT(runtime/int)
std::is_same_v<long, // NOLINT(runtime/int)
int64_t>,
_guard_long_unique_dummy<T>,
T>;
} // namespace detail
CAFFE_DECLARE_KNOWN_TYPE(std::string, std_string)
CAFFE_DECLARE_KNOWN_TYPE(char, char)
CAFFE_DECLARE_KNOWN_TYPE(std::unique_ptr<std::mutex>, std_unique_ptr_std_mutex)
CAFFE_DECLARE_KNOWN_TYPE(std::unique_ptr<std::atomic<bool>>,
std_unique_ptr_std_atomic_bool)
CAFFE_DECLARE_KNOWN_TYPE(std::vector<int32_t>, std_vector_int32_t)
CAFFE_DECLARE_KNOWN_TYPE(std::vector<int64_t>, std_vector_int64_t)
CAFFE_DECLARE_KNOWN_TYPE(std::vector<unsigned long>, // NOLINT(runtime/int)
std_vector_unsigned_long)
CAFFE_DECLARE_KNOWN_TYPE(bool*, bool_ptr)
CAFFE_DECLARE_KNOWN_TYPE(char*, char_ptr)
CAFFE_DECLARE_KNOWN_TYPE(int*, int_ptr)
CAFFE_DECLARE_KNOWN_TYPE(
detail::_guard_long_unique<long>, // NOLINT(runtime/int)
detail_guard_long_unique_long)
CAFFE_DECLARE_KNOWN_TYPE(
detail::_guard_long_unique<std::vector<long>>, // NOLINT(runtime/int)
detail_guard_long_unique_std_vector_long)
CAFFE_DECLARE_KNOWN_TYPE(float*, float_ptr)
CAFFE_DECLARE_KNOWN_TYPE(at::Half*, at_Half)
} // namespace caffe2