mirrors/deepseek-ai--3fs
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Implement deterministic shuffle with configurable g++/libstdc++ compatibility. (#369)

Browse Source
This commit is contained in:
Yiyuan Liu
2025-12-26 14:11:59 +08:00
committed by GitHub
parent c669cbc91a
commit bbb1e1ee04
6 changed files with 414 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+26
View File
@@ -27,6 +27,32 @@ if(ENABLE_ASSERTIONS)
endif()
endif()
set(SHUFFLE_METHODS "g++10" "g++11" "stdshuffle")
set(SHUFFLE_METHOD "" CACHE STRING "Choose the shuffle method (REQUIRED)")
set_property(CACHE SHUFFLE_METHOD PROPERTY STRINGS ${SHUFFLE_METHODS})
if("${SHUFFLE_METHOD}" STREQUAL "")
message(FATAL_ERROR
"\n [ERROR]: SHUFFLE_METHOD is not set!\n"
" You must specify it using: cmake -DSHUFFLE_METHOD=<value>\n"
" Valid options are: ${SHUFFLE_METHODS}"
)
elseif(NOT SHUFFLE_METHOD IN_LIST SHUFFLE_METHODS)
message(FATAL_ERROR
"\n [ERROR]: Invalid SHUFFLE_METHOD: '${SHUFFLE_METHOD}'\n"
" Valid options are: ${SHUFFLE_METHODS}"
)
endif()
message(STATUS "Use Shuffle Method: ${SHUFFLE_METHOD}")
if(SHUFFLE_METHOD STREQUAL "stdshuffle")
add_compile_definitions(USE_STD_SHUFFLE)
elseif(SHUFFLE_METHOD STREQUAL "g++10")
add_compile_definitions(USE_GCC10_SHUFFLE)
elseif(SHUFFLE_METHOD STREQUAL "g++11")
add_compile_definitions(USE_GCC11_SHUFFLE)
endif()
option(OVERRIDE_CXX_NEW_DELETE "Override C++ new/delete operator" OFF)
option(SAVE_ALLOCATE_SIZE "Use more memory to save allocate size" OFF)
README.md
+18 -8
View File
@@ -100,15 +100,25 @@ Install other build prerequisites:
## Build 3FS
- Build 3FS in `build` folder:
Build 3FS in `build` folder:
```
cmake -S . -B build -DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
cmake --build build -j 32
```
- Build 3FS use Docker
- For TencentOS-4: `docker pull docker.io/tencentos/tencentos4-deepseek3fs-build:latest`
- For OpenCloudOS-9: `docker pull docker.io/opencloudos/opencloudos9-deepseek3fs-build:latest`
```bash
# Replace <method> with 'g++10' or 'g++11' based on your environment
cmake -S . -B build \
-DCMAKE_CXX_COMPILER=clang++-14 -DCMAKE_C_COMPILER=clang-14 \
-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DSHUFFLE_METHOD=<method>
cmake --build build -j 32
```
Due to the historical use of `std::shuffle`, binaries compiled with different compiler versions (e.g., `g++10` vs. `g++11 +`) may be incompatible ([issue](https://github.com/deepseek-ai/3FS/issues/368)). To resolve this, you must explicitly specify `-DSHUFFLE_METHOD` during compilation to lock in a consistent shuffle algorithm:
- Existing Clusters: Use the method corresponding to the compiler version previously used to deploy the cluster (`g++10` or `g++11`).
- New Clusters: You can choose either `g++10` or `g++11`. However, once the cluster is deployed, you must stay with the same configuration for all future builds to maintain compatibility.
### Build 3FS use Docker
- For TencentOS-4: `docker pull docker.io/tencentos/tencentos4-deepseek3fs-build:latest`
- For OpenCloudOS-9: `docker pull docker.io/opencloudos/opencloudos9-deepseek3fs-build:latest`
## Run a test cluster
src/common/utils/Shuffle.h
+208
View File
@@ -0,0 +1,208 @@
#pragma once
#include <cassert>
#include <cstdint>
#include <folly/Random.h>
#include <folly/logging/xlog.h>
#include <glog/logging.h>
#include <numeric>
#include <optional>
#include <random>
#include <vector>
#include "common/utils/Int128.h"
namespace hf3fs {
#if defined __UINT64_TYPE__ && defined __UINT32_TYPE__ && __SIZEOF_INT128__
#else
#error "error"
#endif
#if (defined(USE_STD_SHUFFLE) + defined(USE_GCC10_SHUFFLE) + defined(USE_GCC11_SHUFFLE)) > 1
#error "multiple shuffle method defined"
#endif
#if defined(USE_STD_SHUFFLE)
__attribute__((used)) inline constexpr char HF3FS_SHUFFLE_METHOD[] = "HF3FS_SHUFFLE_METHOD=STD_SHUFFLE";
#elif defined(USE_GCC10_SHUFFLE)
__attribute__((used)) inline constexpr char HF3FS_SHUFFLE_METHOD[] = "HF3FS_SHUFFLE_METHOD=GCC10_SHUFFLE";
#elif defined(USE_GCC11_SHUFFLE)
__attribute__((used)) inline constexpr char HF3FS_SHUFFLE_METHOD[] = "HF3FS_SHUFFLE_METHOD=GCC11_SHUFFLE";
#else
#error "shuffle method not defined"
#endif
#if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE)
#else
#error "not libstdc++"
#endif
// Lemire's nearly divisionless algorithm.
// Returns an unbiased random number from __g downscaled to [0,__range)
// using an unsigned type _Wp twice as wide as unsigned type _Up.
// https://github.com/gcc-mirror/gcc/blob/a9c5f33f3a88fb9f6324beced1ccf30a6740f094/libstdc%2B%2B-v3/include/bits/uniform_int_dist.h#L252
inline uint64_t fast_range(std::mt19937_64 &urng, uint64_t range) {
// reference: Fast Random Integer Generation in an Interval
// ACM Transactions on Modeling and Computer Simulation 29 (1), 2019
// https://arxiv.org/abs/1805.10941
uint128_t product = uint128_t(urng()) * uint128_t(range);
uint64_t low = uint64_t(product);
if (low < range) {
uint64_t threshold = -range % range;
while (low < threshold) {
product = uint128_t(urng()) * uint128_t(range);
low = uint64_t(product);
}
}
return product >> 64;
}
// https://github.com/gcc-mirror/gcc/blob/e10dc8fa17ac633dfeca38cadfe0ba974af7e5a3/libstdc%2B%2B-v3/include/bits/uniform_int_dist.h#L288
template <const bool GCC11>
uint64_t gcc_uniform_int(uint64_t a, uint64_t b, std::mt19937_64 &urng) {
static_assert(std::is_same_v<std::mt19937_64::result_type, uint64_t>);
constexpr uint64_t urngrange = std::mt19937_64::max() - std::mt19937_64::min();
static_assert(std::mt19937_64::min() == 0);
static_assert(std::mt19937_64::max() == std::numeric_limits<uint64_t>::max());
static_assert(urngrange == std::numeric_limits<uint64_t>::max());
const uint64_t urange = b - a;
assert(urngrange >= urange);
uint64_t ret;
if (urngrange > urange) {
if constexpr (GCC11) {
// downscaling after g++11
const uint64_t uerange = urange + 1; // urange can be zero
ret = fast_range(urng, uerange);
} else {
// downscaling for g++10
const uint64_t uerange = urange + 1; // __urange can be zero
const uint64_t scaling = urngrange / uerange;
const uint64_t past = uerange * scaling;
do {
ret = uint64_t(urng());
} while (ret >= past);
ret /= scaling;
}
} else {
ret = urng();
}
return ret + a;
}
// https://github.com/gcc-mirror/gcc/blob/e10dc8fa17ac633dfeca38cadfe0ba974af7e5a3/libstdc%2B%2B-v3/include/bits/stl_algo.h#L3646
template <const bool GCC11>
std::pair<uint64_t, uint64_t> gen_two_uniform_ints(uint64_t b0, uint64_t b1, std::mt19937_64 &urng) {
const uint64_t x = gcc_uniform_int<GCC11>(0, (b0 * b1) - 1, urng);
return std::make_pair(x / b1, x % b1);
}
// https://github.com/gcc-mirror/gcc/blob/e10dc8fa17ac633dfeca38cadfe0ba974af7e5a3/libstdc%2B%2B-v3/include/bits/stl_algo.h#L3688
template <typename T, const bool GCC11>
void gcc_shuffle(std::vector<T> &vec, uint64_t mt19937_64_seed) {
if (vec.empty()) return;
std::mt19937_64 urng(mt19937_64_seed);
constexpr uint64_t urngrange = urng.max() - urng.min();
static_assert(urngrange == std::numeric_limits<uint64_t>::max());
const uint64_t urange = vec.size();
auto first = vec.begin();
auto last = vec.end();
if (urngrange / urange >= urange) {
auto i = first + 1;
// Since we know the vector isn't empty, an even number of elements
// means an uneven number of elements /to swap/, in which case we
// do the first one up front:
if ((urange % 2) == 0) {
std::iter_swap(i++, first + gcc_uniform_int<GCC11>(0, 1, urng));
}
// Now we know that last - i is even, so we do the rest in pairs,
// using a single distribution invocation to produce swap positions
// for two successive elements at a time:
while (i != last) {
const uint64_t swap_range = uint64_t(i - first) + 1;
const std::pair<uint64_t, uint64_t> pospos = gen_two_uniform_ints<GCC11>(swap_range, swap_range + 1, urng);
std::iter_swap(i++, first + pospos.first);
std::iter_swap(i++, first + pospos.second);
}
return;
}
for (auto i = first + 1; i != last; ++i) {
std::iter_swap(i, first + gcc_uniform_int<GCC11>(0, i - first, urng));
}
}
template <typename T>
void gcc11_shuffle(std::vector<T> &vec, uint64_t mt19937_64_seed) {
gcc_shuffle<T, true>(vec, mt19937_64_seed);
}
template <typename T>
void gcc10_shuffle(std::vector<T> &vec, uint64_t mt19937_64_seed) {
gcc_shuffle<T, false>(vec, mt19937_64_seed);
}
template <typename T>
void std_shuffle(std::vector<T> &vec, uint64_t mt19937_64_seed) {
std::mt19937_64 gen(mt19937_64_seed);
std::shuffle(vec.begin(), vec.end(), gen);
}
template <typename T>
void hf3fs_shuffle(std::vector<T> &vec, uint64_t mt19937_64_seed) {
#if defined(USE_STD_SHUFFLE)
std_shuffle(vec, mt19937_64_seed);
#elif defined(USE_GCC10_SHUFFLE)
gcc10_shuffle(vec, mt19937_64_seed);
#elif defined(USE_GCC11_SHUFFLE)
gcc11_shuffle(vec, mt19937_64_seed);
#else
#error "shuffle method not defined"
#endif
}
inline bool safe_shuffle_seed(uint32_t vec_len, uint64_t mt19937_64_seed) {
std::vector<uint32_t> vec1(vec_len);
std::iota(vec1.begin(), vec1.end(), 0);
std::vector<uint32_t> vec2 = vec1;
std::vector<uint32_t> vec3 = vec1;
assert(vec1 == vec2 && vec1 == vec3);
std_shuffle(vec1, mt19937_64_seed);
gcc_shuffle<uint32_t, true>(vec2, mt19937_64_seed);
gcc_shuffle<uint32_t, false>(vec3, mt19937_64_seed);
#if (_GLIBCXX_RELEASE >= 11)
XLOGF_IF(DFATAL, vec1 != vec2, "vector length {}, shuffle seed {}", vec_len, mt19937_64_seed);
#elif (_GLIBCXX_RELEASE >= 10)
XLOGF_IF(DFATAL, vec1 != vec3, "vector length {}, shuffle seed {}", vec_len, mt19937_64_seed);
#else
#error "g++ version < 10"
#endif
return vec1 == vec2 && vec1 == vec3;
}
inline std::optional<uint64_t> find_safe_seed(uint32_t vec_len) {
for (size_t i = 0; i < 1000; i++) {
auto seed = folly::Random::rand64();
if (safe_shuffle_seed(vec_len, seed)) {
return seed;
}
}
// probability should be very low
XLOGF(DFATAL, "can't find safe shuffle seed for vec size {}", vec_len);
return std::nullopt;
}
} // namespace hf3fs
src/fbs/meta/Schema.cc
+6 -3
View File
@@ -10,6 +10,7 @@
#include <vector>
#include "common/utils/Result.h"
#include "common/utils/Shuffle.h"
#include "fbs/core/user/User.h"
#include "fbs/meta/Common.h"
#include "fbs/mgmtd/MgmtdTypes.h"
@@ -118,7 +119,10 @@ Layout Layout::newChainRange(ChainTableId table,
uint32_t chunk,
uint32_t stripe,
uint32_t baseChainIndex) {
auto chains = ChainRange(baseChainIndex, ChainRange::Shuffle::STD_SHUFFLE_MT19937, folly::Random::rand64());
auto seed = find_safe_seed(stripe);
auto chains = ChainRange(baseChainIndex,
seed ? ChainRange::Shuffle::STD_SHUFFLE_MT19937 : ChainRange::Shuffle::NO_SHUFFLE,
seed ? *seed : 0);
return Layout{table, tableVer, chunk, stripe, chains};
}
@@ -164,8 +168,7 @@ std::span<const uint32_t> Layout::ChainRange::getChainIndexList(size_t stripe) c
case NO_SHUFFLE:
break;
case STD_SHUFFLE_MT19937: {
auto rng = std::mt19937_64(seed);
std::shuffle(chains.begin(), chains.end(), rng);
hf3fs_shuffle(chains, seed);
break;
}
default:
src/meta/components/ChainAllocator.h
+7 -1
View File
@@ -14,6 +14,7 @@
#include "client/mgmtd/ICommonMgmtdClient.h"
#include "common/utils/Coroutine.h"
#include "common/utils/Result.h"
#include "common/utils/Shuffle.h"
#include "fbs/meta/Schema.h"
#include "fbs/mgmtd/MgmtdTypes.h"
@@ -110,8 +111,13 @@ class ChainAllocator {
fmt::format("try to allocate {} chains from {}, found {}", layout.stripeSize, tableId, chainCnt));
}
auto chainBegin = roundRobin(chainCnt);
// pick a safe shuffle seed
auto seed = find_safe_seed(layout.stripeSize);
if (!seed) {
co_return makeError(StatusCode::kInvalidArg, "Failed to find safe shuffle seed");
}
layout.tableVersion = table->chainTableVersion;
layout.chains = Layout::ChainRange(chainBegin, Layout::ChainRange::STD_SHUFFLE_MT19937, folly::Random::rand64());
layout.chains = Layout::ChainRange(chainBegin, Layout::ChainRange::STD_SHUFFLE_MT19937, *seed);
if (auto valid = layout.valid(false); valid.hasError()) {
XLOGF(DFATAL, "Layout is not valid after alloc {}, error {}", layout, valid.error());
CO_RETURN_ERROR(valid);
tests/common/utils/TestShuffle.cc
+149
View File
@@ -0,0 +1,149 @@
#include <cstdint>
#include <folly/Random.h>
#include <folly/experimental/TestUtil.h>
#include <folly/logging/xlog.h>
#include <gflags/gflags.h>
#include <gtest/gtest.h>
#include "common/utils/Shuffle.h"
#include "fmt/core.h"
DEFINE_int32(test_shuffle, 10000, "shuffle times");
DEFINE_string(test_dump_path, "", "3fs meta dump path");
namespace hf3fs::test {
namespace {
TEST(TestShuffle, Basic) {
// most random seed should be safe if vector size is small
for (int i = 0; i < FLAGS_test_shuffle; i++) {
auto seed = folly::Random::rand64();
auto vecsize = folly::Random::rand32(200);
ASSERT_TRUE(safe_shuffle_seed(vecsize, seed)) << " " << seed << " " << vecsize;
}
}
TEST(TestShuffle, FindUnsafeSeed) {
// for large vector size, we can find mismatch seed
for (int i = 0; i < 10000; i++) {
auto seed = folly::Random::rand64();
auto vecsize = 10000;
if (!safe_shuffle_seed(vecsize, seed)) {
fmt::println("find unsafe seed {} for vecsize {}", seed, vecsize);
return;
}
}
ASSERT_TRUE(false);
}
TEST(TestShuffle, KnownUnsafeSeed) {
// we know some seed is not safe
std::vector<uint64_t> unsafeSeeds = {6853942027173882469ull,
12595889032228965821ull,
8619779154399616190ull,
12360005796445515182ull,
14687818431154681368ull,
17026414701752999916ull,
12928634241705049562ull,
5961001693986322440ull,
8264875872430788387ull,
8715369310580772031ull,
17199606764515290853ull,
15088078859152357184ull,
13543500868882399137ull,
472380080658457497ull};
for (auto seed : unsafeSeeds) {
ASSERT_FALSE(safe_shuffle_seed(200, seed)) << seed;
#if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE)
std::vector<uint32_t> vec1(200);
std::iota(vec1.begin(), vec1.end(), 0);
std::vector<uint32_t> vec2 = vec1;
std_shuffle(vec1, seed);
#if (_GLIBCXX_RELEASE >= 11)
gcc11_shuffle(vec2, seed);
#elif (_GLIBCXX_RELEASE >= 10)
gcc10_shuffle(vec2, seed);
#else
static_assert(false);
#endif
ASSERT_EQ(vec1, vec2) << "seed: " << seed;
#else
static_assert(false);
#endif
}
}
TEST(TestShuffle, CheckDump) {
if (FLAGS_test_dump_path.empty()) {
GTEST_SKIP();
}
std::ifstream file(FLAGS_test_dump_path);
ASSERT_TRUE(file.is_open()) << "failed to open: " << FLAGS_test_dump_path;
std::string line;
std::getline(file, line); // skip header
std::cerr << "id,seed,size,match_gcc10,match_gcc11" << std::endl;
uint64_t total = 0;
uint64_t mismatch_g10 = 0, mismatch_g11 = 0, mismatch_both = 0;
while (std::getline(file, line)) {
std::istringstream iss(line);
std::string id_str, seed_str, size_str;
std::getline(iss, id_str, ',');
std::getline(iss, seed_str, ',');
std::getline(iss, size_str, ',');
uint64_t id = std::stoull(id_str, nullptr, 16);
uint64_t seed = std::stoull(seed_str);
uint64_t size = std::stoull(size_str);
std::vector<uint64_t> vec_std(size), vec_g10(size), vec_g11(size);
std::iota(vec_std.begin(), vec_std.end(), 0);
std::iota(vec_g10.begin(), vec_g10.end(), 0);
std::iota(vec_g11.begin(), vec_g11.end(), 0);
std_shuffle(vec_std, seed);
gcc10_shuffle(vec_g10, seed);
gcc11_shuffle(vec_g11, seed);
bool match_g10 = (vec_std == vec_g10);
bool match_g11 = (vec_std == vec_g11);
if (!match_g10 || !match_g11) {
std::cerr << std::hex << id << std::dec << ","
<< seed << ","
<< size << ","
<< (match_g10 ? "true" : "false") << ","
<< (match_g11 ? "true" : "false") << std::endl;
}
if (!match_g10 && !match_g11) {
mismatch_both++;
} else if (!match_g10) {
mismatch_g10++;
} else if (!match_g11) {
mismatch_g11++;
}
total++;
if (total % 10000 == 0) {
std::cout << "Progress: " << total
<< ", g++10_only_mismatch: " << mismatch_g10
<< ", g++11_only_mismatch: " << mismatch_g11
<< ", both_mismatch: " << mismatch_both << std::endl;
}
}
std::cout << "Done. Total: " << total
<< ", g++10_only_mismatch: " << mismatch_g10
<< ", g++11_only_mismatch: " << mismatch_g11
<< ", both_mismatch: " << mismatch_both << std::endl;
}
} // namespace
} // namespace hf3fs::test