// Copyright 2025-present the zvec project // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. #include #include #include #include #include #include #include #include #include "db/common/file_helper.h" #include "index/utils/utils.h" #include "zvec/db/collection.h" #include "zvec/db/doc.h" #include "zvec/db/options.h" #include "zvec/db/schema.h" #include "zvec/db/status.h" #if defined(_WIN32) || defined(_WIN64) #define WIN32_LEAN_AND_MEAN #define NOMINMAX #include #undef DeleteFile #undef RemoveDirectory #undef CreateFile #undef DELETE #endif using namespace zvec; using namespace zvec::test; // UTF-8 test path containing Chinese characters. // If the path handling is broken, the OS will show garbled names (mojibake). static const std::string kUtf8Dir = "utf8_col_\xe4\xb8\xad\xe6\x96\x87\xe6\xb5\x8b\xe8\xaf\x95"; // utf8_col_中文测试 class Utf8CollectionTest : public ::testing::Test { protected: void SetUp() override { ailego::FileHelper::RemovePath(kUtf8Dir.c_str()); } void TearDown() override { ailego::FileHelper::RemovePath(kUtf8Dir.c_str()); } }; // --------------------------------------------------------------------------- // 1. Create a collection in a UTF-8 path, insert docs, flush, reopen // --------------------------------------------------------------------------- TEST_F(Utf8CollectionTest, CreateInsertFlushReopen) { CollectionOptions opts; opts.read_only_ = false; opts.enable_mmap_ = true; auto schema = TestHelper::CreateNormalSchema(); auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts); if (!result.has_value()) { std::cout << result.error().message() << std::endl; } ASSERT_TRUE(result.has_value()); ASSERT_TRUE(ailego::FileHelper::IsExist(kUtf8Dir.c_str())); auto col = std::move(result).value(); ASSERT_EQ(col->Path(), kUtf8Dir); const int kDocCount = 10; auto s = TestHelper::CollectionInsertDoc(col, 0, kDocCount); ASSERT_TRUE(s.ok()) << s.message(); ASSERT_TRUE(col->Flush().ok()); auto stats = col->Stats().value(); ASSERT_EQ(stats.doc_count, kDocCount); col.reset(); // Reopen and verify doc count survives auto reopen = Collection::Open(kUtf8Dir, opts); ASSERT_TRUE(reopen.has_value()) << reopen.error().message(); auto col2 = std::move(reopen).value(); auto stats2 = col2->Stats().value(); ASSERT_EQ(stats2.doc_count, kDocCount); } // --------------------------------------------------------------------------- // 2. Destroy a collection in a UTF-8 path // --------------------------------------------------------------------------- TEST_F(Utf8CollectionTest, CreateAndDestroy) { CollectionOptions opts; opts.read_only_ = false; opts.enable_mmap_ = true; auto schema = TestHelper::CreateNormalSchema(); auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts); ASSERT_TRUE(result.has_value()); auto col = std::move(result).value(); ASSERT_EQ(col->Destroy(), Status::OK()); ASSERT_FALSE(ailego::FileHelper::IsExist(kUtf8Dir.c_str())); } // --------------------------------------------------------------------------- // 3. Verify the on-disk layout contains correct UTF-8 names (no mojibake) // by listing files via the wide-char Windows API or POSIX readdir. // --------------------------------------------------------------------------- #if defined(_WIN32) || defined(_WIN64) TEST_F(Utf8CollectionTest, NoDirGarble) { CollectionOptions opts; opts.read_only_ = false; opts.enable_mmap_ = true; auto schema = TestHelper::CreateNormalSchema(); auto result = Collection::CreateAndOpen(kUtf8Dir, *schema, opts); ASSERT_TRUE(result.has_value()); auto col = std::move(result).value(); const int kDocCount = 5; auto s = TestHelper::CollectionInsertDoc(col, 0, kDocCount); ASSERT_TRUE(s.ok()) << s.message(); ASSERT_TRUE(col->Flush().ok()); // --- Check parent directory via FindFirstFileW --- { WIN32_FIND_DATAW fd; HANDLE hFind = FindFirstFileW(L".\\*", &fd); ASSERT_NE(INVALID_HANDLE_VALUE, hFind); std::wstring expected = ailego::FileHelper::Utf8ToWide(kUtf8Dir); bool found = false; do { if (std::wstring(fd.cFileName) == expected) { found = true; break; } } while (FindNextFileW(hFind, &fd)); FindClose(hFind); EXPECT_TRUE(found) << "Collection dir '" << kUtf8Dir << "' not found via FindFirstFileW (garbled Unicode?)"; } // --- Check inside collection via "cmd /u /c dir /b" redirected to file --- { std::wstring wdir = ailego::FileHelper::Utf8ToWide(kUtf8Dir); std::string tmpfile = "utf8_col_dir_output.txt"; std::wstring wtmp = ailego::FileHelper::Utf8ToWide(tmpfile); std::wstring cmd = L"cmd /u /c dir /b \"" + wdir + L"\" >\"" + wtmp + L"\""; _wsystem(cmd.c_str()); std::ifstream ifs; ailego::FileHelper::OpenIfstream(ifs, tmpfile, std::ios::binary); ASSERT_TRUE(ifs.is_open()); std::string raw((std::istreambuf_iterator(ifs)), std::istreambuf_iterator()); ifs.close(); ailego::FileHelper::DeleteFile(tmpfile.c_str()); const wchar_t *wdata = reinterpret_cast(raw.data()); size_t wlen = raw.size() / sizeof(wchar_t); if (wlen > 0 && wdata[0] == L'\xFEFF') { wdata++; wlen--; } std::wstring woutput(wdata, wlen); std::string output = ailego::FileHelper::WideToUtf8(woutput); EXPECT_FALSE(output.empty()) << "dir listing inside collection is empty"; std::cout << "[NoDirGarble] Contents of " << kUtf8Dir << ":\n" << output << std::endl; } } #endif // --------------------------------------------------------------------------- // 4. MakeWalPath / MakeSegmentPath etc. produce correct UTF-8 results // --------------------------------------------------------------------------- TEST_F(Utf8CollectionTest, FileHelperPaths) { std::string wal = FileHelper::MakeWalPath(kUtf8Dir, 0, 1); std::string seg = FileHelper::MakeSegmentPath(kUtf8Dir, 0); std::string fwd = FileHelper::MakeForwardBlockPath(kUtf8Dir, 0, 1, true); std::string inv = FileHelper::MakeInvertIndexPath(kUtf8Dir, 0, 1); std::string vec = FileHelper::MakeVectorIndexPath(kUtf8Dir, "vec1", 0, 1); // All paths should start with our UTF-8 base EXPECT_EQ(0u, wal.find(kUtf8Dir)); EXPECT_EQ(0u, seg.find(kUtf8Dir)); EXPECT_EQ(0u, fwd.find(kUtf8Dir)); EXPECT_EQ(0u, inv.find(kUtf8Dir)); EXPECT_EQ(0u, vec.find(kUtf8Dir)); // Should not contain forward slash on Windows #if defined(_WIN32) || defined(_WIN64) EXPECT_EQ(std::string::npos, wal.find('/')); EXPECT_EQ(std::string::npos, seg.find('/')); EXPECT_EQ(std::string::npos, fwd.find('/')); EXPECT_EQ(std::string::npos, inv.find('/')); EXPECT_EQ(std::string::npos, vec.find('/')); #endif }