wehub/alibaba--zvec
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:47:42 +08:00
commit be3ef883e1
1214 changed files with 431743 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/01-ci-pipeline.yml
+116
View File
@@ -0,0 +1,116 @@
name: Main
on:
push:
branches: [ "main" ]
paths-ignore: &paths-ignore
- '**.md'
- '.github/CODEOWNERS'
- '.github/ISSUE_TEMPLATE/**'
- '.github/auto-assign-config.yml'
- '.github/codecov.yml'
- '.github/dependabot.yml'
- '.github/workflows/auto-assign.yml'
- '.github/workflows/community-pr-handler.yml'
- '.github/workflows/issue-auto-assign.yml'
- '.github/workflows/build_wheel.yml'
- '.github/workflows/build_test_wheel.yml'
- '.github/workflows/_build_wheel_job.yml'
- '.github/workflows/continuous_bench.yml'
- '.github/workflows/nightly_coverage.yml'
- '.github/workflows/07-linux-riscv-build.yml'
- '.github/workflows/docker/**'
- '.github/workflows/scripts/**'
merge_group:
pull_request:
branches: [ "main" ]
paths-ignore: *paths-ignore
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || '' }}-${{ github.base_ref || '' }}-${{ github.ref != 'refs/heads/main' || github.sha }}
cancel-in-progress: true
permissions:
contents: read
jobs:
# Code quality checks (fast, run first)
lint:
if: >-
github.event_name != 'pull_request' ||
(!github.event.pull_request.draft &&
!contains(github.event.pull_request.title, 'wip'))
uses: ./.github/workflows/02-lint-check.yml
# Static analysis: clang-tidy on changed C/C++ files
clang-tidy:
if: >-
github.event_name != 'pull_request' ||
(!github.event.pull_request.draft &&
!contains(github.event.pull_request.title, 'wip'))
uses: ./.github/workflows/clang_tidy.yml
# Main build and test matrix
build-and-test-macos-arm64:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (macos-arm64)
needs: [lint, clang-tidy]
uses: ./.github/workflows/03-macos-linux-build.yml
with:
platform: macos-arm64
os: macos-15
build-and-test-macos-26-arm64:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (macos-26-arm64)
needs: [lint, clang-tidy]
uses: ./.github/workflows/03-macos-linux-build.yml
with:
platform: macos-arm64
os: macos-26
build-and-test-linux-arm64:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (linux-arm64)
needs: [lint, clang-tidy]
uses: ./.github/workflows/03-macos-linux-build.yml
with:
platform: linux-arm64
os: ubuntu-24.04-arm
build-and-test-linux-x64:
name: Build & Test (linux-x64)
needs: [lint, clang-tidy]
uses: ./.github/workflows/03-macos-linux-build.yml
with:
platform: linux-x64
os: ubuntu-24.04
build-and-test-linux-x64-clang:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (linux-x64-clang)
needs: [lint, clang-tidy]
uses: ./.github/workflows/03-macos-linux-build.yml
with:
platform: linux-x64-clang
os: ubuntu-24.04
compiler: clang
build-android:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (android)
needs: [lint, clang-tidy]
uses: ./.github/workflows/04-android-build.yml
build-and-test-on-windows:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (Windows)
needs: [lint, clang-tidy]
uses: ./.github/workflows/05-windows-build.yml
build-ios:
if: github.event_name != 'push' || github.ref != 'refs/heads/main'
name: Build & Test (iOS)
needs: [lint, clang-tidy]
uses: ./.github/workflows/06-ios-build.yml
.github/workflows/02-lint-check.yml
+52
View File
@@ -0,0 +1,52 @@
name: Lint
on:
workflow_call:
jobs:
lint:
name: Code Quality Checks
runs-on: ubuntu-24.04
steps:
- name: Checkout code
uses: actions/checkout@v7
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install linting tools
run: |
python -m pip install --upgrade pip \
ruff==v0.14.4 \
clang-format==18.1.8
shell: bash
- name: Run Ruff Linter
run: python -m ruff check .
shell: bash
- name: Run Ruff Formatter Check
run: python -m ruff format --check .
shell: bash
- name: Run clang-format Check
run: |
CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \
! -path "./build/*" \
! -path "./tests/*" \
! -path "./scripts/*" \
! -path "./python/*" \
! -path "./thirdparty/*" \
! -path "./.git/*")
if [ -z "$CPP_FILES" ]; then
echo "No C++ files found to check."
exit 0
fi
clang-format --dry-run --Werror $CPP_FILES
shell: bash
.github/workflows/03-macos-linux-build.yml
+185
View File
@@ -0,0 +1,185 @@
name: MacOS & Linux Build
on:
workflow_call:
inputs:
platform:
description: 'Platform identifier'
required: true
type: string
os:
description: 'GitHub Actions runner OS'
required: true
type: string
compiler:
description: 'C++ compiler to use (gcc or clang)'
required: false
type: string
default: ''
permissions:
contents: read
jobs:
# Build and test matrix (parallel execution)
build-and-test:
name: Build & Test (${{ inputs.platform }})
runs-on: ${{ inputs.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ${{ inputs.os }}
platform: ${{ inputs.platform }}
arch_flag: "" # Use appropriate architecture
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
submodules: recursive
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: ${{ inputs.platform }}-${{ inputs.os }}-${{ inputs.compiler || 'gcc' }}
max-size: 150M
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install Clang
if: inputs.compiler == 'clang' && runner.os == 'Linux'
run: |
sudo apt-get update -y
sudo apt-get install -y clang libomp-dev
shell: bash
- name: Install AIO
if: runner.os == 'Linux' && runner.arch == 'X64'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libaio-dev
shell: bash
- name: Print CPU info
if: runner.os == 'Linux'
run: lscpu
shell: bash
- name: Set up environment variables
run: |
# Set number of processors for parallel builds
if [[ "${{ matrix.platform }}" == "macos-arm64" ]]; then
NPROC=$(sysctl -n hw.ncpu 2>/dev/null || echo 2)
else
NPROC=$(nproc 2>/dev/null || echo 2)
fi
echo "NPROC=$NPROC" >> $GITHUB_ENV
echo "Using $NPROC parallel jobs for builds"
# Set compiler when clang is requested
if [[ "${{ inputs.compiler }}" == "clang" ]]; then
echo "CC=clang" >> $GITHUB_ENV
echo "CXX=clang++" >> $GITHUB_ENV
fi
# Add Python user base bin to PATH for pip-installed CLI tools
echo "$(python -c 'import site; print(site.USER_BASE)')/bin" >> $GITHUB_PATH
shell: bash
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install \
pybind11==3.0 \
cmake==3.30.0 \
ninja==1.11.1 \
pytest \
pytest-xdist \
scikit-build-core \
setuptools_scm
shell: bash
- name: Check CMake subproject integration
if: matrix.platform == 'linux-x64'
run: |
SMOKE_SOURCE="$RUNNER_TEMP/zvec-subproject-smoke"
SMOKE_BUILD="$RUNNER_TEMP/zvec-subproject-build"
mkdir -p "$SMOKE_SOURCE" "$SMOKE_BUILD"
cp "$GITHUB_WORKSPACE/.github/cmake/subproject-integration/CMakeLists.txt" \
"$SMOKE_SOURCE/CMakeLists.txt"
cmake -S "$SMOKE_SOURCE" -B "$SMOKE_BUILD" -G Ninja \
-DZVEC_SOURCE_DIR="$GITHUB_WORKSPACE" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TOOLS=OFF \
-DBUILD_C_BINDINGS=OFF \
-DBUILD_PYTHON_BINDINGS=OFF \
-DENABLE_WERROR=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
shell: bash
- name: Build from source
run: |
cd "$GITHUB_WORKSPACE"
CMAKE_GENERATOR="Ninja" \
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
python -m pip install -v . \
--no-build-isolation \
--config-settings='cmake.define.BUILD_TOOLS=ON' \
--config-settings='cmake.define.ENABLE_WERROR=ON' \
--config-settings='cmake.define.CMAKE_C_COMPILER_LAUNCHER=ccache' \
--config-settings='cmake.define.CMAKE_CXX_COMPILER_LAUNCHER=ccache' \
${{ matrix.arch_flag }}
shell: bash
- name: Run C++ Tests
run: |
cd "$GITHUB_WORKSPACE/build"
cmake --build . --target unittest --parallel $NPROC
shell: bash
- name: Run Python Tests
run: |
cd "$GITHUB_WORKSPACE"
python -m pytest python/tests/
shell: bash
- name: Run C++ Examples
run: |
cd "$GITHUB_WORKSPACE/examples/c++"
mkdir build && cd build
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build . --parallel $NPROC
./db-example
./core-example
./ailego-example
shell: bash
- name: Run C Examples
run: |
cd "$GITHUB_WORKSPACE/examples/c"
mkdir build && cd build
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build . --parallel $NPROC
./c_api_basic_example
./c_api_collection_schema_example
./c_api_doc_example
./c_api_field_schema_example
./c_api_index_example
./c_api_optimized_example
shell: bash
.github/workflows/04-android-build.yml
+437
View File
@@ -0,0 +1,437 @@
name: Android Cross Build & Test
on:
workflow_call:
workflow_dispatch:
inputs:
api:
description: 'Android API level'
required: false
default: '34'
type: string
permissions:
contents: read
env:
NDK_VERSION: '26.1.10909125'
jobs:
build-and-test:
runs-on: ubuntu-24.04
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
abi: [x86_64]
api: ${{ github.event.inputs.api && fromJSON(format('["{0}"]', github.event.inputs.api)) || fromJSON('["34"]') }}
steps:
# ── Environment setup ──────────────────────────────────────────────
- name: Checkout
uses: actions/checkout@v7
with:
submodules: recursive
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
cmake ninja-build git ca-certificates python3 \
build-essential make unzip curl
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: android-${{ matrix.abi }}
max-size: 300M
- name: Setup Java 17
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: '17'
- name: Setup Android SDK
uses: android-actions/setup-android@v4
- name: Enable KVM
run: sudo chmod 666 /dev/kvm || true
- name: Install NDK, emulator and system image
shell: bash
run: |
sdkmanager --install \
"ndk;$NDK_VERSION" \
"platform-tools" \
"platforms;android-${{ matrix.api }}" \
"emulator"
# Install x86_64 system image (try variants in order of availability)
sdkmanager --install "system-images;android-${{ matrix.api }};google_apis;x86_64" 2>/dev/null || \
sdkmanager --install "system-images;android-${{ matrix.api }};google_apis_playstore;x86_64" 2>/dev/null || \
sdkmanager --install "system-images;android-${{ matrix.api }};default;x86_64"
# ── Step 1: build host protoc (using HOST compiler, NOT NDK) ───────
- name: Cache host protoc
uses: actions/cache@v6
with:
path: build_host
key: ${{ runner.os }}-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }}
restore-keys: |
${{ runner.os }}-host-protoc-
- name: 'Step 1: Build host protoc'
shell: bash
run: |
if [ ! -f "build_host/bin/protoc" ]; then
git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
cmake -S . -B build_host \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_TOOLCHAIN_FILE="" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-G Ninja
cmake --build build_host --target protoc --parallel
else
echo "Using cached host protoc"
fi
# ── Step 2: cross-compile zvec + tests for Android ─────────────────
- name: 'Step 2: Cross-compile zvec and tests'
shell: bash
env:
BUILD_DIR: build_android_${{ matrix.abi }}
run: |
ANDROID_NDK_HOME="$ANDROID_HOME/ndk/$NDK_VERSION"
# Reset thirdparty so the cross toolchain can patch cleanly
git submodule foreach --recursive 'git stash --include-untracked' 2>/dev/null || true
# Force reconfigure to pick up any cmake changes
rm -f "$BUILD_DIR/CMakeCache.txt"
cmake -S . -B "$BUILD_DIR" -G Ninja \
-DANDROID_NDK="$ANDROID_NDK_HOME" \
-DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake" \
-DANDROID_ABI=${{ matrix.abi }} \
-DANDROID_NATIVE_API_LEVEL=${{ matrix.api }} \
-DANDROID_STL=c++_static \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_PYTHON_BINDINGS=OFF \
-DBUILD_TOOLS=OFF \
-DENABLE_NATIVE=OFF \
-DAUTO_DETECT_ARCH=OFF \
-DENABLE_WERROR=ON \
-DCMAKE_INSTALL_PREFIX="$BUILD_DIR/install" \
-DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
echo "Building all targets..."
cmake --build "$BUILD_DIR" --parallel
# Discover test targets from ctest metadata
echo "Discovering test targets..."
TEST_NAMES=()
while IFS= read -r line; do
name=$(echo "$line" | sed -n 's/.*Test[[:space:]]*#[0-9]*:[[:space:]]*//p')
[ -n "$name" ] && TEST_NAMES+=("$name")
done < <(cd "$BUILD_DIR" && ctest --show-only 2>/dev/null || true)
# Fallback: scan ninja targets for *_test
if [ ${#TEST_NAMES[@]} -eq 0 ]; then
echo "ctest unavailable, scanning ninja targets..."
while IFS= read -r line; do
name=$(echo "$line" | sed -n 's/^\([a-zA-Z0-9_]*_test\): .*/\1/p')
[ -n "$name" ] && TEST_NAMES+=("$name")
done < <(ninja -C "$BUILD_DIR" -t targets all 2>/dev/null || true)
fi
echo "Building ${#TEST_NAMES[@]} test executables..."
ninja -C "$BUILD_DIR" -j$(nproc) "${TEST_NAMES[@]}"
# ── Step 3: start emulator ─────────────────────────────────────────
- name: 'Step 3: Start Android emulator'
shell: bash
run: |
AVD_NAME="zvec_test_avd"
AVD_DIR="$HOME/.android/avd/${AVD_NAME}.avd"
AVD_INI="$HOME/.android/avd/${AVD_NAME}.ini"
# Find installed system image (same priority as build_android.sh)
SYS_IMG=""
for variant in google_apis google_apis_playstore default; do
candidate="$ANDROID_HOME/system-images/android-${{ matrix.api }}/$variant/x86_64"
if [ -d "$candidate" ]; then
SYS_IMG="$candidate"
break
fi
done
if [ -z "$SYS_IMG" ]; then
SYS_IMG=$(find "$ANDROID_HOME/system-images" -type d -name "x86_64" 2>/dev/null | head -1)
fi
if [ -z "$SYS_IMG" ]; then
echo "ERROR: No x86_64 system image found"
exit 1
fi
echo "System image: $SYS_IMG"
# Extract tag from path (e.g. .../google_apis/x86_64 -> google_apis)
SYS_TAG=$(basename "$(dirname "$SYS_IMG")")
# Create AVD via INI files (faster and more reliable than avdmanager)
mkdir -p "$AVD_DIR"
cat > "$AVD_INI" << EOAVD
avd.ini.encoding=UTF-8
path=${AVD_DIR}
path.rel=avd/${AVD_NAME}.avd
target=android-${{ matrix.api }}
EOAVD
cat > "$AVD_DIR/config.ini" << EOCFG
AvdId=${AVD_NAME}
PlayStore.enabled=false
abi.type=x86_64
avd.ini.displayname=${AVD_NAME}
avd.ini.encoding=UTF-8
disk.dataPartition.size=8G
hw.accelerator.isConfigured=true
hw.cpu.arch=x86_64
hw.cpu.ncore=4
hw.lcd.density=420
hw.lcd.height=1920
hw.lcd.width=1080
hw.ramSize=4096
image.sysdir.1=${SYS_IMG}/
tag.display=${SYS_TAG}
tag.id=${SYS_TAG}
EOCFG
echo "Created AVD: $AVD_NAME"
# Launch emulator in background
$ANDROID_HOME/emulator/emulator -avd "$AVD_NAME" \
-no-window -no-audio -no-boot-anim \
-gpu swiftshader_indirect \
-netdelay none -netspeed full \
-memory 4096 \
-no-snapshot \
-wipe-data &
echo "EMULATOR_PID=$!" >> "$GITHUB_ENV"
# Wait for device to be reachable
adb wait-for-device
# Poll for boot completion with timeout
echo "Waiting for boot to complete..."
TIMEOUT=300
ELAPSED=0
while true; do
BOOTED=$(adb shell getprop sys.boot_completed 2>/dev/null | tr -d '\r\n ' || true)
if [ "$BOOTED" = "1" ]; then
echo "Emulator ready (took ${ELAPSED}s)"
break
fi
if [ $ELAPSED -ge $TIMEOUT ]; then
echo "ERROR: Emulator failed to boot within ${TIMEOUT}s"
exit 1
fi
sleep 3
ELAPSED=$((ELAPSED + 3))
done
echo "Device ABI: $(adb shell getprop ro.product.cpu.abi | tr -d '\r')"
echo "ABI list : $(adb shell getprop ro.product.cpu.abilist | tr -d '\r')"
# ── Step 4: run unit tests on emulator ─────────────────────────────
- name: 'Step 4: Run unit tests on emulator'
shell: bash
env:
BUILD_DIR: build_android_${{ matrix.abi }}
run: |
DEVICE_TEST_DIR="/data/local/tmp/zvec_tests"
DEVICE_LIB_DIR="$DEVICE_TEST_DIR/lib"
adb shell "mkdir -p $DEVICE_TEST_DIR $DEVICE_LIB_DIR"
# Push shared libraries
echo "Pushing shared libraries..."
SO_COUNT=0
while IFS= read -r so_file; do
adb push "$so_file" "$DEVICE_LIB_DIR/$(basename "$so_file")" > /dev/null 2>&1
SO_COUNT=$((SO_COUNT + 1))
done < <(find "$BUILD_DIR/lib" -name "*.so" -type f 2>/dev/null)
echo "Pushed $SO_COUNT shared libraries"
# Push helper binaries (needed by crash_recovery tests which fork+exec them)
echo "Pushing helper binaries..."
for helper_name in data_generator collection_optimizer; do
helper_path=$(find "$BUILD_DIR" -name "$helper_name" -type f -executable ! -name "*_test" 2>/dev/null | head -1)
if [ -n "$helper_path" ]; then
adb push "$helper_path" "$DEVICE_TEST_DIR/$helper_name" > /dev/null 2>&1
adb shell "chmod 755 $DEVICE_TEST_DIR/$helper_name"
echo " Pushed $helper_name"
fi
done
# Discover test targets from ctest metadata
TEST_NAMES=()
while IFS= read -r line; do
name=$(echo "$line" | sed -n 's/.*Test[[:space:]]*#[0-9]*:[[:space:]]*//p')
[ -n "$name" ] && TEST_NAMES+=("$name")
done < <(cd "$BUILD_DIR" && ctest --show-only 2>/dev/null || true)
if [ ${#TEST_NAMES[@]} -eq 0 ]; then
while IFS= read -r line; do
name=$(echo "$line" | sed -n 's/^\([a-zA-Z0-9_]*_test\): .*/\1/p')
[ -n "$name" ] && TEST_NAMES+=("$name")
done < <(ninja -C "$BUILD_DIR" -t targets all 2>/dev/null || true)
fi
# Collect test binaries
TEST_BINS=()
for name in "${TEST_NAMES[@]}"; do
bin_path=$(find "$BUILD_DIR" -name "$name" -type f -executable 2>/dev/null | head -1)
if [ -n "$bin_path" ]; then
TEST_BINS+=("$bin_path")
else
echo "WARNING: binary not found for '$name'"
fi
done
TOTAL=${#TEST_BINS[@]}
PASSED=0
FAILED=0
FAILED_NAMES=()
IDX=0
echo "Running $TOTAL unit tests on emulator..."
for test_bin in "${TEST_BINS[@]}"; do
IDX=$((IDX + 1))
test_name=$(basename "$test_bin")
device_path="$DEVICE_TEST_DIR/$test_name"
# Give each test its own working directory to avoid name collisions
WORK_DIR="$DEVICE_TEST_DIR/workdir_${test_name}"
echo ""
echo "────────────────────────────────────────"
echo " [$IDX/$TOTAL] $test_name"
echo "────────────────────────────────────────"
set +e
# Create isolated working directory
adb shell "mkdir -p $WORK_DIR" 2>/dev/null
# Copy helper binaries into working directory so crash_recovery tests
# (which fork+exec data_generator / collection_optimizer) can find them
adb shell "for h in $DEVICE_TEST_DIR/data_generator $DEVICE_TEST_DIR/collection_optimizer; do [ -f \$h ] && cp \$h $WORK_DIR/; done" 2>/dev/null
# Push test binary
adb push "$test_bin" "$device_path" > /dev/null 2>&1
adb shell "chmod 755 $device_path" 2>/dev/null
# Run test from its own working directory with LD_LIBRARY_PATH
OUTPUT=$(adb shell "cd $WORK_DIR && LD_LIBRARY_PATH=$DEVICE_LIB_DIR $device_path 2>&1; echo EXIT_CODE=\$?" 2>&1)
# Extract exit code from the output
EXIT_CODE=$(echo "$OUTPUT" | grep -o 'EXIT_CODE=[0-9]*' | tail -1 | cut -d= -f2)
set -e
# Print test output (without the EXIT_CODE marker)
echo "$OUTPUT" | grep -v 'EXIT_CODE=' | sed 's/^/ /' || true
if [ "$EXIT_CODE" = "0" ]; then
echo " >>> PASSED"
PASSED=$((PASSED + 1))
else
# Detect "crash-on-exit" pattern: all gtest assertions passed but
# process crashed during static destructor teardown (common with c++_static STL)
GTEST_PASSED_LINE=$(echo "$OUTPUT" | grep '\[ PASSED \]' | tail -1 || true)
GTEST_FAILED_LINE=$(echo "$OUTPUT" | grep '\[ FAILED \]' | head -1 || true)
if [ -n "$GTEST_PASSED_LINE" ] && [ -z "$GTEST_FAILED_LINE" ] && \
{ [ "$EXIT_CODE" = "139" ] || [ "$EXIT_CODE" = "134" ] || [ "$EXIT_CODE" = "135" ]; }; then
echo " >>> PASSED (crash-on-exit ignored, exit=$EXIT_CODE)"
PASSED=$((PASSED + 1))
else
echo " >>> FAILED (exit=$EXIT_CODE)"
FAILED=$((FAILED + 1))
FAILED_NAMES+=("$test_name")
fi
fi
# Clean up binary and working directory to reclaim disk space
adb shell "rm -rf $device_path $WORK_DIR" 2>/dev/null || true
done
echo ""
echo "============================================================"
echo " Test Summary"
echo "============================================================"
echo " Total : $TOTAL"
echo " Passed : $PASSED"
echo " Failed : $FAILED"
if [ $FAILED -gt 0 ]; then
echo ""
echo " Failed tests:"
for name in "${FAILED_NAMES[@]}"; do
echo " - $name"
done
fi
echo "============================================================"
if [ $FAILED -gt 0 ]; then
exit 1
fi
echo "All tests passed!"
# ── Step 5: build and run examples ─────────────────────────────────
- name: 'Step 5: Build and run examples'
shell: bash
env:
BUILD_DIR: build_android_${{ matrix.abi }}
run: |
ANDROID_NDK_HOME="$ANDROID_HOME/ndk/$NDK_VERSION"
EXAMPLES_BUILD="examples/c++/build-android-examples-${{ matrix.abi }}"
cmake -S examples/c++ -B "$EXAMPLES_BUILD" -G Ninja \
-DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake" \
-DANDROID_ABI=${{ matrix.abi }} \
-DANDROID_PLATFORM=android-${{ matrix.api }} \
-DANDROID_STL=c++_static \
-DCMAKE_BUILD_TYPE=Release \
-DHOST_BUILD_DIR="$BUILD_DIR" \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build "$EXAMPLES_BUILD" --parallel
# Reuse the shared-library directory from Step 4; push again in
# case Step 4 was skipped or the directory was cleaned.
DEVICE_LIB_DIR="/data/local/tmp/zvec_tests/lib"
adb shell "mkdir -p $DEVICE_LIB_DIR" 2>/dev/null || true
SO_COUNT=0
while IFS= read -r so_file; do
adb push "$so_file" "$DEVICE_LIB_DIR/$(basename "$so_file")" > /dev/null 2>&1
SO_COUNT=$((SO_COUNT + 1))
done < <(find "$BUILD_DIR/lib" -name "*.so" -type f 2>/dev/null)
echo "Pushed $SO_COUNT shared libraries to $DEVICE_LIB_DIR"
for example in ailego-example core-example db-example; do
if [ -f "$EXAMPLES_BUILD/$example" ]; then
echo "=== Running $example ==="
adb push "$EXAMPLES_BUILD/$example" "/data/local/tmp/$example" > /dev/null 2>&1
adb shell "chmod 755 /data/local/tmp/$example && cd /data/local/tmp && LD_LIBRARY_PATH=$DEVICE_LIB_DIR ./$example"
adb shell "rm -f /data/local/tmp/$example"
fi
done
# ── Cleanup ────────────────────────────────────────────────────────
- name: Stop emulator
if: always()
shell: bash
run: |
adb emu kill 2>/dev/null || true
sleep 2
if [ -n "$EMULATOR_PID" ]; then
kill "$EMULATOR_PID" 2>/dev/null || true
fi
.github/workflows/05-windows-build.yml
+157
View File
@@ -0,0 +1,157 @@
name: Windows Build
on:
workflow_call:
workflow_dispatch:
permissions:
contents: read
jobs:
# Windows build and test matrix
build-and-test-windows:
name: Build & Test (${{ matrix.platform }})
runs-on: ${{ matrix.platform }}
strategy:
fail-fast: false
matrix:
include:
- platform: windows-2022
- platform: windows-2025
env:
SCCACHE_GHA_ENABLED: "true"
steps:
- name: Show env info
run: |
Get-CimInstance -ClassName Win32_Processor
where cl
& "C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe" -all -products * -prerelease -format json
shell: powershell
- name: Checkout code
uses: actions/checkout@v7
with:
submodules: recursive
- name: Cleanup left marker files
run: |
git submodule foreach --recursive 'git reset --hard && git clean -ffdx'
shell: powershell
- name: Setup sccache
uses: mozilla-actions/sccache-action@v0.0.10
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Set up MSVC environment
uses: ilammy/msvc-dev-cmd@v1.13.0
with:
arch: x64
- name: Set up environment variables
run: |
$nproc = (Get-CimInstance Win32_ComputerSystem).NumberOfLogicalProcessors
echo "NPROC=$nproc" >> $env:GITHUB_ENV
echo "Using $nproc parallel jobs for builds"
shell: powershell
- name: Install dependencies
run: |
python -m pip install --upgrade pip `
pybind11==3.0 `
cmake==3.30.0 `
ninja==1.11.1 `
pytest `
pytest-xdist `
scikit-build-core `
setuptools_scm
shell: powershell
- name: Build from source
run: |
cd "$env:GITHUB_WORKSPACE"
$env:CMAKE_GENERATOR = "Ninja"
$env:CMAKE_BUILD_PARALLEL_LEVEL = "$env:NPROC"
python -m pip install -v . `
--no-build-isolation `
--config-settings='cmake.define.BUILD_TOOLS=ON' `
--config-settings='cmake.define.ENABLE_WERROR=ON' `
--config-settings='cmake.define.CMAKE_C_COMPILER_LAUNCHER=sccache' `
--config-settings='cmake.define.CMAKE_CXX_COMPILER_LAUNCHER=sccache'
shell: powershell
- name: Show sccache statistics after pip install
if: always()
run: sccache --show-stats
shell: powershell
- name: Run C++ Tests
run: |
cd "$env:GITHUB_WORKSPACE\build"
cmake --build . --target unittest --config Release --parallel $env:NPROC
shell: powershell
- name: Show sccache statistics after tests
if: always()
run: sccache --show-stats
shell: powershell
- name: Run Python Tests
run: |
cd "$env:GITHUB_WORKSPACE"
python -m pytest python/tests/ --basetemp=./.pytest_tmp
shell: powershell
- name: Run C++ Examples
run: |
cd "$env:GITHUB_WORKSPACE\examples\c++"
mkdir build
cd build
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release `
-DCMAKE_C_COMPILER_LAUNCHER=sccache `
-DCMAKE_CXX_COMPILER_LAUNCHER=sccache
cmake --build . --config Release --parallel $env:NPROC
# Copy zvec DLLs next to the example executables so Windows can find them.
# CMake places DLLs in bin/ (RUNTIME output) and import libs in lib/.
$buildDir = "$env:GITHUB_WORKSPACE\build"
foreach ($dllName in @("zvec.dll", "zvec_core.dll", "zvec_ailego.dll")) {
$found = $false
foreach ($sub in @("$buildDir\bin", "$buildDir\bin\Release", "$buildDir\lib", "$buildDir\lib\Release")) {
$dllPath = Join-Path $sub $dllName
if (Test-Path $dllPath) {
Copy-Item $dllPath -Destination . -Force
Write-Host "Copied $dllName from $sub"
$found = $true
break
}
}
if (-not $found) {
Write-Host "WARNING: $dllName not found, searching recursively..."
$dll = Get-ChildItem -Path $buildDir -Filter $dllName -Recurse -ErrorAction SilentlyContinue | Select-Object -First 1
if ($dll) {
Copy-Item $dll.FullName -Destination . -Force
Write-Host "Copied $dllName from $($dll.DirectoryName)"
} else {
Write-Error "$dllName not found anywhere under $buildDir"
}
}
}
.\db-example.exe
.\core-example.exe
.\ailego-example.exe
shell: powershell
- name: Show sccache statistics
if: always()
run: sccache --show-stats
shell: powershell
.github/workflows/06-ios-build.yml
+163
View File
@@ -0,0 +1,163 @@
name: iOS Cross Build
on:
workflow_call:
workflow_dispatch:
permissions:
contents: read
jobs:
build-ios:
runs-on: macos-15
strategy:
fail-fast: false
matrix:
include:
- platform: SIMULATORARM64
arch: arm64
sdk: iphonesimulator
test_on_simulator: true
- platform: OS
arch: arm64
sdk: iphoneos
test_on_simulator: false
name: iOS (${{ matrix.platform }})
steps:
- name: Checkout
uses: actions/checkout@v7
with:
submodules: recursive
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: ios-${{ matrix.platform }}
max-size: 150M
- name: Cache host protoc build
uses: actions/cache@v6
with:
path: build_host
key: macos-host-protoc-${{ hashFiles('thirdparty/protobuf/**', 'CMakeLists.txt') }}
restore-keys: |
macos-host-protoc-
- name: Build host protoc
run: |
if [ ! -f "build_host/bin/protoc" ]; then
cmake -S . -B build_host -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build build_host --target protoc --parallel $(sysctl -n hw.ncpu)
else
echo "Using cached host protoc"
fi
- name: Configure and Build
run: |
git submodule foreach --recursive 'git stash --include-untracked' || true
SDK_PATH=$(xcrun --sdk ${{ matrix.sdk }} --show-sdk-path)
NPROC=$(sysctl -n hw.ncpu)
cmake -S . -B build_ios_${{ matrix.platform }} \
-DCMAKE_SYSTEM_NAME=iOS \
-DCMAKE_OSX_DEPLOYMENT_TARGET="13.0" \
-DCMAKE_OSX_ARCHITECTURES="${{ matrix.arch }}" \
-DCMAKE_OSX_SYSROOT="$SDK_PATH" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_PYTHON_BINDINGS=OFF \
-DBUILD_TOOLS=OFF \
-DENABLE_WERROR=ON \
-DCMAKE_INSTALL_PREFIX="./install" \
-DGLOBAL_CC_PROTOBUF_PROTOC="$GITHUB_WORKSPACE/build_host/bin/protoc" \
-DIOS=ON \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
cmake --build build_ios_${{ matrix.platform }} --parallel $NPROC
- name: Build test targets
if: matrix.test_on_simulator
run: |
NPROC=$(sysctl -n hw.ncpu)
cmake --build build_ios_${{ matrix.platform }} --target unittest --parallel $NPROC
- name: Boot iOS Simulator
if: matrix.test_on_simulator
run: |
DEVICE_ID=$(xcrun simctl list devices available -j \
| python3 -c "
import json, sys
data = json.load(sys.stdin)
for runtime, devices in data['devices'].items():
if 'iOS' in runtime:
for d in devices:
if 'iPhone' in d['name'] and d['isAvailable']:
print(d['udid'])
sys.exit(0)
sys.exit(1)
")
echo "DEVICE_ID=$DEVICE_ID" >> $GITHUB_ENV
xcrun simctl boot "$DEVICE_ID"
echo "Booted simulator: $DEVICE_ID"
- name: Run all tests on simulator
if: matrix.test_on_simulator
run: |
FAILED_TESTS=""
PASSED=0
TOTAL=0
for APP in build_ios_${{ matrix.platform }}/bin/*_test.app; do
[ -d "$APP" ] || continue
TEST_NAME=$(basename "$APP" .app)
BUNDLE_ID="com.zvec.${TEST_NAME}"
TOTAL=$((TOTAL + 1))
echo "::group::Running ${TEST_NAME}"
xcrun simctl install "$DEVICE_ID" "$APP"
set +eo pipefail
for attempt in 1 2 3; do
xcrun simctl launch --console "$DEVICE_ID" "$BUNDLE_ID" 2>&1 | tee /tmp/${TEST_NAME}.log
LAUNCH_EXIT=${PIPESTATUS[0]}
if ! grep -q "unknown to FrontBoard" /tmp/${TEST_NAME}.log; then
break
fi
echo "::warning::Attempt ${attempt}/3: FrontBoard has not registered ${TEST_NAME} yet, retrying in 3s..."
sleep 3
done
set -eo pipefail
if grep -q '\[ FAILED \]' /tmp/${TEST_NAME}.log; then
echo "::error::${TEST_NAME} has failing tests"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
elif grep -q '\[ PASSED \]' /tmp/${TEST_NAME}.log; then
PASSED=$((PASSED + 1))
elif grep -qE 'Failed: 0$' /tmp/${TEST_NAME}.log; then
# c_api_test uses a custom test framework (not GTest)
PASSED=$((PASSED + 1))
elif [ "$LAUNCH_EXIT" -eq 0 ]; then
echo "::warning::${TEST_NAME} exited 0 but produced no recognisable test summary"
PASSED=$((PASSED + 1))
else
echo "::error::${TEST_NAME} exited ${LAUNCH_EXIT} with no test summary"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
fi
echo "::endgroup::"
done
echo "Test summary: ${PASSED}/${TOTAL} passed"
if [ -n "$FAILED_TESTS" ]; then
echo "::error::Failed tests:${FAILED_TESTS}"
exit 1
fi
- name: Shutdown Simulator
if: matrix.test_on_simulator && always()
run: |
xcrun simctl shutdown "$DEVICE_ID" || true
.github/workflows/07-linux-riscv-build.yml
+342
View File
@@ -0,0 +1,342 @@
name: Weekly Linux RISC-V Build
on:
schedule:
# Saturday 00:00 UTC+8 (Friday 16:00 UTC)
- cron: '0 16 * * 5'
workflow_dispatch:
permissions:
contents: read
env:
RISE_PYPI: https://gitlab.com/api/v4/projects/56254198/packages/pypi/simple
PIP_BREAK_SYSTEM_PACKAGES: 1
jobs:
build:
name: Build (linux-riscv64)
runs-on: ubuntu-24.04-riscv
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
submodules: recursive
- name: Install build dependencies
run: |
sudo mkdir -p /var/lib/dpkg/updates
sudo mkdir -p /var/lib/apt/lists/
sudo mkdir -p /var/cache/apt/archives/
sudo touch /var/lib/dpkg/status
sudo apt-get purge -y byobu || true
sudo apt-get update -o Dpkg::Lock::Timeout=300
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
-o Dpkg::Lock::Timeout=300 \
ccache python3-pybind11 pybind11-dev
shell: bash
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: linux-riscv64-${{ runner.os }}-gcc
max-size: 150M
- name: Build from source
run: |
cd "$GITHUB_WORKSPACE"
NPROC=$(nproc 2>/dev/null || echo 2)
echo "Using $NPROC parallel jobs for builds"
cmake -S . -B build \
-G "Unix Makefiles" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TOOLS=ON \
-DBUILD_PYTHON_BINDINGS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
make -C build -j"$NPROC"
shell: bash
- name: Archive entire workspace
run: |
cd "$GITHUB_WORKSPACE"
tar -cf linux-riscv64-workspace.tar .
shell: bash
- name: Upload workspace artifacts
uses: actions/upload-artifact@v7
with:
name: linux-riscv64-workspace
path: ${{ github.workspace }}/linux-riscv64-workspace.tar
if-no-files-found: error
cpp-tests:
name: C++ Tests
runs-on: ubuntu-24.04-riscv
needs: build
steps:
- name: Download workspace artifacts
uses: actions/download-artifact@v8
with:
name: linux-riscv64-workspace
path: ${{ github.workspace }}
- name: Extract workspace
run: |
cd "$GITHUB_WORKSPACE"
tar -xf linux-riscv64-workspace.tar
shell: bash
- name: Install test dependencies
run: |
sudo mkdir -p /var/lib/dpkg/updates
sudo mkdir -p /var/lib/apt/lists/
sudo mkdir -p /var/cache/apt/archives/
sudo touch /var/lib/dpkg/status
sudo apt-get purge -y byobu || true
sudo apt-get update -o Dpkg::Lock::Timeout=300
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
-o Dpkg::Lock::Timeout=300 \
ccache python3-pybind11 pybind11-dev libgtest-dev liburing-dev
shell: bash
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: linux-riscv64-${{ runner.os }}-gcc
max-size: 150M
- name: Reconfigure build directory
run: |
cd "$GITHUB_WORKSPACE"
cmake -S . -B build \
-G "Unix Makefiles" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TOOLS=ON \
-DBUILD_PYTHON_BINDINGS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
shell: bash
- name: Run C++ Tests
run: |
cd "$GITHUB_WORKSPACE/build"
NPROC=$(nproc 2>/dev/null || echo 2)
make unittest -j"$NPROC"
shell: bash
python-tests:
name: Python Tests
runs-on: ubuntu-24.04-riscv
needs: build
steps:
- name: Select Python
run: |
if command -v python3 >/dev/null 2>&1; then
PYTHON_BIN=python3
elif command -v python >/dev/null 2>&1; then
PYTHON_BIN=python
else
echo "No local Python interpreter found on PATH"
exit 1
fi
"$PYTHON_BIN" --version
echo "PYTHON=$PYTHON_BIN" >> "$GITHUB_ENV"
shell: bash
- name: Download workspace artifacts
uses: actions/download-artifact@v8
with:
name: linux-riscv64-workspace
path: ${{ github.workspace }}
- name: Extract workspace
run: |
cd "$GITHUB_WORKSPACE"
tar -xf linux-riscv64-workspace.tar
echo "$($PYTHON -c 'import site; print(site.USER_BASE)')/bin" >> "$GITHUB_PATH"
shell: bash
- name: Cache pip
uses: actions/cache@v6
with:
path: ~/.cache/pip
key: linux-riscv64-${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }}
restore-keys: |
linux-riscv64-${{ runner.os }}-pip-
- name: Install dependencies
run: |
sudo mkdir -p /var/lib/dpkg/updates
sudo mkdir -p /var/lib/apt/lists/
sudo mkdir -p /var/cache/apt/archives/
sudo touch /var/lib/dpkg/status
sudo apt-get purge -y byobu || true
sudo apt-get update -o Dpkg::Lock::Timeout=300
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
-o Dpkg::Lock::Timeout=300 \
ccache libgtest-dev liburing-dev
$PYTHON -m pip install --upgrade pip
$PYTHON -m pip install numpy==2.2.2 cmake==3.30.0 ninja==1.11.1.1 --index-url "$RISE_PYPI"
$PYTHON -m pip install pybind11==3.0 pytest scikit-build-core setuptools_scm pytest-xdist
shell: bash
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: linux-riscv64-${{ runner.os }}-gcc
max-size: 150M
- name: Reconfigure build directory
run: |
cd "$GITHUB_WORKSPACE"
cmake -S . -B build \
-G "Unix Makefiles" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_TOOLS=ON \
-DBUILD_PYTHON_BINDINGS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-Dpybind11_DIR="$($PYTHON -c 'import pybind11; print(pybind11.get_cmake_dir())')"
shell: bash
- name: Install from existing build directory
run: |
cd "$GITHUB_WORKSPACE"
NPROC=$(nproc 2>/dev/null || echo 2)
export SKBUILD_BUILD_DIR="$GITHUB_WORKSPACE/build"
CMAKE_GENERATOR="Unix Makefiles" \
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
$PYTHON -m pip install -v . \
--no-build-isolation \
--config-settings='cmake.define.BUILD_TOOLS="ON"' \
--config-settings='cmake.define.ENABLE_WERROR=ON' \
--config-settings='cmake.define.CMAKE_C_COMPILER_LAUNCHER=ccache' \
--config-settings='cmake.define.CMAKE_CXX_COMPILER_LAUNCHER=ccache'
shell: bash
- name: Run Python Tests
run: |
cd "$GITHUB_WORKSPACE"
$PYTHON -m pytest python/tests/
shell: bash
cpp-examples:
name: C++ Examples
runs-on: ubuntu-24.04-riscv
needs: build
steps:
- name: Download workspace artifacts
uses: actions/download-artifact@v8
with:
name: linux-riscv64-workspace
path: ${{ github.workspace }}
- name: Extract workspace
run: |
cd "$GITHUB_WORKSPACE"
tar -xf linux-riscv64-workspace.tar
shell: bash
- name: Install ccache
run: |
sudo mkdir -p /var/lib/dpkg/updates
sudo mkdir -p /var/lib/apt/lists/
sudo mkdir -p /var/cache/apt/archives/
sudo touch /var/lib/dpkg/status
sudo apt-get purge -y byobu || true
sudo apt-get update -o Dpkg::Lock::Timeout=300
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
-o Dpkg::Lock::Timeout=300 \
ccache
shell: bash
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: linux-riscv64-${{ runner.os }}-gcc
max-size: 150M
- name: Run C++ Examples
run: |
cd "$GITHUB_WORKSPACE/examples/c++"
NPROC=$(nproc 2>/dev/null || echo 2)
mkdir -p build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
make -j "$NPROC"
./db-example
./core-example
./ailego-example
shell: bash
c-examples:
name: C Examples
runs-on: ubuntu-24.04-riscv
needs: build
steps:
- name: Download workspace artifacts
uses: actions/download-artifact@v8
with:
name: linux-riscv64-workspace
path: ${{ github.workspace }}
- name: Extract workspace
run: |
cd "$GITHUB_WORKSPACE"
tar -xf linux-riscv64-workspace.tar
shell: bash
- name: Install ccache
run: |
sudo mkdir -p /var/lib/dpkg/updates
sudo mkdir -p /var/lib/apt/lists/
sudo mkdir -p /var/cache/apt/archives/
sudo touch /var/lib/dpkg/status
sudo apt-get purge -y byobu || true
sudo apt-get update -o Dpkg::Lock::Timeout=300
sudo DEBIAN_FRONTEND=noninteractive apt-get install -yq \
-o Dpkg::Options::="--force-confdef" \
-o Dpkg::Options::="--force-confold" \
-o Dpkg::Lock::Timeout=300 \
ccache
shell: bash
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: linux-riscv64-${{ runner.os }}-gcc
max-size: 150M
- name: Run C Examples
run: |
cd "$GITHUB_WORKSPACE/examples/c"
NPROC=$(nproc 2>/dev/null || echo 2)
mkdir -p build && cd build
cmake .. -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
make -j "$NPROC"
./c_api_basic_example
./c_api_collection_schema_example
./c_api_doc_example
./c_api_field_schema_example
./c_api_index_example
./c_api_optimized_example
shell: bash
.github/workflows/08-cmake-subproject-integration.yml
+102
View File
@@ -0,0 +1,102 @@
name: Nightly CMake Subproject Integration
on:
schedule:
# Run after the other nightly jobs to avoid starting every heavy job at once.
- cron: '30 16 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
subproject-integration:
name: Subproject Build & Example
runs-on: ubuntu-24.04
timeout-minutes: 90
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
submodules: recursive
- name: Setup ccache
uses: hendrikmuhs/ccache-action@v1.2
with:
key: cmake-subproject-linux-x64
max-size: 150M
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libaio-dev
shell: bash
- name: Set up environment variables
run: |
NPROC=$(nproc 2>/dev/null || echo 2)
echo "NPROC=$NPROC" >> "$GITHUB_ENV"
echo "$(python -c 'import site; print(site.USER_BASE)')/bin" >> "$GITHUB_PATH"
shell: bash
- name: Install dependencies
run: |
python -m pip install --upgrade pip
python -m pip install \
cmake==3.30.0 \
ninja==1.11.1
shell: bash
- name: Configure subproject integration
run: |
SMOKE_SOURCE="$RUNNER_TEMP/zvec-subproject-nightly"
SMOKE_BUILD="$RUNNER_TEMP/zvec-subproject-nightly-build"
mkdir -p "$SMOKE_SOURCE" "$SMOKE_BUILD"
cp "$GITHUB_WORKSPACE/.github/cmake/subproject-integration/CMakeLists.txt" \
"$SMOKE_SOURCE/CMakeLists.txt"
CMAKE_ARGS=(
-DZVEC_SOURCE_DIR="$GITHUB_WORKSPACE"
-DCMAKE_BUILD_TYPE=Release
-DENABLE_WERROR=ON
-DUSE_OSS_MIRROR=ON
-DCMAKE_C_COMPILER_LAUNCHER=ccache
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-DBUILD_TOOLS=OFF
-DBUILD_C_BINDINGS=OFF
-DBUILD_PYTHON_BINDINGS=OFF
)
cmake -S "$SMOKE_SOURCE" -B "$SMOKE_BUILD" -G Ninja \
"${CMAKE_ARGS[@]}"
echo "SMOKE_BUILD=$SMOKE_BUILD" >> "$GITHUB_ENV"
shell: bash
- name: Build subproject targets
run: |
cmake --build "$SMOKE_BUILD" --target \
core_knn_diskann \
zvec_embed_db_example \
zvec_embed_core_example \
zvec_embed_ailego_example \
--parallel "$NPROC"
shell: bash
- name: Run subproject examples
run: |
cd "$SMOKE_BUILD"
./bin/zvec_embed_db_example
./bin/zvec_embed_core_example
./bin/zvec_embed_ailego_example
shell: bash
.github/workflows/_build_wheel_job.yml
+104
View File
@@ -0,0 +1,104 @@
name: "(Reusable) Build, Publish and Smoke-test a Wheel"
on:
workflow_call:
inputs:
runner:
description: "GitHub Actions runner label"
required: true
type: string
pypi_repository_url:
description: "PyPI repository URL (empty string means official PyPI)"
required: false
type: string
default: ""
secrets:
PYPI_API_TOKEN:
required: true
jobs:
build_publish_test:
name: Build / publish / smoke-test on ${{ inputs.runner }}
runs-on: ${{ inputs.runner }}
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
submodules: recursive
fetch-tags: true
fetch-depth: 0
- name: Set up Python (for cibuildwheel controller)
uses: actions/setup-python@v6
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
pip install --upgrade pip
pip install cibuildwheel==3.4.0
- name: Build wheels using cibuildwheel
run: |
python -m cibuildwheel --output-dir wheelhouse
# Save list of built wheels for publishing
ls wheelhouse/*.whl | tee $GITHUB_STEP_SUMMARY
echo "wheels=$(ls wheelhouse/*.whl | tr '\n' ' ')" >> $GITHUB_ENV
- name: Publish to PyPI
if: success() && github.event_name == 'workflow_dispatch'
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
TWINE_REPOSITORY_URL: ${{ inputs.pypi_repository_url }}
run: |
pip install twine
twine upload --skip-existing --verbose wheelhouse/*.whl
- name: Smoke test from PyPI
if: success() && github.event_name == 'workflow_dispatch'
shell: bash
env:
PYPI_REPOSITORY_URL: ${{ inputs.pypi_repository_url }}
run: |
# Extract version from wheel filename (e.g. zvec-0.2.1.dev24-cp311-...whl -> 0.2.1.dev24)
WHEEL_FILE=$(ls wheelhouse/zvec-*.whl | head -1)
ZVEC_VERSION=$(basename "$WHEEL_FILE" | sed 's/zvec-\([^-]*\)-.*/\1/')
# Build index-url flags: use TestPyPI when repository URL is set, otherwise official PyPI
if [ -n "$PYPI_REPOSITORY_URL" ]; then
INDEX_FLAGS="--index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/"
echo "Waiting for zvec==$ZVEC_VERSION to become available on TestPyPI..."
else
INDEX_FLAGS=""
echo "Waiting for zvec==$ZVEC_VERSION to become available on PyPI..."
fi
# Poll until the version is available (max 5 minutes)
FOUND=0
for i in $(seq 1 30); do
if pip install $INDEX_FLAGS --dry-run "zvec==$ZVEC_VERSION" > /dev/null 2>&1; then
echo "Version $ZVEC_VERSION is available."
FOUND=1
break
fi
echo "Attempt $i/30: not yet available, retrying in 10s..."
sleep 10
done
if [ "$FOUND" -eq 0 ]; then
echo "ERROR: Timed out (5 min) waiting for zvec==$ZVEC_VERSION on PyPI. Aborting smoke test."
exit 1
fi
# Create a clean venv and install
python -m venv test_env
source test_env/bin/activate
pip install --upgrade pip
pip install $INDEX_FLAGS "zvec==$ZVEC_VERSION"
pip install --upgrade pip
pip install $INDEX_FLAGS "zvec==$ZVEC_VERSION"
# Run a simple smoke test
python -c "import zvec; print('Import OK:', zvec.__version__)"
.github/workflows/auto-assign.yml
+25
View File
@@ -0,0 +1,25 @@
name: Auto Assign PR
on:
pull_request_target:
types: [opened, reopened, ready_for_review]
permissions:
contents: read
pull-requests: write
jobs:
auto-assign:
runs-on: ubuntu-latest
steps:
- name: Checkout for config
uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.base.ref }}
sparse-checkout: .github/auto-assign-config.yml
sparse-checkout-cone-mode: false
- name: Auto Assign PR Author
uses: kentaro-m/auto-assign-action@v2.0.2
with:
configuration-path: '.github/auto-assign-config.yml'
.github/workflows/clang_tidy.yml
+197
View File
@@ -0,0 +1,197 @@
name: Clang-Tidy
on:
workflow_call:
workflow_dispatch:
permissions:
contents: read
jobs:
clang_tidy:
name: Clang-Tidy Checks
runs-on: ubuntu-24.04
steps:
- name: Shallow checkout (no submodules)
uses: actions/checkout@v7
with:
submodules: false
fetch-depth: 0
- name: Collect changed C/C++ files
id: changed_files
uses: tj-actions/changed-files@v47
with:
files: |
**/*.c
**/*.cc
**/*.cpp
**/*.cxx
**/*.h
**/*.hpp
files_ignore: |
thirdparty/**
build/**
src/db/sqlengine/antlr/gen/**
src/db/index/column/fts_column/gen/**
- name: No C/C++ files changed - skip
if: steps.changed_files.outputs.any_changed != 'true'
run: echo "No C/C++ files changed. Skipping clang-tidy."
- name: Checkout submodules
if: steps.changed_files.outputs.any_changed == 'true'
run: git submodule update --init --recursive --depth 1
- name: Install dependencies
if: steps.changed_files.outputs.any_changed == 'true'
run: |
sudo apt-get update
sudo apt-get install -y clang-tidy=1:18.0-59~exp2 cmake ninja-build libomp-dev libaio-dev
- name: Setup ccache
if: steps.changed_files.outputs.any_changed == 'true'
uses: hendrikmuhs/ccache-action@v1.2
with:
key: clang-tidy
max-size: 500M
- name: Configure CMake and export compile commands
if: steps.changed_files.outputs.any_changed == 'true'
run: |
cmake -S . -B build -G Ninja \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DBUILD_TOOLS=ON \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache
- name: Filter changed files against compile_commands.json
id: tidy_files
if: steps.changed_files.outputs.any_changed == 'true'
run: |
python3 - <<'PY'
import json
import os
from pathlib import Path
raw = os.environ.get("ALL_CHANGED_FILES", "")
changed = [f for f in raw.split() if f]
compile_db_path = Path("build/compile_commands.json")
with compile_db_path.open("r", encoding="utf-8") as fh:
compile_db = json.load(fh)
compile_entries = set()
for entry in compile_db:
file_field = entry.get("file", "")
if file_field:
compile_entries.add(os.path.normpath(file_field))
cwd = Path.cwd().resolve()
selected = []
skipped = []
for rel_path in changed:
abs_path = os.path.normpath(str((cwd / rel_path).resolve()))
if abs_path in compile_entries:
selected.append(rel_path)
elif not Path(rel_path).is_file():
skipped.append(f"{rel_path} (file not found)")
else:
skipped.append(f"{rel_path} (not in compile_commands.json)")
github_output = os.environ["GITHUB_OUTPUT"]
with open(github_output, "a", encoding="utf-8") as out:
out.write(f"any_tidy_files={'true' if selected else 'false'}\n")
out.write("all_tidy_files<<TIDY_EOF\n")
out.write("\n".join(selected) + "\n")
out.write("TIDY_EOF\n")
out.write("skipped_files<<SKIP_EOF\n")
out.write("\n".join(skipped) + "\n")
out.write("SKIP_EOF\n")
PY
env:
ALL_CHANGED_FILES: ${{ steps.changed_files.outputs.all_changed_files }}
- name: Show skipped files
if: steps.changed_files.outputs.any_changed == 'true' && steps.tidy_files.outputs.skipped_files != ''
run: |
echo "=== Files skipped by clang-tidy ==="
printf '%s\n' "${{ steps.tidy_files.outputs.skipped_files }}"
- name: Compute submodule commits hash
id: submodule_hash
if: steps.changed_files.outputs.any_changed == 'true'
run: |
hash=$(git submodule status --recursive | awk '{print $1}' | sort | sha256sum | awk '{print $1}')
echo "value=${hash}" >> "$GITHUB_OUTPUT"
- name: Restore generated headers cache
id: cache_headers
if: steps.changed_files.outputs.any_changed == 'true'
uses: actions/cache/restore@v6
with:
path: |
build/external
build/thirdparty
build/src/db/proto
key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }}
- name: Build generated headers only
if: steps.tidy_files.outputs.any_tidy_files == 'true' && steps.cache_headers.outputs.cache-hit != 'true'
run: |
ninja -C build clang_tidy_deps
- name: Save generated headers cache
if: steps.tidy_files.outputs.any_tidy_files == 'true' && steps.cache_headers.outputs.cache-hit != 'true'
uses: actions/cache/save@v6
with:
path: |
build/external
build/thirdparty
build/src/db/proto
key: clang-tidy-headers-${{ runner.os }}-${{ hashFiles('thirdparty/**/*.cmake', 'thirdparty/**/CMakeLists.txt', 'thirdparty/**/*.patch', 'src/db/proto/*.proto') }}-${{ steps.submodule_hash.outputs.value }}
- name: Run clang-tidy on changed files (parallel)
if: steps.tidy_files.outputs.any_tidy_files == 'true'
run: |
mapfile -t files_to_check <<'TIDY_EOF'
${{ steps.tidy_files.outputs.all_tidy_files }}
TIDY_EOF
log_dir=$(mktemp -d)
printf '%s\n' "${files_to_check[@]}" \
| grep -v '^\s*$' \
| xargs -I{} -P "$(nproc)" sh -c '
file="{}"
if [ -f "$file" ]; then
log="'"$log_dir"'/$$.log"
echo "$file" > "$log"
if clang-tidy -p build --quiet --warnings-as-errors="*" "$file" >> "$log" 2>&1; then
echo "PASS: $file"
rm -f "$log"
else
echo "FAIL: $file"
fi
fi'
failed=0
for f in "$log_dir"/*.log; do
[ -e "$f" ] || break
failed=1
src=$(head -1 "$f")
echo ""
echo "::group::clang-tidy errors: $src"
tail -n +2 "$f"
echo "::endgroup::"
done
rm -rf "$log_dir"
if [ "$failed" -eq 1 ]; then
echo "::error::clang-tidy found issues in one or more files"
exit 1
fi
- name: No files to analyse
if: steps.changed_files.outputs.any_changed == 'true' && steps.tidy_files.outputs.any_tidy_files != 'true'
run: echo "Changed C/C++ files not in compile_commands.json. Nothing to analyse."
.github/workflows/community-pr-handler.yml
+148
View File
@@ -0,0 +1,148 @@
name: Community PR Handler
on:
pull_request_target:
types: [opened, reopened, ready_for_review]
permissions:
contents: read
pull-requests: write
jobs:
handle-community-pr:
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
- name: Check if community contributor
id: check
uses: actions/github-script@v9
with:
script: |
const pr = context.payload.pull_request;
const prAuthor = pr.user.login;
const association = pr.author_association;
console.log('========== PR Info ==========');
console.log(`PR Number: #${pr.number}`);
console.log(`PR Title: ${pr.title}`);
console.log(`PR Author: ${prAuthor}`);
console.log(`Author Association: ${association}`);
console.log(`Base Branch: ${pr.base.ref}`);
console.log(`Head Branch: ${pr.head.ref}`);
console.log(`Head Repo: ${pr.head.repo?.full_name || 'same repo'}`);
// OWNER, MEMBER, COLLABORATOR are maintainers; others are community contributors
const maintainerRoles = ['OWNER', 'MEMBER', 'COLLABORATOR'];
const isCommunity = !maintainerRoles.includes(association);
console.log('========== Decision ==========');
console.log(`Maintainer Roles: ${maintainerRoles.join(', ')}`);
console.log(`Is Community Contributor: ${isCommunity}`);
if (!isCommunity) {
console.log('Skipping: PR author is a maintainer, no action needed.');
}
core.setOutput('is_community', isCommunity);
- name: Checkout base branch for CODEOWNERS
if: steps.check.outputs.is_community == 'true'
uses: actions/checkout@v7
with:
ref: ${{ github.event.pull_request.base.ref }}
sparse-checkout: .github/CODEOWNERS
sparse-checkout-cone-mode: false
- name: Assign reviewer as assignee
if: steps.check.outputs.is_community == 'true'
uses: actions/github-script@v9
with:
script: |
const fs = require('fs');
const prNumber = context.payload.pull_request.number;
console.log('========== Fetching Changed Files ==========');
// Get changed files
const { data: files } = await github.rest.pulls.listFiles({
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: prNumber
});
const changedFiles = files.map(f => f.filename);
console.log(`Total changed files: ${changedFiles.length}`);
changedFiles.forEach((f, i) => console.log(` [${i + 1}] ${f}`));
console.log('========== Parsing CODEOWNERS ==========');
// Parse CODEOWNERS
let codeowners = [];
try {
const content = fs.readFileSync('.github/CODEOWNERS', 'utf8');
const lines = content.split('\n').filter(l => l.trim() && !l.trim().startsWith('#'));
console.log(`Found ${lines.length} rules in CODEOWNERS`);
for (const line of lines) {
const parts = line.trim().split(/\s+/);
if (parts.length >= 2) {
const pattern = parts[0];
const owners = parts.slice(1).map(o => o.replace('@', ''));
codeowners.push({ pattern, owners });
console.log(` Rule: "${pattern}" -> [${owners.join(', ')}]`);
}
}
} catch (e) {
console.log('ERROR: Could not read CODEOWNERS:', e.message);
return;
}
console.log('========== Matching Files to Owners ==========');
// Find matching owners for changed files
const matchedOwners = new Set();
for (const file of changedFiles) {
let matchedOwner = null;
let matchedPattern = null;
for (const rule of codeowners) {
const pattern = rule.pattern;
if (pattern === '*') {
matchedOwner = rule.owners[0];
matchedPattern = pattern;
} else if (pattern.endsWith('/')) {
const dir = pattern.replace(/^\//, '').replace(/\/$/, '');
if (file.startsWith(dir + '/') || file.startsWith(dir)) {
matchedOwner = rule.owners[0];
matchedPattern = pattern;
}
} else if (file === pattern || file === pattern.replace(/^\//, '')) {
matchedOwner = rule.owners[0];
matchedPattern = pattern;
}
}
if (matchedOwner) {
matchedOwners.add(matchedOwner);
console.log(` "${file}" -> matched "${matchedPattern}" -> @${matchedOwner}`);
} else {
console.log(` "${file}" -> NO MATCH`);
}
}
console.log('========== Setting Assignees ==========');
// Set assignees
const assignees = Array.from(matchedOwners);
console.log(`Assignees to set: [${assignees.join(', ')}]`);
if (assignees.length > 0) {
try {
await github.rest.issues.addAssignees({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
assignees: assignees
});
console.log('SUCCESS: Assignees set successfully!');
} catch (e) {
console.log('ERROR: Failed to set assignees:', e.message);
console.log('Error details:', JSON.stringify(e, null, 2));
}
} else {
console.log('WARNING: No assignees to set - no matching owners found.');
}
console.log('========== Done ==========');
.github/workflows/continuous_bench.yml
+33
View File
@@ -0,0 +1,33 @@
name: Continuous Benchmark
on:
# Temporarily disabled. Restore original triggers below to re-enable.
# push:
# branches: [ "main"]
# paths-ignore:
# - '**.md'
# workflow_dispatch:
workflow_dispatch:
inputs:
__disabled__:
description: 'This workflow is temporarily disabled.'
required: false
concurrency:
group: cb-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
benchmark:
if: false
runs-on: vdbbench
steps:
- uses: actions/checkout@v7
- name: Run VectorDBBench
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
run: |
bash .github/workflows/scripts/run_vdb.sh
.github/workflows/docker/Dockerfile.linux_x64_glibc228
+83
View File
@@ -0,0 +1,83 @@
# =============================================================================
# Dockerfile.linux_x64_glibc228
# Purpose: Ubuntu 18.10 gcc-9 + glibc 2.28 + CMake 3.30.0 + PyBind11 build environment
# Warning: ubuntu:18.10 is EOL; use only for glibc 2.28 compatibility testing.
# =============================================================================
# Use official Ubuntu 18.10 (Cosmic Cuttlefish)
# glibc version: 2.28 (confirmed via `ldd --version`)
FROM ubuntu:18.10
# Replace Ubuntu mirror with old-releases.ubuntu.com for older glibc compatibility
RUN sed -i 's|http://\(.*\)/ubuntu|http://old-releases.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
sed -i 's|http://security.ubuntu.com/ubuntu|http://old-releases.ubuntu.com/ubuntu|g' /etc/apt/sources.list
# Add Ubuntu 20.04 (focal) repo for GCC 9 ONLY
RUN echo "deb http://archive.ubuntu.com/ubuntu/ focal main universe" >> /etc/apt/sources.list && \
echo "deb http://security.ubuntu.com/ubuntu/ focal-security main universe" >> /etc/apt/sources.list
# Prevent interactive prompts & set non-root user
ENV DEBIAN_FRONTEND=noninteractive \
TZ=Etc/UTC
# Create non-root user for safety (optional but recommended)
RUN useradd -m -u 1000 builder && \
mkdir -p /workspace && chown builder:builder /workspace
# Install base system dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends \
build-essential \
gcc-9 g++-9 \
ninja-build git curl ca-certificates vim wget lcov gnupg clang-format-18\
rsync lsb-release \
uuid-dev zlib1g-dev libssl-dev libffi-dev \
pybind11-dev && \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 \
--slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
rm -rf /var/lib/apt/lists/*
# Install Miniforge (Conda) as root, then assign to builder
ENV MINIFORGE_VERSION="latest"
ENV MINIFORGE_HOME="/opt/miniforge3"
RUN curl -sSL "https://github.com/conda-forge/miniforge/releases/${MINIFORGE_VERSION}/download/Miniforge3-Linux-x86_64.sh" -o miniforge.sh && \
bash miniforge.sh -b -p ${MINIFORGE_HOME} && \
rm miniforge.sh && \
chown -R builder:builder ${MINIFORGE_HOME}
# Switch to non-root user
USER builder
ENV PATH="${MINIFORGE_HOME}/bin:${PATH}"
WORKDIR /workspace
# Create conda envs for supported Python versions
RUN conda create -n py310 python=3.10 -y && \
conda create -n py311 python=3.11 -y && \
conda create -n py312 python=3.12 -y
RUN conda clean --all -f -y
# Install CMake 3.30.0 from Kitware official binary
# Ref: https://github.com/Kitware/CMake/releases/tag/v3.30.0
RUN mkdir -p /tmp/cmake && cd /tmp/cmake && \
curl -sSL -o cmake.tar.gz \
"https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-x86_64.tar.gz" && \
tar -xzf cmake.tar.gz --strip-components=1 -C /tmp/cmake && \
mkdir -p /home/builder/.local && \
mv * /home/builder/.local/ && \
chown -R builder:builder /home/builder/.local && \
rm -rf /tmp/cmake
# Add CMake to PATH
ENV PATH="/home/builder/.local/bin:${PATH}"
# Verify installations
RUN cmake --version && \
conda info && \
conda env list && \
python --version && \
gcc --version && \
ldd --version | head -n1
# Final setup
WORKDIR /workspace
.github/workflows/issue-auto-assign.yml
+99
View File
@@ -0,0 +1,99 @@
name: Issue Auto Assign
on:
issues:
types: [opened]
jobs:
auto-assign:
runs-on: ubuntu-latest
permissions:
issues: write
steps:
- name: Parse issue and assign
uses: actions/github-script@v9
with:
script: |
const issue = context.payload.issue;
const issueBody = issue.body || '';
const issueLabels = (issue.labels || []).map(l => l.name);
// Default assignees per template type (based on labels)
const defaultAssignees = {
'bug': 'zhourrr',
'feature': 'feihongxu0824',
'benchmark': 'egolearner',
'enhancement': 'feihongxu0824',
'integration': 'chinaux',
'profile': 'richyreachy'
};
// Global fallback assignee
const fallbackAssignee = 'feihongxu0824';
// Parse user-selected assignee from issue body
// The input field renders as: "### Preferred Assignee\n\n<entered_value>"
let selectedAssignee = null;
const assigneeMatch = issueBody.match(/### Preferred Assignee\s*\n+([^\n#]+)/);
if (assigneeMatch) {
const selection = assigneeMatch[1].trim();
console.log(`Parsed assignee input: "${selection}"`);
// If user entered a valid GitHub username (not empty, not placeholder text)
if (selection &&
selection !== '_No response_' &&
selection !== 'None' &&
!selection.toLowerCase().includes('leave empty') &&
!selection.startsWith('e.g.,')) {
// Clean up the username (remove @ if present)
selectedAssignee = selection.replace(/^@/, '').trim();
}
}
// Determine final assignee
let finalAssignee = selectedAssignee;
// If no user selection, use default based on label
if (!finalAssignee && issueLabels.length > 0) {
for (const [label, assignee] of Object.entries(defaultAssignees)) {
if (issueLabels.includes(label)) {
finalAssignee = assignee;
console.log(`Matched label "${label}" -> assignee "${assignee}"`);
break;
}
}
}
// Fallback to default assignee if no match
if (!finalAssignee) {
finalAssignee = fallbackAssignee;
console.log(`No match found, using fallback assignee: ${fallbackAssignee}`);
}
console.log(`Issue #${issue.number}: Labels = [${issueLabels.join(', ')}]`);
console.log(`User selected assignee: ${selectedAssignee || 'None (Auto)'}`);
console.log(`Final assignee: ${finalAssignee}`);
// Assign the issue
try {
await github.rest.issues.addAssignees({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
assignees: [finalAssignee]
});
console.log(`Successfully assigned issue #${issue.number} to ${finalAssignee}`);
} catch (error) {
console.error(`Failed to assign issue: ${error.message}`);
// If assignment fails (user may not have permission), add a comment
try {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue.number,
body: `⚠️ Auto-assignment to \`${finalAssignee}\` failed. Please assign manually.`
});
} catch (commentError) {
console.error(`Failed to create comment: ${commentError.message}`);
}
}
.github/workflows/nightly_coverage.yml
+102
View File
@@ -0,0 +1,102 @@
name: Nightly Coverage Report
on:
schedule:
# Runs daily at 00:00 CST (China Standard Time) = 16:00 UTC
- cron: '0 16 * * *'
workflow_dispatch:
permissions:
contents: read
jobs:
coverage:
name: Nightly Coverage Report
runs-on: ubuntu-24.04
strategy:
matrix:
python-version: ['3.10']
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@v7
with:
ref: main # Always use main for nightly
submodules: recursive
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
cache-dependency-path: 'pyproject.toml'
- name: Set up environment variables
run: |
# Set number of processors for parallel builds
NPROC=$(nproc 2>/dev/null || echo 2)
echo "NPROC=$NPROC" >> $GITHUB_ENV
echo "Using $NPROC parallel jobs for builds"
# Add Python user base bin to PATH for pip-installed CLI tools
echo "$(python -c 'import site; print(site.USER_BASE)')/bin" >> $GITHUB_PATH
shell: bash
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
lcov libaio-dev
shell: bash
- name: Install dependencies
run: |
python -m pip install --upgrade pip \
pybind11==3.0 \
cmake==3.30.0 \
ninja==1.11.1 \
pytest \
pytest-cov \
pytest-xdist \
scikit-build-core \
setuptools_scm
shell: bash
- name: Build with COVERAGE config
run: |
cd "$GITHUB_WORKSPACE"
CMAKE_GENERATOR="Unix Makefiles" \
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
python -m pip install -v . \
--no-build-isolation \
--config-settings="cmake.build-type=COVERAGE"
shell: bash
- name: Run Python Tests with Coverage
run: |
cd "$GITHUB_WORKSPACE"
python -m pytest python/tests/ --cov=zvec --cov-report=xml \
--deselect=python/tests/test_gil_release.py::TestGILRelease::test_gil_released_during_query
shell: bash
- name: Run C++ Tests and Generate Coverage
run: |
cd "$GITHUB_WORKSPACE/build"
make unittest -j$NPROC
cd "$GITHUB_WORKSPACE"
# Ensure gcov.sh is executable
chmod +x scripts/gcov.sh
bash scripts/gcov.sh -k
shell: bash
- name: Upload Coverage to Codecov
uses: codecov/codecov-action@v7
with:
files: ./proxima-zvec-filtered.lcov.info,./coverage.xml
flags: python,cpp,nightly
name: nightly-linux-py${{ matrix.python-version }}
token: ${{ secrets.CODECOV_TOKEN }}
.github/workflows/scripts/run_vdb.sh
+88
View File
@@ -0,0 +1,88 @@
set -e
QUANTIZE_TYPE_LIST="int8 int4 fp16 fp32"
CASE_TYPE_LIST="Performance768D1M Performance768D10M Performance1536D500K" # respectively test cosine, ip # Performance960D1M l2 metrics
LOG_FILE="bench.log"
DATE=$(date +%Y-%m-%d_%H-%M-%S)
NPROC=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2)
# COMMIT_ID = branch-date-sha
COMMIT_ID=${GITHUB_REF_NAME}-"$DATE"-$(echo ${GITHUB_WORKFLOW_SHA} | cut -c1-8)
COMMIT_ID=$(echo "$COMMIT_ID" | sed 's/\//_/g')
echo "COMMIT_ID: $COMMIT_ID"
echo "GITHUB_WORKFLOW_SHA: $GITHUB_WORKFLOW_SHA"
echo "workspace: $GITHUB_WORKSPACE"
DB_LABEL_PREFIX="Zvec16c64g-$COMMIT_ID"
# install zvec
git submodule update --init
# for debug
#cd ..
#export SKBUILD_BUILD_DIR="$GITHUB_WORKSPACE/../build"
pwd
python3 -m venv .venv
source .venv/bin/activate
pip install cmake ninja psycopg2-binary loguru fire
pip install -e /opt/VectorDBBench
CMAKE_GENERATOR="Unix Makefiles" \
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
pip install -v "$GITHUB_WORKSPACE"
for CASE_TYPE in $CASE_TYPE_LIST; do
echo "Running VectorDBBench for $CASE_TYPE"
DATASET_DESC=""
if [ "$CASE_TYPE" == "Performance768D1M" ]; then
DATASET_DESC="Performance768D1M - Cohere Cosine"
elif [ "$CASE_TYPE" == "Performance768D10M" ]; then
DATASET_DESC="Performance768D10M - Cohere Cosine"
else
DATASET_DESC="Performance1536D500K - OpenAI IP"
fi
for QUANTIZE_TYPE in $QUANTIZE_TYPE_LIST; do
DB_LABEL="$DB_LABEL_PREFIX-$CASE_TYPE-$QUANTIZE_TYPE"
echo "Running VectorDBBench for $DB_LABEL"
VDB_PARAMS="--path ${DB_LABEL} --db-label ${DB_LABEL} --case-type ${CASE_TYPE} --num-concurrency 12,14,16,18,20"
if [ "$CASE_TYPE" == "Performance768D1M" ]; then
VDB_PARAMS="${VDB_PARAMS} --m 15 --ef-search 180"
elif [ "$CASE_TYPE" == "Performance768D10M" ]; then
VDB_PARAMS="${VDB_PARAMS} --m 50 --ef-search 118 --is-using-refiner"
else #Performance1536D500K using default params + refiner to monitor performance degradation
VDB_PARAMS="${VDB_PARAMS} --m 50 --ef-search 100 --is-using-refiner"
fi
if [ "$QUANTIZE_TYPE" == "fp32" ]; then
vectordbbench zvec ${VDB_PARAMS} 2>&1 | tee $LOG_FILE
else
vectordbbench zvec ${VDB_PARAMS} --quantize-type "${QUANTIZE_TYPE}" 2>&1 | tee $LOG_FILE
fi
RESULT_JSON_PATH=$(grep -o "/opt/VectorDBBench/.*\.json" $LOG_FILE)
QPS=$(jq -r '.results[0].metrics.qps' "$RESULT_JSON_PATH")
RECALL=$(jq -r '.results[0].metrics.recall' "$RESULT_JSON_PATH")
LATENCY_P99=$(jq -r '.results[0].metrics.serial_latency_p99' "$RESULT_JSON_PATH")
LOAD_DURATION=$(jq -r '.results[0].metrics.load_duration' "$RESULT_JSON_PATH")
#quote the var to avoid space in the label
label_list="case_type=\"${CASE_TYPE}\",dataset_desc=\"${DATASET_DESC}\",db_label=\"${DB_LABEL}\",commit=\"${COMMIT_ID}\",date=\"${DATE}\",quantize_type=\"${QUANTIZE_TYPE}\""
# replace `/` with `_` in label_list
label_list=$(echo "$label_list" | sed 's/\//_/g')
cat <<EOF > prom_metrics.txt
# TYPE vdb_bench_qps gauge
vdb_bench_qps{$label_list} $QPS
# TYPE vdb_bench_recall gauge
vdb_bench_recall{$label_list} $RECALL
# TYPE vdb_bench_latency_p99 gauge
vdb_bench_latency_p99{$label_list} $LATENCY_P99
# TYPE vdb_bench_load_duration gauge
vdb_bench_load_duration{$label_list} $LOAD_DURATION
EOF
echo "prom_metrics:"
cat prom_metrics.txt
curl --data-binary @prom_metrics.txt "http://47.93.34.27:9091/metrics/job/benchmarks-${CASE_TYPE}/case_type/${CASE_TYPE}/quantize_type/${QUANTIZE_TYPE}" -v
done
done